synapse/storage/search.py
def search_rooms(self, room_ids, search_term, keys, limit, pagination_token=None):
        """Performs a full text search over events with given keys.

        Args:
            room_id (list): The room_ids to search in
            search_term (str): Search term to search for
            keys (list): List of keys to search in, currently supports
                "content.body", "content.name", "content.topic"
            pagination_token (str): A pagination token previously returned

        Returns:
            list of dicts
        """
        clauses = []

        search_query = search_query = _parse_query(self.database_engine, search_term)

        args = []

        # Make sure we don't explode because the person is in too many rooms.
        # We filter the results below regardless.
        if len(room_ids) < 500:
            clauses.append(
                "room_id IN (%s)" % (",".join(["?"] * len(room_ids)),)
            )
            args.extend(room_ids)

        local_clauses = []
        for key in keys:
            local_clauses.append("key = ?")
            args.append(key)

        clauses.append(
            "(%s)" % (" OR ".join(local_clauses),)
        )

        # take copies of the current args and clauses lists, before adding
        # pagination clauses to main query.
        count_args = list(args)
        count_clauses = list(clauses)

        if pagination_token:
            try:
                origin_server_ts, stream = pagination_token.split(",")
                origin_server_ts = int(origin_server_ts)
                stream = int(stream)
            except Exception:
                raise SynapseError(400, "Invalid pagination token")

            clauses.append(
                "(origin_server_ts < ?"
                " OR (origin_server_ts = ? AND stream_ordering < ?))"
            )
            args.extend([origin_server_ts, origin_server_ts, stream])

        if isinstance(self.database_engine, PostgresEngine):
            sql = (
                "SELECT ts_rank_cd(vector, to_tsquery('english', ?)) as rank,"
                " origin_server_ts, stream_ordering, room_id, event_id"
                " FROM event_search"
                " WHERE vector @@ to_tsquery('english', ?) AND "
            )
            args = [search_query, search_query] + args

            count_sql = (
                "SELECT room_id, count(*) as count FROM event_search"
                " WHERE vector @@ to_tsquery('english', ?) AND "
            )
            count_args = [search_query] + count_args
        elif isinstance(self.database_engine, Sqlite3Engine):
            # We use CROSS JOIN here to ensure we use the right indexes.
            # https://sqlite.org/optoverview.html#crossjoin
            #
            # We want to use the full text search index on event_search to
            # extract all possible matches first, then lookup those matches
            # in the events table to get the topological ordering. We need
            # to use the indexes in this order because sqlite refuses to
            # MATCH unless it uses the full text search index
            sql = (
                "SELECT rank(matchinfo) as rank, room_id, event_id,"
                " origin_server_ts, stream_ordering"
                " FROM (SELECT key, event_id, matchinfo(event_search) as matchinfo"
                " FROM event_search"
                " WHERE value MATCH ?"
                " )"
                " CROSS JOIN events USING (event_id)"
                " WHERE "
            )
            args = [search_query] + args

            count_sql = (
                "SELECT room_id, count(*) as count FROM event_search"
                " WHERE value MATCH ? AND "
            )
            count_args = [search_term] + count_args
        else:
            # This should be unreachable.
            raise Exception("Unrecognized database engine")

        sql += " AND ".join(clauses)
        count_sql += " AND ".join(count_clauses)

        # We add an arbitrary limit here to ensure we don't try to pull the
        # entire table from the database.
        if isinstance(self.database_engine, PostgresEngine):
            sql += (
                " ORDER BY origin_server_ts DESC NULLS LAST,"
                " stream_ordering DESC NULLS LAST LIMIT ?"
            )
        elif isinstance(self.database_engine, Sqlite3Engine):
            sql += " ORDER BY origin_server_ts DESC, stream_ordering DESC LIMIT ?"
        else:
            raise Exception("Unrecognized database engine")

        args.append(limit)

        results = yield self._execute(
            "search_rooms", self.cursor_to_dict, sql, *args
        )

        results = filter(lambda row: row["room_id"] in room_ids, results)

        events = yield self._get_events([r["event_id"] for r in results])

        event_map = {
            ev.event_id: ev
            for ev in events
        }

        highlights = None
        if isinstance(self.database_engine, PostgresEngine):
            highlights = yield self._find_highlights_in_postgres(search_query, events)

        count_sql += " GROUP BY room_id"

        count_results = yield self._execute(
            "search_rooms_count", self.cursor_to_dict, count_sql, *count_args
        )

        count = sum(row["count"] for row in count_results if row["room_id"] in room_ids)

        defer.returnValue({
            "results": [
                {
                    "event": event_map[r["event_id"]],
                    "rank": r["rank"],
                    "pagination_token": "%s,%s" % (
                        r["origin_server_ts"], r["stream_ordering"]
                    ),
                }
                for r in results
                if r["event_id"] in event_map
            ],
            "highlights": highlights,
            "count": count,
        })
Performs a full text search over events with given keys.

        Args:
            room_id (list): The room_ids to search in
            search_term (str): Search term to search for
            keys (list): List of keys to search in, currently supports
                "content.body", "content.name", "content.topic"
            pagination_token (str): A pagination token previously returned

        Returns:
            list of dicts
        
Performs a full text search over events with given keys.
Performs a full text search over events with given keys .
def search_rooms(self, room_ids, search_term, keys, limit, pagination_token=None):
        clauses = []
        search_query = search_query = _parse_query(self.database_engine, search_term)
        args = []
        if len(room_ids) < 500:
            clauses.append(
                "room_id IN (%s)" % (",".join(["?"] * len(room_ids)),)
            )
            args.extend(room_ids)
        local_clauses = []
        for key in keys:
            local_clauses.append("key = ?")
            args.append(key)
        clauses.append(
            "(%s)" % (" OR ".join(local_clauses),)
        )
        count_args = list(args)
        count_clauses = list(clauses)
        if pagination_token:
            try:
                origin_server_ts, stream = pagination_token.split(",")
                origin_server_ts = int(origin_server_ts)
                stream = int(stream)
            except Exception:
                raise SynapseError(400, "Invalid pagination token")
            clauses.append(
                "(origin_server_ts < ?"
                " OR (origin_server_ts = ? AND stream_ordering < ?))"
            )
            args.extend([origin_server_ts, origin_server_ts, stream])
        if isinstance(self.database_engine, PostgresEngine):
            sql = (
                "SELECT ts_rank_cd(vector, to_tsquery('english', ?)) as rank,"
                " origin_server_ts, stream_ordering, room_id, event_id"
                " FROM event_search"
                " WHERE vector @@ to_tsquery('english', ?) AND "
            )
            args = [search_query, search_query] + args
            count_sql = (
                "SELECT room_id, count(*) as count FROM event_search"
                " WHERE vector @@ to_tsquery('english', ?) AND "
            )
            count_args = [search_query] + count_args
        elif isinstance(self.database_engine, Sqlite3Engine):
             We want to use the full text search index on event_search to
             extract all possible matches first, then lookup those matches
             in the events table to get the topological ordering. We need
             to use the indexes in this order because sqlite refuses to
             MATCH unless it uses the full text search index
            sql = (
                "SELECT rank(matchinfo) as rank, room_id, event_id,"
                " origin_server_ts, stream_ordering"
                " FROM (SELECT key, event_id, matchinfo(event_search) as matchinfo"
                " FROM event_search"
                " WHERE value MATCH ?"
                " )"
                " CROSS JOIN events USING (event_id)"
                " WHERE "
            )
            args = [search_query] + args
            count_sql = (
                "SELECT room_id, count(*) as count FROM event_search"
                " WHERE value MATCH ? AND "
            )
            count_args = [search_term] + count_args
        else:
             This should be unreachable.
            raise Exception("Unrecognized database engine")
        sql += " AND ".join(clauses)
        count_sql += " AND ".join(count_clauses)
         We add an arbitrary limit here to ensure we don't try to pull the
         entire table from the database.
        if isinstance(self.database_engine, PostgresEngine):
            sql += (
                " ORDER BY origin_server_ts DESC NULLS LAST,"
                " stream_ordering DESC NULLS LAST LIMIT ?"
            )
        elif isinstance(self.database_engine, Sqlite3Engine):
            sql += " ORDER BY origin_server_ts DESC, stream_ordering DESC LIMIT ?"
        else:
            raise Exception("Unrecognized database engine")
        args.append(limit)
        results = yield self._execute(
            "search_rooms", self.cursor_to_dict, sql, *args
        )
        results = filter(lambda row: row["room_id"] in room_ids, results)
        events = yield self._get_events([r["event_id"] for r in results])
        event_map = {
            ev.event_id: ev
            for ev in events
        }
        highlights = None
        if isinstance(self.database_engine, PostgresEngine):
            highlights = yield self._find_highlights_in_postgres(search_query, events)
        count_sql += " GROUP BY room_id"
        count_results = yield self._execute(
            "search_rooms_count", self.cursor_to_dict, count_sql, *count_args
        )
        count = sum(row["count"] for row in count_results if row["room_id"] in room_ids)
        defer.returnValue({
            "results": [
                {
                    "event": event_map[r["event_id"]],
                    "rank": r["rank"],
                    "pagination_token": "%s,%s" % (
                        r["origin_server_ts"], r["stream_ordering"]
                    ),
                }
                for r in results
                if r["event_id"] in event_map
            ],
            "highlights": highlights,
            "count": count,
        })
def search_rooms ( self , room_ids , search_term , keys , limit , pagination_token = None ) : clauses = [ ] search_query = search_query = _parse_query ( self . database_engine , search_term ) args = [ ] if len ( room_ids ) < 500 : clauses . append ( "room_id IN (%s)" % ( "," . join ( [ "?" ] * len ( room_ids ) ) , ) ) args . extend ( room_ids ) local_clauses = [ ] for key in keys : local_clauses . append ( "key = ?" ) args . append ( key ) clauses . append ( "(%s)" % ( " OR " . join ( local_clauses ) , ) ) count_args = list ( args ) count_clauses = list ( clauses ) if pagination_token : try : origin_server_ts , stream = pagination_token . split ( "," ) origin_server_ts = int ( origin_server_ts ) stream = int ( stream ) except Exception : raise SynapseError ( 400 , "Invalid pagination token" ) clauses . append ( "(origin_server_ts < ?" " OR (origin_server_ts = ? AND stream_ordering < ?))" ) args . extend ( [ origin_server_ts , origin_server_ts , stream ] ) if isinstance ( self . database_engine , PostgresEngine ) : sql = ( "SELECT ts_rank_cd(vector, to_tsquery('english', ?)) as rank," " origin_server_ts, stream_ordering, room_id, event_id" " FROM event_search" " WHERE vector @@ to_tsquery('english', ?) AND " ) args = [ search_query , search_query ] + args count_sql = ( "SELECT room_id, count(*) as count FROM event_search" " WHERE vector @@ to_tsquery('english', ?) AND " ) count_args = [ search_query ] + count_args elif isinstance ( self . database_engine , Sqlite3Engine ) : sql = ( "SELECT rank(matchinfo) as rank, room_id, event_id," " origin_server_ts, stream_ordering" " FROM (SELECT key, event_id, matchinfo(event_search) as matchinfo" " FROM event_search" " WHERE value MATCH ?" " )" " CROSS JOIN events USING (event_id)" " WHERE " ) args = [ search_query ] + args count_sql = ( "SELECT room_id, count(*) as count FROM event_search" " WHERE value MATCH ? AND " ) count_args = [ search_term ] + count_args else : raise Exception ( "Unrecognized database engine" ) sql += " AND " . join ( clauses ) count_sql += " AND " . join ( count_clauses ) if isinstance ( self . database_engine , PostgresEngine ) : sql += ( " ORDER BY origin_server_ts DESC NULLS LAST," " stream_ordering DESC NULLS LAST LIMIT ?" ) elif isinstance ( self . database_engine , Sqlite3Engine ) : sql += " ORDER BY origin_server_ts DESC, stream_ordering DESC LIMIT ?" else : raise Exception ( "Unrecognized database engine" ) args . append ( limit ) results = yield self . _execute ( "search_rooms" , self . cursor_to_dict , sql , * args ) results = filter ( lambda row : row [ "room_id" ] in room_ids , results ) events = yield self . _get_events ( [ r [ "event_id" ] for r in results ] ) event_map = { ev . event_id : ev for ev in events } highlights = None if isinstance ( self . database_engine , PostgresEngine ) : highlights = yield self . _find_highlights_in_postgres ( search_query , events ) count_sql += " GROUP BY room_id" count_results = yield self . _execute ( "search_rooms_count" , self . cursor_to_dict , count_sql , * count_args ) count = sum ( row [ "count" ] for row in count_results if row [ "room_id" ] in room_ids ) defer . returnValue ( { "results" : [ { "event" : event_map [ r [ "event_id" ] ] , "rank" : r [ "rank" ] , "pagination_token" : "%s,%s" % ( r [ "origin_server_ts" ] , r [ "stream_ordering" ] ) , } for r in results if r [ "event_id" ] in event_map ] , "highlights" : highlights , "count" : count , } )
Performs a full text search over events with given keys.
Performs a full text search over events with given keys .
"""Performs a full text search over events with given keys.

        Args:
            room_id (list): The room_ids to search in
            search_term (str): Search term to search for
            keys (list): List of keys to search in, currently supports
                "content.body", "content.name", "content.topic"
            pagination_token (str): A pagination token previously returned

        Returns:
            list of dicts
        """ # Make sure we don't explode because the person is in too many rooms. # We filter the results below regardless. # take copies of the current args and clauses lists, before adding # pagination clauses to main query. # We use CROSS JOIN here to ensure we use the right indexes. # https://sqlite.org/optoverview.html#crossjoin # # We want to use the full text search index on event_search to # extract all possible matches first, then lookup those matches # in the events table to get the topological ordering. We need # to use the indexes in this order because sqlite refuses to # MATCH unless it uses the full text search index # This should be unreachable. # We add an arbitrary limit here to ensure we don't try to pull the # entire table from the database.
list of dicts
list of dicts
self
room_ids
search_term
Search term to search for
Search term to search for
keys
List of keys to search in, currently supports
"content.body", "content.name", "content.topic"
List of keys to search in currently supports " content . body " " content . name " " content . topic "
limit
pagination_token
A pagination token previously returned
A pagination token previously returned
room_id
The room_ids to search in
The room_ids to search in