src/python/txtai/scoring.py
def index(self, documents):
        """
        Indexes a collection of documents using a scoring method. Documents are tuples of (id, text|tokens, tags).

        Args:
            documents: input documents
        """

        # Calculate word frequency, total tokens and total documents
        for _, tokens, tags in documents:
            # Convert to tokens if necessary
            if isinstance(tokens, str):
                tokens = Tokenizer.tokenize(tokens)

            # Total number of times token appears, count all tokens
            self.wordfreq.update(tokens)

            # Total number of documents a token is in, count unique tokens
            self.docfreq.update(set(tokens))

            # Get list of unique tags
            if tags:
                self.tags.update(tags.split())

            # Total document count
            self.total += 1

        # Calculate total token frequency
        self.tokens = sum(self.wordfreq.values())

        # Calculate average frequency per token
        self.avgfreq = self.tokens / len(self.wordfreq.values())

        # Calculate average document length in tokens
        self.avgdl = self.tokens / self.total

        # Compute IDF scores
        for word, freq in self.docfreq.items():
            self.idf[word] = self.computeIDF(freq)

        # Average IDF score per token
        self.avgidf = sum(self.idf.values()) / len(self.idf)

        # Filter for tags that appear in at least 1% of the documents
        self.tags = {tag:number for tag, number in self.tags.items() if number >= self.total * 0.005}

        Indexes a collection of documents using a scoring method. Documents are tuples of (id, text|tokens, tags).

        Args:
            documents: input documents
        
Indexes a collection of documents using a scoring method. Documents are tuples of (id, text|tokens, tags).
Indexes a collection of documents using a scoring method . Documents are tuples of ( id text|tokens tags ) .
def index(self, documents):
        for _, tokens, tags in documents:
            if isinstance(tokens, str):
                tokens = Tokenizer.tokenize(tokens)
            self.wordfreq.update(tokens)
            self.docfreq.update(set(tokens))
            if tags:
                self.tags.update(tags.split())
            self.total += 1
        self.tokens = sum(self.wordfreq.values())
        self.avgfreq = self.tokens / len(self.wordfreq.values())
        self.avgdl = self.tokens / self.total
        for word, freq in self.docfreq.items():
            self.idf[word] = self.computeIDF(freq)
        self.avgidf = sum(self.idf.values()) / len(self.idf)
        self.tags = {tag:number for tag, number in self.tags.items() if number >= self.total * 0.005}
def index ( self , documents ) : for _ , tokens , tags in documents : if isinstance ( tokens , str ) : tokens = Tokenizer . tokenize ( tokens ) self . wordfreq . update ( tokens ) self . docfreq . update ( set ( tokens ) ) if tags : self . tags . update ( tags . split ( ) ) self . total += 1 self . tokens = sum ( self . wordfreq . values ( ) ) self . avgfreq = self . tokens / len ( self . wordfreq . values ( ) ) self . avgdl = self . tokens / self . total for word , freq in self . docfreq . items ( ) : self . idf [ word ] = self . computeIDF ( freq ) self . avgidf = sum ( self . idf . values ( ) ) / len ( self . idf ) self . tags = { tag : number for tag , number in self . tags . items ( ) if number >= self . total * 0.005 }
Indexes a collection of documents using a scoring method.
Indexes a collection of documents using a scoring method .
"""
        Indexes a collection of documents using a scoring method. Documents are tuples of (id, text|tokens, tags).

        Args:
            documents: input documents
        """ # Calculate word frequency, total tokens and total documents # Convert to tokens if necessary # Total number of times token appears, count all tokens # Total number of documents a token is in, count unique tokens # Get list of unique tags # Total document count # Calculate total token frequency # Calculate average frequency per token # Calculate average document length in tokens # Compute IDF scores # Average IDF score per token # Filter for tags that appear in at least 1% of the documents
self
documents
None