ktext/preprocess.py
def transform_parallel(self, data: List[str]) -> List[List[int]]:
        """
        Transform List of documents into List[List[int]].  Uses process based
        threading on all available cores.  If only processing a small number of
        documents ( < 10k ) then consider using the method `transform` instead.

        ex:
        >> pp = processor()
        >> pp.fit(docs)
        >> new_docs = [["The quick brown fox"], ["jumps over the lazy dog"]]
        >> pp.transform_parallel(new_docs)
        [[1, 2, 3, 4], [5, 6, 1, 7, 8]]
        """
        logging.warning(f'...tokenizing data')
        tokenized_data = self.parallel_process_text(data)
        logging.warning(f'...indexing data')
        indexed_data = self.indexer.tokenized_texts_to_sequences(tokenized_data)
        logging.warning(f'...padding data')
        return self.pad(indexed_data)
def transform_parallel(self, data: List[str]) -> List[List[int]]:
        """
        Transform List of documents into List[List[int]].  Uses process based
        threading on all available cores.  If only processing a small number of
        documents ( < 10k ) then consider using the method `transform` instead.

        ex:
        >> pp = processor()
        >> pp.fit(docs)
        >> new_docs = [["The quick brown fox"], ["jumps over the lazy dog"]]
        >> pp.transform_parallel(new_docs)
        [[1, 2, 3, 4], [5, 6, 1, 7, 8]]
        """
        logging.warning(f'...tokenizing data')
        tokenized_data = self.parallel_process_text(data)
        logging.warning(f'...indexing data')
        indexed_data = self.indexer.tokenized_texts_to_sequences(tokenized_data)
        logging.warning(f'...padding data')
        return self.pad(indexed_data)
def transform_parallel ( self , data : List [ str ] ) -> List [ List [ int ] ] : logging . warning ( f'...tokenizing data' ) tokenized_data = self . parallel_process_text ( data ) logging . warning ( f'...indexing data' ) indexed_data = self . indexer . tokenized_texts_to_sequences ( tokenized_data ) logging . warning ( f'...padding data' ) return self . pad ( indexed_data )
Transform List of documents into List[List[int]].  Uses process based
        threading on all available cores.  If only processing a small number of
        documents ( < 10k ) then consider using the method `transform` instead.

        ex:
        >> pp = processor()
        >> pp.fit(docs)
        >> new_docs = [["The quick brown fox"], ["jumps over the lazy dog"]]
        >> pp.transform_parallel(new_docs)
        [[1, 2, 3, 4], [5, 6, 1, 7, 8]]
Transform List of documents into List [ List [ int ]] . Uses process based threading on all available cores . If only processing a small number of documents ( < 10k ) then consider using the method transform instead .