ktext/preprocess.py
def fit(self,
            data: List[str],
            return_tokenized_data: bool = False) -> Union[None, List[List[str]]]:
        """
        TODO: update docs

        Apply cleaner and tokenzier to raw data and build vocabulary.

        Parameters
        ----------
        data : List[str]
            These are raw documents, which are a list of strings. ex:
            [["The quick brown fox"], ["jumps over the lazy dog"]]
        return_tokenized_data : bool
            Return the tokenized strings.  This is primarly used for debugging
            purposes.

        Returns
        -------
        None or List[List[str]]
            if return_tokenized_data=True then will return tokenized documents,
            otherwise will not return anything.
        """
        self.__clear_data()
        now = get_time()
        logging.warning(f'....tokenizing data')
        tokenized_data = self.parallel_process_text(data)

        if not self.padding_maxlen:
            # its not worth the overhead to parallelize document length counts
            length_counts = map(count_len, tokenized_data)
            self.document_length_histogram = Counter(length_counts)
            self.generate_doc_length_stats()

        # Learn corpus on single thread
        logging.warning(f'(1/2) done. {time_diff(now)} sec')
        logging.warning(f'....building corpus')
        now = get_time()
        self.indexer = custom_Indexer(num_words=self.keep_n)
        self.indexer.fit_on_tokenized_texts(tokenized_data)

        # Build Dictionary accounting For 0 padding, and reserve 1 for unknown and rare Words
        self.token2id = self.indexer.word_index
        self.id2token = {v: k for k, v in self.token2id.items()}
        self.n_tokens = max(self.indexer.word_index.values())

        # logging
        logging.warning(f'(2/2) done. {time_diff(now)} sec')
        logging.warning(f'Finished parsing {self.indexer.document_count:,} documents.')

        if return_tokenized_data:
            return tokenized_data
def fit(self,
            data: List[str],
            return_tokenized_data: bool = False) -> Union[None, List[List[str]]]:
        """
        TODO: update docs

        Apply cleaner and tokenzier to raw data and build vocabulary.

        Parameters
        ----------
        data : List[str]
            These are raw documents, which are a list of strings. ex:
            [["The quick brown fox"], ["jumps over the lazy dog"]]
        return_tokenized_data : bool
            Return the tokenized strings.  This is primarly used for debugging
            purposes.

        Returns
        -------
        None or List[List[str]]
            if return_tokenized_data=True then will return tokenized documents,
            otherwise will not return anything.
        """
        self.__clear_data()
        now = get_time()
        logging.warning(f'....tokenizing data')
        tokenized_data = self.parallel_process_text(data)

        if not self.padding_maxlen:
            # its not worth the overhead to parallelize document length counts
            length_counts = map(count_len, tokenized_data)
            self.document_length_histogram = Counter(length_counts)
            self.generate_doc_length_stats()

        # Learn corpus on single thread
        logging.warning(f'(1/2) done. {time_diff(now)} sec')
        logging.warning(f'....building corpus')
        now = get_time()
        self.indexer = custom_Indexer(num_words=self.keep_n)
        self.indexer.fit_on_tokenized_texts(tokenized_data)

        # Build Dictionary accounting For 0 padding, and reserve 1 for unknown and rare Words
        self.token2id = self.indexer.word_index
        self.id2token = {v: k for k, v in self.token2id.items()}
        self.n_tokens = max(self.indexer.word_index.values())

        # logging
        logging.warning(f'(2/2) done. {time_diff(now)} sec')
        logging.warning(f'Finished parsing {self.indexer.document_count:,} documents.')

        if return_tokenized_data:
            return tokenized_data
def fit ( self , data : List [ str ] , return_tokenized_data : bool = False ) -> Union [ None , List [ List [ str ] ] ] : self . __clear_data ( ) now = get_time ( ) logging . warning ( f'....tokenizing data' ) tokenized_data = self . parallel_process_text ( data ) if not self . padding_maxlen : # its not worth the overhead to parallelize document length counts length_counts = map ( count_len , tokenized_data ) self . document_length_histogram = Counter ( length_counts ) self . generate_doc_length_stats ( ) # Learn corpus on single thread logging . warning ( f'(1/2) done. {time_diff(now)} sec' ) logging . warning ( f'....building corpus' ) now = get_time ( ) self . indexer = custom_Indexer ( num_words = self . keep_n ) self . indexer . fit_on_tokenized_texts ( tokenized_data ) # Build Dictionary accounting For 0 padding, and reserve 1 for unknown and rare Words self . token2id = self . indexer . word_index self . id2token = { v : k for k , v in self . token2id . items ( ) } self . n_tokens = max ( self . indexer . word_index . values ( ) ) # logging logging . warning ( f'(2/2) done. {time_diff(now)} sec' ) logging . warning ( f'Finished parsing {self.indexer.document_count:,} documents.' ) if return_tokenized_data : return tokenized_data
TODO: update docs

        Apply cleaner and tokenzier to raw data and build vocabulary.

        Parameters
        ----------
        data : List[str]
            These are raw documents, which are a list of strings. ex:
            [["The quick brown fox"], ["jumps over the lazy dog"]]
        return_tokenized_data : bool
            Return the tokenized strings.  This is primarly used for debugging
            purposes.

        Returns
        -------
        None or List[List[str]]
            if return_tokenized_data=True then will return tokenized documents,
            otherwise will not return anything.
TODO : update docs