splearn/feature_extraction/text.py
def _init_vocab(self, analyzed_docs):
        """Create vocabulary
        """
        class SetAccum(AccumulatorParam):

            def zero(self, initialValue):
                return set(initialValue)

            def addInPlace(self, v1, v2):
                v1 |= v2
                return v1

        if not self.fixed_vocabulary_:
            accum = analyzed_docs._rdd.context.accumulator(set(), SetAccum())
            analyzed_docs.foreach(
                lambda x: accum.add(set(chain.from_iterable(x))))
            vocabulary = {t: i for i, t in enumerate(accum.value)}
        else:
            vocabulary = self.vocabulary_

        if not vocabulary:
            raise ValueError("empty vocabulary; perhaps the documents only"
                             " contain stop words")
        return vocabulary
def _init_vocab(self, analyzed_docs):
        """Create vocabulary
        """
        class SetAccum(AccumulatorParam):

            def zero(self, initialValue):
                return set(initialValue)

            def addInPlace(self, v1, v2):
                v1 |= v2
                return v1

        if not self.fixed_vocabulary_:
            accum = analyzed_docs._rdd.context.accumulator(set(), SetAccum())
            analyzed_docs.foreach(
                lambda x: accum.add(set(chain.from_iterable(x))))
            vocabulary = {t: i for i, t in enumerate(accum.value)}
        else:
            vocabulary = self.vocabulary_

        if not vocabulary:
            raise ValueError("empty vocabulary; perhaps the documents only"
                             " contain stop words")
        return vocabulary
def _init_vocab ( self , analyzed_docs ) : class SetAccum ( AccumulatorParam ) : def zero ( self , initialValue ) : return set ( initialValue ) def addInPlace ( self , v1 , v2 ) : v1 |= v2 return v1 if not self . fixed_vocabulary_ : accum = analyzed_docs . _rdd . context . accumulator ( set ( ) , SetAccum ( ) ) analyzed_docs . foreach ( lambda x : accum . add ( set ( chain . from_iterable ( x ) ) ) ) vocabulary = { t : i for i , t in enumerate ( accum . value ) } else : vocabulary = self . vocabulary_ if not vocabulary : raise ValueError ( "empty vocabulary; perhaps the documents only" " contain stop words" ) return vocabulary
Create vocabulary
Create vocabulary