src/spark_NB.py
def create_pipeline():
    """
    creates model pipeline
    Currently uses RegexTokenizer to get bytewords as tokens, hashingTF to
    featurize the tokens as word counts, and NaiveBayes to fit and classify

    This is where most of the work will be done in improving the model
    """

    tokenizer = RegexTokenizer(inputCol="text", outputCol="words",
                               pattern="(?<=\\s)..", gaps=False)
    ngram = NGram(n=2, inputCol="words", outputCol="grams")
    hashingTF = HashingTF(numFeatures=65792, inputCol=ngram.getOutputCol(),
                          outputCol="features")
    nb = NaiveBayes(smoothing=1)
    pipeline = Pipeline(stages=[tokenizer, ngram, hashingTF, nb])

    return(pipeline)

    creates model pipeline
    Currently uses RegexTokenizer to get bytewords as tokens, hashingTF to
    featurize the tokens as word counts, and NaiveBayes to fit and classify

    This is where most of the work will be done in improving the model
    
creates model pipeline
Currently uses RegexTokenizer to get bytewords as tokens, hashingTF to
featurize the tokens as word counts, and NaiveBayes to fit and classify

This is where most of the work will be done in improving the model
creates model pipeline Currently uses RegexTokenizer to get bytewords as tokens hashingTF to featurize the tokens as word counts and NaiveBayes to fit and classify This is where most of the work will be done in improving the model
def create_pipeline():
    tokenizer = RegexTokenizer(inputCol="text", outputCol="words",
                               pattern="(?<=\\s)..", gaps=False)
    ngram = NGram(n=2, inputCol="words", outputCol="grams")
    hashingTF = HashingTF(numFeatures=65792, inputCol=ngram.getOutputCol(),
                          outputCol="features")
    nb = NaiveBayes(smoothing=1)
    pipeline = Pipeline(stages=[tokenizer, ngram, hashingTF, nb])
    return(pipeline)
def create_pipeline ( ) : tokenizer = RegexTokenizer ( inputCol = "text" , outputCol = "words" , pattern = "(?<=\\s).." , gaps = False ) ngram = NGram ( n = 2 , inputCol = "words" , outputCol = "grams" ) hashingTF = HashingTF ( numFeatures = 65792 , inputCol = ngram . getOutputCol ( ) , outputCol = "features" ) nb = NaiveBayes ( smoothing = 1 ) pipeline = Pipeline ( stages = [ tokenizer , ngram , hashingTF , nb ] ) return ( pipeline )
creates model pipeline
Currently uses RegexTokenizer to get bytewords as tokens, hashingTF to
featurize the tokens as word counts, and NaiveBayes to fit and classify
creates model pipeline Currently uses RegexTokenizer to get bytewords as tokens hashingTF to featurize the tokens as word counts and NaiveBayes to fit and classify
"""
    creates model pipeline
    Currently uses RegexTokenizer to get bytewords as tokens, hashingTF to
    featurize the tokens as word counts, and NaiveBayes to fit and classify

    This is where most of the work will be done in improving the model
    """