title: Bigram Tokenization and Unigram Tokenizer
body: <p>I'm having an issue of the Bigram tokenization displaying the same results as the ngram tokenization. It keeps showing only word vs two words on the graph. I'm a newbie at R and so I was trying to follow along with this tutorial:</p>
  
  <p><a href="https://rstudio-pubs-static.s3.amazonaws.com/40817_63c8586e26ea49d0a06bcba4e794e43d.html" rel="nofollow noreferrer">https://rstudio-pubs-static.s3.amazonaws.com/40817_63c8586e26ea49d0a06bcba4e794e43d.html</a></p>
  
  <p>I've used most of the code from the post, but have also tried to use some from a different source that I've been playing with. I did read that changing the corpus to a VCorpus cleared this issue up for several people. I did try changing this code: <code>corpus &lt;- Corpus(review_source) to corpus &lt;- VCorpus(review_source)</code> with the same results. Any guidance would be appreciated. Here is the code I'm currently working with</p>
  
  <pre><code>##Load in the data setwd("G:\\Customer Analytics\\AdHoc")
  reviews &lt;- read.csv("Reseller Ratings_clean.csv", stringsAsFactors = FALSE, 
  header=TRUE)
  
  review_text &lt;- paste(reviews$reviews)
  
  #setting up source and corpus
  review_source &lt;- VectorSource(review_text)
  corpus &lt;- Corpus(review_source)
  
  ########################
  # Data Cleaning - Applying Transformations
  ########################
  toSpace &lt;- content_transformer(function(x, pattern) gsub(pattern, " ", x))
  corpus &lt;- tm_map(corpus, toSpace, "/|@|//|$|:|:)|*|&amp;|!|?|_|-|#|")  ## 
  replace special characters by space
  corpus &lt;- tm_map(corpus, content_transformer(tolower)) # Conversion to Lower 
  Case
  corpus &lt;- tm_map(corpus, removePunctuation) # Punctuation can provide 
  gramatical context which supports 
  corpus &lt;- tm_map(corpus, removeWords, stopwords("english")) # common stop 
  Words like for, very, and, of, are, etc,
  corpus &lt;- tm_map(corpus, removeWords, c("the", "will", "The", "also", 
  "that", "and", "for", "in", "is", "it", "not", "to"))
  corpus &lt;- tm_map(corpus, removeNumbers) # removal of numbers
  corpus &lt;- tm_map(corpus, stripWhitespace) # removal of whitespace
  corpus &lt;- tm_map(corpus, stemDocument) # Stemming uses an algorithm that 
  removes common word endings for English
  
  #Making a document-term matrix
  dtm &lt;- DocumentTermMatrix(corpus)
  
  ################################
  # The transpose of the document term matrix 
  ################################
  tdm &lt;- TermDocumentMatrix(corpus)
  
  # Frequency
  freq &lt;- sort(colSums(as.matrix(dtm)), decreasing=TRUE)
  wf &lt;- data.frame(word=names(freq), freq=freq)
  
  # Plot Histogram
  subset(wf, freq&gt;500) %&gt;%
  ggplot(aes(word, freq)) +
  geom_bar(stat="identity", fill="darkred", colour="darkgreen") +
  theme(axis.text.x=element_text(angle=45, hjust=1))
  
  # Create Wordcloud
  library(wordcloud)
  set.seed(100)
  wordcloud(names(freq), freq, min.freq=100, colors=brewer.pal(6, "Dark2"))
  
  ###############################
  # N-gram tokenization of the Corpus 
  ###############################
  OnegramTokenizer &lt;- function(x) NGramTokenizer(x, 
                                               Weka_control(min = 1, max =1))
  dtm &lt;- DocumentTermMatrix(docs, control = list(tokenize = OnegramTokenizer))
  freq &lt;- sort(colSums(as.matrix(dtm)), decreasing=TRUE)
  wof &lt;- data.frame(word=names(freq), freq=freq)
  
  pl &lt;- ggplot(subset(wof, freq &gt; 500), aes(word, freq))
  pl &lt;- pl + geom_bar(stat="identity", fill="darkred", colour="blue")
  pl + theme(axis.text.x=element_text(angle=45, hjust=1)) + ggtitle("Uni-Gram 
  Frequency")
  
  ############################
  # Bi-Gram Tokenization of the Corpus
  ############################
  
  BigramTokenizer &lt;- function(x) NGramTokenizer(x, 
                                              Weka_control(min = 2, max = 2))
  dtm &lt;- DocumentTermMatrix(docs, control = list(tokenize = BigramTokenizer))
  freq &lt;- sort(colSums(as.matrix(dtm)), decreasing=TRUE)
  wof &lt;- data.frame(word=names(freq), freq=freq)
  
  pl &lt;- ggplot(subset(wof, freq &gt;500) ,aes(word, freq))
  pl &lt;- pl + geom_bar(stat="identity", fill="darkgreen", colour="blue")
  pl + theme(axis.text.x=element_text(angle=45, hjust=1)) + ggtitle("Bi-Gram 
  Frequency")
  </code></pre>