title: I am trying to tokenize the text / json file. Dont know why , but only the first tweet is getting tokenized. the code is below
body: <pre><code>from nltk.tokenize import word_tokenize
  import json
  import re
  
  emoticons_str = r"""
      (?:
          [:=;] # Eyes
          [oO\-]? # Nose (optional)
          [D\)\]\(\]/\\OpP] # Mouth
      )"""
  
  regex_str = [
      emoticons_str,
      r'&lt;[^&gt;]+&gt;', # HTML tags
      r'(?:@[\w_]+)', # @-mentions
      r"(?:\#+[\w_]+[\w\'_\-]*[\w_]+)", # hash-tags
      r'http[s]?://(?:[a-z]|[0-9]|[$-_@.&amp;+]|[!*\(\),]|(?:%[0-9a-f][0-  9a-f]))+', # URLs
      r'(?:(?:\d+,?)+(?:\.?\d+)?)', # numbers
      r"(?:[a-z][a-z'\-_]+[a-z])", # words with - and '
      r'(?:[\w_]+)', # other words
      r'(?:\S)' # anything else
      ]
  
  tokens_re = re.compile(r'('+'|'.join(regex_str)+')', re.VERBOSE | re.IGNORECASE)
  emoticon_re = re.compile(r'^'+emoticons_str+'$', re.VERBOSE | re.IGNORECASE)
  
  
  def tokenize(s):
      return tokens_re.findall(s)
  
  
  def preprocess(s, lowercase=False):
      tokens = tokenize(s)
      if lowercase:
          tokens = [token if emoticon_re.search(token) else token.lower() for token in tokens]
      return tokens
  
  
  #tweet = "RT @marcobonzanini: just an example! :D http://example.com #NLP"
  #print(preprocess(tweet))
  # ['RT', '@marcobonzanini', ':', 'just', 'an', 'example', '!', ':D', 'http://example.com', '#NLP']
  
  with open('../script/iphone.txt', 'r') as f: 
     for line in f:
          tweet = json.loads(line)
          tokens = preprocess(tweet['text'])
          #do_something_else(tokens)
          print(json.dumps(tokens, indent=4)
  </code></pre>
  
  <p><img src="https://i.stack.imgur.com/N18Li.png" alt="this is how it looks"></p>