text: import pandas as pd
  csv_in = '/Users/YeTian/.../tweetstxt.csv'
  pddf_in = pd.read_csv(csv_in, index_col=None, header=0, sep=';', encoding='utf-8')
  
  print('tweets pandas dataframe - count:', pddf_in.count())
  print('tweets pandas dataframe - shape:', pddf_in.shape)
  print('tweets pandas dataframe - colns:', pddf_in.columns)
  
  regexp = {"RT": "^RT", "MT": r"^MT", "ALNUM": r"(@[a-zA-Z0-9_]+)",
            "HASHTAG": r"(#[\w\d]+)", "URL": r"([https://|http://]?[a-zA-Z\d\/]+[\.]+[a-zA-Z\d\/\.]+)",
            "SPACES":r"\s+"}
  regexp = dict((key, re.compile(value)) for key, value in regexp.items())
  
  
  ef getAttributeRT(tweet):
      """ see if tweet is a RT """
      return re.search(regexp["RT"], tweet.strip()) != None
  ​
  def getAttributeMT(tweet):
      """ see if tweet is a MT """
      return re.search(regexp["MT"], tweet.strip()) != None
  ​
  def getUserHandles(tweet):
      """ given a tweet we try and extract all user handles in order of occurrence"""
      return re.findall(regexp["ALNUM"], tweet)
  ​
  def getHashtags(tweet):
      """ return all hashtags"""
      return re.findall(regexp["HASHTAG"], tweet)
  ​
  def getURLs(tweet):
      """ URL : [http://]?[\w\.?/]+"""
      return re.findall(regexp["URL"], tweet)
  ​
  def getTextNoURLsUsers(tweet):
      """ return parsed text terms stripped of URLS and User Names in tweet text
          ' '.join(re.sub("(@[A-Za-z0-9]+)|([^0-9A-Za-z \t])|(\w+:\/\/\S+)"," ",x).split()) """
      return ' '.join(re.sub("(@[A-Za-z0-9]+)|([^0-9A-Za-z \t])|(\w+:\/\/\S+)|(RT)"," ", tweet).lower().split())
  ​
  def setTag(tweet):
      """ set tags to tweet_text based on search terms from tags_list"""
      tags_list = ['spark', 'python', 'clinton', 'trump', 'gaga', 'bieber']
      lower_text = tweet.lower()
      return filter(lambda x:x.lower() in lower_text,tags_list)
  ​
  def decode_date(s):
      """ parse Twitter date into format yyyy-mm-dd hh:mm:ss"""
      return time.strftime('%Y-%m-%d %H:%M:%S', time.strptime(s,'%a %b %d %H:%M:%S +0000 %Y'))
  
  
  
  pddf_in.columns
  pddf_in.drop(['Unnamed: 0'], inplace=True, axis=1)
  pddf_in['htag'] = pddf_in.tweet_text.apply(getHashtags)
  pddf_in['user_handles'] = pddf_in.tweet_text.apply(getUserHandles)
  pddf_in['urls'] = pddf_in.tweet_text.apply(getURLs)
  pddf_in['txt_terms'] = pddf_in.tweet_text.apply(getTextNoURLsUsers)
  pddf_in['search_grp'] = pddf_in.tweet_text.apply(setTag)
  pddf_in['date'] = pddf_in.created_at.apply(decode_date)
  
  f_name = '/Users/YeTian/.../tweetstxt.csv'
  pddf_in.to_csv(f_name, sep=';', encoding='utf-8', index=False)
  
  import pandas as pd
  csv_in = '/Users/YeTian/.../spark_tweets.csv'
  tspark_df = pd.read_csv(csv_in, index_col=None, header=0, sep=',', encoding='utf-8')
  
  