text: import pandas as pd
  import ijson
  import json, re, csv
  from nltk.corpus import wordnet as wn
  from googletrans import Translator
  import string
  import unicodedata
  import nltk.stem.snowball as snowball
  import nltk
  import numpy as np
  import tensorflow as tf
  import pickle
  from xlsxwriter.workbook import Workbook
  from helper_functions import *
  
  #business dict_keys(['business_id', 'name', 'neighborhood', 'address', 'city', 'state', 'postal_code', 'latitude', 'longitude', 'stars', 'review_count', 'is_open', 'attributes', 'categories', 'hours'])
  #review dict_keys(['review_id', 'user_id', 'business_id', 'stars', 'date', 'text', 'useful', 'funny', 'cool'])
  #tip dict_keys(['text', 'date', 'likes', 'business_id', 'user_id'])
  #photos dict_keys(['photo_id', 'business_id', 'caption', 'label'])
  #checkin dict_keys(['time', 'business_id'])
  #user dict_keys(['user_id', 'name', 'review_count', 'yelping_since', 'friends', 'useful', 'funny', 'cool', 'fans', 'elite', 'average_stars', 'compliment_hot', 'compliment_more', 'compliment_profile', 'compliment_cute', 'compliment_list', 'compliment_note', 'compliment_plain', 'compliment_cool', 'compliment_funny', 'compliment_writer', 'compliment_photos'])
  
  city2category2count    = dict()
  city2business_count    = dict()
  business2review_count  = dict()
  business_id2categories = dict()
  word2rating            = dict()
  category2reviews       = dict()
  business2reviews_count = dict()
  category2reviews_count = dict()
  business_id2name       = dict()
  business_id2city       = dict()
  
  reviews = []
  avg_reviews_for_users = [0.0, 0]
  with open(r"dataset\user.json", 'r', encoding='utf-8') as f:
      for line in f:
          j = json.loads(line)
          avg_reviews_for_users[0] += j["review_count"]
          avg_reviews_for_users[1] += 1
  
  print("Average review count for users:",round(avg_reviews_for_users[0]/avg_reviews_for_users[1],5))
  
  with open(r"dataset\business.json", 'r', encoding='utf-8') as f:
      for line in f:
          j = json.loads(line)
          reviews.append(j)
          city = j["city"]
          if city not in city2category2count:
              city2category2count[city] = dict()
              city2business_count[city] = 0
          city2business_count[city] += 1
          categories  = j["categories"]
          business_id = j["business_id"]
          name = j["name"]
          business_id2name[business_id] = name
          business_id2city[business_id] = city
          business_id2categories[business_id] = categories
          business2reviews_count[business_id] = [0] * 5
          for category in categories:
              if category not in category2reviews_count:
                  category2reviews_count[category] = [0] * 5
  
  business2word_count = dict()
  
  texts = []
  reviews = []
  with open(r"dataset\review.json", 'r', encoding='utf-8') as f:
      for line in f:
          review = json.loads(line)
          texts.append(review["text"])
          reviews.append(review)
          business_id = review["business_id"]
          stars = review["stars"]
          business2reviews_count[business_id][stars - 1] += 1
          for category in business_id2categories[business_id]:
              if category not in category2reviews:
                  category2reviews[category] = []
              category2reviews[category].append(review)
              category2reviews_count[category][stars - 1] += 1
  
  
  common_header = []
  for i in range(1,6):
      common_header.append(str(i) + " Star Reviews")
  
  common_header.append("Positive Reviews")
  common_header.append("Negative Reviews")
  common_header.append("All Reviews Count")
  common_header.append("Average Rating")
  
  city2reviews_count = dict()
  city2businesses    = dict()
  for business in business2reviews_count:
      city = business_id2city[business]
      if city not in city2reviews_count:
          city2reviews_count[city] = [0] * 5
          city2businesses[city]    = 0
      city2businesses[city]    += 1
      for k in range(5):
          city2reviews_count[city][k] += business2reviews_count[business][k]
  
  category_rows    = [["Category"] + common_header]
  business_id_rows = [["Business Name","Business ID","Number of Categories","Categories","City"] + common_header]
  city_rows        = [["City","Businesses" ] + common_header]
  
  for category in category2reviews_count:
      star_counts = category2reviews_count[category]
      row = [category]
      score_sum = 0.0
      for i in range(5):
          score_sum += star_counts[i] * (i+1)
          row.append(star_counts[i])
      row.append(star_counts[3] + star_counts[4])
      row.append(star_counts[0] + star_counts[1])
      number_of_reviews = sum(star_counts)
      row.append(number_of_reviews)
      row.append(score_sum/number_of_reviews)
      category_rows.append(row)
  
  write_csv_and_xlsx("stats/category_stats", category_rows)
  
  for business in business2reviews_count:
      city = business_id2city[business]
      business_name = business_id2name[business]
      categories = business_id2categories[business]
      star_counts = business2reviews_count[business]
      row = [business, business_name, len(categories), categories, city]
      score_sum = 0.0
      for i in range(5):
          score_sum += star_counts[i] * (i+1)
          row.append(star_counts[i])
      row.append(star_counts[3] + star_counts[4])
      row.append(star_counts[0] + star_counts[1])
      number_of_reviews = sum(star_counts)
      row.append(number_of_reviews)
      if number_of_reviews>0:
          row.append(score_sum/number_of_reviews)
      else:
          row.append(0)
      business_id_rows.append(row)
  
  write_csv_and_xlsx("stats/business_stats", business_id_rows)
  
  for city in city2reviews_count:
      star_counts = city2reviews_count[city]
      row = [city, city2businesses[city]]
      score_sum = 0.0
      for i in range(5):
          score_sum += star_counts[i] * (i+1)
          row.append(star_counts[i])
      row.append(star_counts[3] + star_counts[4])
      row.append(star_counts[0] + star_counts[1])
      number_of_reviews = sum(star_counts)
      row.append(number_of_reviews)
      if number_of_reviews>0:
          row.append(score_sum/number_of_reviews)
      else:
          row.append(0)
      city_rows.append(row)
  
  write_csv_and_xlsx("stats/city_stats", city_rows)
  
  import math
  
  split_texts = []
  number_of_splits = 100
  numuber_of_reviews_per_split = math.ceil(float(len(texts))/number_of_splits)
  for i in range(number_of_splits):
      text_chunk = texts[numuber_of_reviews_per_split*i:numuber_of_reviews_per_split*(i+1)]
      text_chunk = process_words(text_chunk, "en", stem=True)
      texts = texts[:numuber_of_reviews_per_split*i] + text_chunk + texts[numuber_of_reviews_per_split*(i+1):]
  
  i = 0
  while i < len(texts):
      if texts[i] == []:
          del texts[i]
          del reviews[i]
      else:
          i += 1
  
  for text in texts:
      for word in text:
          if word not in word2rating:
              word2rating[word] = [0.0, 0]
          word2rating[word][1] += 1
  
  frequncy_boundries = [0, 5, 10, 50, 100, 500, 1000, 5000, 10000]
  frequency_cnts = [0] * len(frequncy_boundries)
  
  for text in texts:
      for word in text:
          for k, frequncy_boundry in enumerate(frequncy_boundries):
              if word2rating[word][1] >= frequncy_boundry:
                  frequency_cnts[k] += 1
  
  for k, frequncy_boundry in enumerate(frequncy_boundries):
      print("Number of words that appear a minimum of", frequncy_boundry,":",frequency_cnts[k])
  
  MIN_DOCUMENT_LENGTH = 10000000
  MAX_DOCUMENT_LENGTH = 0
  word_total = 0.0
  for text in texts:
      length = len(text)
      if length > MAX_DOCUMENT_LENGTH:
          MAX_DOCUMENT_LENGTH = length
      if length < MIN_DOCUMENT_LENGTH:
          MIN_DOCUMENT_LENGTH = length
      word_total += length
  print("Average number of words per review:",word_total/len(texts))
  print("Min number of words:", MIN_DOCUMENT_LENGTH)
  print("Max number of words:", MAX_DOCUMENT_LENGTH)
  
  