text: import nltk
  import pprint
  import yaml
  from os import walk
  
  class Splitter(object):
  	def __init__(self):
  		self.nltk_splitter = nltk.data.load('tokenizers/punkt/english.pickle')
  		self.nltk_tokenizer = nltk.tokenize.TreebankWordTokenizer()
  
  	def split(self, text):
  		sentences = self.nltk_splitter.tokenize(text)
  		tokenized_sentences = [self.nltk_tokenizer.tokenize(sent) for sent in sentences]
  		return tokenized_sentences
  
  class POSTagger(object):
  	def __init__(self):
  		pass
  
  	def pos_tag(self, sentences):
  		pos = [nltk.pos_tag(sentence) for sentence in sentences]
  		pos = [[(word, word, [postag]) for (word, postag) in sentence] for sentence in pos]
  		return pos
  
  
  class DictionaryTagger(object):
  	def __init__(self, dictionary_paths):
  		files = [open(path, 'r') for path in dictionary_paths]
  		dictionaries = [yaml.load(dict_file) for dict_file in files]
  		map(lambda x: x.close(), files)
  		self.dictionary = {}
  		self.max_key_size = 0
  		for curr_dict in dictionaries:
  			for key in curr_dict:
  				if key in self.dictionary:
  					self.dictionary[key].extend(curr_dict[key])
  				else:
  					self.dictionary[key] = curr_dict[key]
  					self.max_key_size = max(self.max_key_size, len(key))
  	
  	def tag(self, postagged_sentences):
  		return [self.tag_sentence(sentence) for sentence in postagged_sentences]
  	
  	def tag_sentence(self, sentence, tag_with_lemmas = False):
  		tag_sentence = []
  		N = len(sentence)
  		if self.max_key_size ==0:
  			self.max_key_size = N
  		i = 0
  		while (i < N):
  			j = min(i + self.max_key_size, N)
  			tagged = False
  			while ( j > i ):
  				expression_form = ' '.join([word[0] for word in sentence[i:j]]).lower()
  				expression_lemma = ' '.join([word[1] for word in sentence[i:j]]).lower()
  				if tag_with_lemmas:
  					literal = expression_lemma
  				else:
  					literal = expression_form
  				if literal in self.dictionary:
  					is_single_token = j - i == 1
  					original_position = i
  					i = j
  					taggings = [tag for tag in self.dictionary[literal]]
  					tagged_expression = (expression_form, expression_lemma, taggings)
  					if is_single_token:
  						original_token_taggin = sentence[original_position][2]
  						tagged_expression[2].extend(original_token_taggin)
  					tag_sentence.append(tagged_expression)
  					tagged = True
  				else:
  					j = j - 1
  			if not tagged:
  				tag_sentence.append(sentence[i])
  				i += 1
  		return tag_sentence
  
  def value_of(sentiment):
  	if sentiment == 'positive': return 1;
  	if sentiment == 'negative': return -1;
  	return 0
  
  def sentence_score(sentence_tokens, previous_token, acum_score):
  	if not sentence_tokens:
  		return acum_score
  	else:
  		current_token = sentence_tokens[0]
  		tags = current_token[2]
  		token_score = sum(value_of(tag) for tag in tags)
  		if previous_token is not None:
  			previous_tags = previous_token[2]
  			if 'inc' in previous_tags:
  				token_score *= 2.0
  			elif 'dec' in previous_tags:
  				token_score /= 2.0
  			elif 'inv' in previous_tags:
  				token_score *= -1.0
  		return sentence_score(sentence_tokens[1:], current_token, acum_score+token_score)
  
  def sentiment_score(review):
  	return sum([sentence_score(sentence, None, 0.0) for sentence in review])
  
  splitter = Splitter()
  postagger = POSTagger()
  
  dicttagger = DictionaryTagger(['dicts/positive.yml', 'dicts/negative.yml', 'dicts/inc.yml', 'dicts/dec.yml', 'dicts/inv.yml'])
  
  out = open("output", 'w')
  yelp_path = "twitter"
  for (dirpath, dirnames, filenames) in walk(yelp_path):
          shop = dirpath[len(yelp_path) + 1 :]
          for filename in filenames:
                  f = open(dirpath + '/' + filename, 'r')
                  print dirpath + '/' + filename
                  #out.write(dirpath + '/' + filename + '\n')
                  review = ""
                  while 1:
                          line = f.readline()
                          if not line:
                                  break
                          review = review + line.decode('ascii', 'ignore')
                  #print review
                  splitted_sentences = splitter.split(review)
  		pos_tagged_sentences = postagger.pos_tag(splitted_sentences)
  		dict_tagged_sentences = dicttagger.tag(pos_tagged_sentences)
                  #print shop + ' ' + repr(rator.sentiment_score(splitted_sentences))
                  out.write( repr(sentiment_score(dict_tagged_sentences)) + '\n' )
                  out.write( shop + '\n' )
  
  out.close()
  
  