parser.py
def add_vector_to_corpus(corpus_path, new_corpus_path, write_every=1000):
		"""
		Create a new CleanedText and Vector column on the corpus
		Separate the URLs by space if many
		:param write_every: write in the final file every x lines
		:param corpus_path:
		:param new_corpus_path:
		:return:
		"""
		parser = Parser()
		parser.load_w2v_model()

		corpus = open(corpus_path, 'r', encoding='utf-8')
		new_corpus = open(new_corpus_path, 'w', encoding='utf-8')

		lines = corpus.readlines()
		corpus.close()
		new_lines = []
		last_written = -1
		new_lines.append(lines[0][:-1] + '\tCleanedText\tVector\n')

		for i in range(1, len(lines)):
			parts = lines[i][:-1].split('\t')
			cleaned_tweet = parser.clean_tweet(parts[-6])
			urls = parts[5:-6]
			new_lines.append(
				'\t'.join(parts[:5]) + '\t' +  # TweetID Sentiment TopicID Country Gender
				' '.join(urls) + '\t' +  # URLs separated by space
				parts[-6] + '\t' +  # Text
				parts[-5] + '\t' +  # User_ID
				parts[-4][1:-1] + '\t' +  # User_Name without quotes
				parts[-3][1:-1] + '\t' +  # Date without quotes
				'\t'.join(parts[-2:]) + '\t' +  # Hashtags Indication
				' '.join(cleaned_tweet) + '\t' +  # CleanedText
				str(list(parser.tweet2vec(cleaned_tweet)))  # Vector
				+ '\n')

			if i % write_every == 0:
				new_corpus.write(''.join(new_lines[(last_written + 1):]))
				last_written = i
				print(str(last_written) + '/' + str(len(lines)) + ' treated')

		new_corpus.write(''.join(new_lines[(last_written + 1):]))
		new_corpus.close()

		Create a new CleanedText and Vector column on the corpus
		Separate the URLs by space if many
		:param write_every: write in the final file every x lines
		:param corpus_path:
		:param new_corpus_path:
		:return:
		
Create a new CleanedText and Vector column on the corpus
Separate the URLs by space if many
Create a new CleanedText and Vector column on the corpus Separate the URLs by space if many
def add_vector_to_corpus(corpus_path, new_corpus_path, write_every=1000):
		parser = Parser()
		parser.load_w2v_model()
		corpus = open(corpus_path, 'r', encoding='utf-8')
		new_corpus = open(new_corpus_path, 'w', encoding='utf-8')
		lines = corpus.readlines()
		corpus.close()
		new_lines = []
		last_written = -1
		new_lines.append(lines[0][:-1] + '\tCleanedText\tVector\n')
		for i in range(1, len(lines)):
			parts = lines[i][:-1].split('\t')
			cleaned_tweet = parser.clean_tweet(parts[-6])
			urls = parts[5:-6]
			new_lines.append(
				'\t'.join(parts[:5]) + '\t' +  
				' '.join(urls) + '\t' +  
				parts[-6] + '\t' +  
				parts[-5] + '\t' +  
				parts[-4][1:-1] + '\t' +  
				parts[-3][1:-1] + '\t' +  
				'\t'.join(parts[-2:]) + '\t' +  
				' '.join(cleaned_tweet) + '\t' +  
				str(list(parser.tweet2vec(cleaned_tweet)))  
				+ '\n')
			if i % write_every == 0:
				new_corpus.write(''.join(new_lines[(last_written + 1):]))
				last_written = i
				print(str(last_written) + '/' + str(len(lines)) + ' treated')
		new_corpus.write(''.join(new_lines[(last_written + 1):]))
		new_corpus.close()
def add_vector_to_corpus ( corpus_path , new_corpus_path , write_every = 1000 ) : parser = Parser ( ) parser . load_w2v_model ( ) corpus = open ( corpus_path , 'r' , encoding = 'utf-8' ) new_corpus = open ( new_corpus_path , 'w' , encoding = 'utf-8' ) lines = corpus . readlines ( ) corpus . close ( ) new_lines = [ ] last_written = - 1 new_lines . append ( lines [ 0 ] [ : - 1 ] + '\tCleanedText\tVector\n' ) for i in range ( 1 , len ( lines ) ) : parts = lines [ i ] [ : - 1 ] . split ( '\t' ) cleaned_tweet = parser . clean_tweet ( parts [ - 6 ] ) urls = parts [ 5 : - 6 ] new_lines . append ( '\t' . join ( parts [ : 5 ] ) + '\t' + ' ' . join ( urls ) + '\t' + parts [ - 6 ] + '\t' + parts [ - 5 ] + '\t' + parts [ - 4 ] [ 1 : - 1 ] + '\t' + parts [ - 3 ] [ 1 : - 1 ] + '\t' + '\t' . join ( parts [ - 2 : ] ) + '\t' + ' ' . join ( cleaned_tweet ) + '\t' + str ( list ( parser . tweet2vec ( cleaned_tweet ) ) ) + '\n' ) if i % write_every == 0 : new_corpus . write ( '' . join ( new_lines [ ( last_written + 1 ) : ] ) ) last_written = i print ( str ( last_written ) + '/' + str ( len ( lines ) ) + ' treated' ) new_corpus . write ( '' . join ( new_lines [ ( last_written + 1 ) : ] ) ) new_corpus . close ( )
Create a new CleanedText and Vector column on the corpus
Separate the URLs by space if many
Create a new CleanedText and Vector column on the corpus Separate the URLs by space if many
"""
		Create a new CleanedText and Vector column on the corpus
		Separate the URLs by space if many
		:param write_every: write in the final file every x lines
		:param corpus_path:
		:param new_corpus_path:
		:return:
		""" # TweetID Sentiment TopicID Country Gender # URLs separated by space # Text # User_ID # User_Name without quotes # Date without quotes # Hashtags Indication # CleanedText # Vector
None
corpus_path
None
new_corpus_path
None
write_every
write in the final file every x lines
write in the final file every x lines