merge_machine/preprocess_fields_v3.py
def file_to_variant_map(fileName, sep = '|', includeSelf = False, tokenize = False):
	''' The input format is pipe-separated, column 1 is the main variant, column 2 an alternative variant.

		Returns a reverse index, namely a map from original alternative variant to original main variant

		Parameters:
		includeSelf if True, then the main variant will be included in the list of alternative variants
			(so as to enable partial matching simultaneously).
	'''
	otherToMain = dict()
	mainToOther = defaultdict(list)
	for row in file_row_iter(fileName, sep):
		if len(row) < 2: continue
		r = list([normalize_or_not(e, tokenize = tokenize)  for e in row])
		main, alts = r[0], set(r[1:])
		alts.discard(main)
		for alt in alts: 
			if alt not in otherToMain:
				otherToMain[alt] = main
			elif main not in otherToMain and otherToMain[alt] != main:
				otherToMain[main] = otherToMain[alt]
		mainToOther[main].extend(list(alts))
	l = list(otherToMain.items())
	if includeSelf: l = list([(main, main) for main in mainToOther.keys()]) + l
	return dict(l)
 The input format is pipe-separated, column 1 is the main variant, column 2 an alternative variant.

		Returns a reverse index, namely a map from original alternative variant to original main variant

		Parameters:
		includeSelf if True, then the main variant will be included in the list of alternative variants
			(so as to enable partial matching simultaneously).
	
The input format is pipe-separated, column 1 is the main variant, column 2 an alternative variant.
Returns a reverse index, namely a map from original alternative variant to original main variant

includeSelf if True, then the main variant will be included in the list of alternative variants
(so as to enable partial matching simultaneously).
The input format is pipe - separated column 1 is the main variant column 2 an alternative variant . Returns a reverse index namely a map from original alternative variant to original main variant includeSelf if True then the main variant will be included in the list of alternative variants ( so as to enable partial matching simultaneously ) .
def file_to_variant_map(fileName, sep = '|', includeSelf = False, tokenize = False):
	otherToMain = dict()
	mainToOther = defaultdict(list)
	for row in file_row_iter(fileName, sep):
		if len(row) < 2: continue
		r = list([normalize_or_not(e, tokenize = tokenize)  for e in row])
		main, alts = r[0], set(r[1:])
		alts.discard(main)
		for alt in alts: 
			if alt not in otherToMain:
				otherToMain[alt] = main
			elif main not in otherToMain and otherToMain[alt] != main:
				otherToMain[main] = otherToMain[alt]
		mainToOther[main].extend(list(alts))
	l = list(otherToMain.items())
	if includeSelf: l = list([(main, main) for main in mainToOther.keys()]) + l
	return dict(l)
def file_to_variant_map ( fileName , sep = '|' , includeSelf = False , tokenize = False ) : otherToMain = dict ( ) mainToOther = defaultdict ( list ) for row in file_row_iter ( fileName , sep ) : if len ( row ) < 2 : continue r = list ( [ normalize_or_not ( e , tokenize = tokenize ) for e in row ] ) main , alts = r [ 0 ] , set ( r [ 1 : ] ) alts . discard ( main ) for alt in alts : if alt not in otherToMain : otherToMain [ alt ] = main elif main not in otherToMain and otherToMain [ alt ] != main : otherToMain [ main ] = otherToMain [ alt ] mainToOther [ main ] . extend ( list ( alts ) ) l = list ( otherToMain . items ( ) ) if includeSelf : l = list ( [ ( main , main ) for main in mainToOther . keys ( ) ] ) + l return dict ( l )
The input format is pipe-separated, column 1 is the main variant, column 2 an alternative variant.
The input format is pipe - separated column 1 is the main variant column 2 an alternative variant .
''' The input format is pipe-separated, column 1 is the main variant, column 2 an alternative variant.

		Returns a reverse index, namely a map from original alternative variant to original main variant

		Parameters:
		includeSelf if True, then the main variant will be included in the list of alternative variants
			(so as to enable partial matching simultaneously).
	'''
fileName
sep
includeSelf
tokenize