merge_machine/preprocess_fields_v3.py
def infer_types(self):
		''' Returns a dictionary mapping input field name to likeliest type.
			Fields for which no type has been inferred will be missing from the output dictionary.'''
		self.match_headers_and_values()
		types = dict()
		f2t = defaultdict(list)
		t2f = defaultdict(list)
		for (h, f) in self.fields.items():
			fieldName = h.value
			lht = h.likeliest_type()
			logging.info('Likeliest type for %s header: %s', fieldName, lht)
			if lht is not None: types[fieldName] = lht
			for (t, s) in f.scored_types().items():
				if s < COLUMN_SCORE_THRESHOLD: continue
				f2t[fieldName].append((t, s))
				t2f[t].append((fieldName, s))
		for (h, f) in self.fields.items():
			fieldName = h.value
			ts = sorted(f2t[fieldName], key = itemgetter(1), reverse = True)
			logging.info('Sorted types for {}: {}'. format(fieldName, '; '.join(['{} ({}) '.format(t, s) for (t, s) in ts])))
			for (t, s) in ts:
				betterField = any((p[1] > s and p[0] not in types) for p in t2f[t])
				betterChild = t in SUBTYPING_RELS and any((p[1] * SUBTYPING_RATIO > s and p[0] in SUBTYPING_RELS[t]) for p in ts)
				if not (betterField or betterChild):
					logging.info('Likeliest type for %s values: %s', fieldName, t)
					types[fieldName] = t
					break
			if fieldName not in types: logging.info('Could not infer type for %s values', fieldName)
		return types
 Returns a dictionary mapping input field name to likeliest type.
			Fields for which no type has been inferred will be missing from the output dictionary.
Returns a dictionary mapping input field name to likeliest type.
Fields for which no type has been inferred will be missing from the output dictionary.
Returns a dictionary mapping input field name to likeliest type . Fields for which no type has been inferred will be missing from the output dictionary .
def infer_types(self):
		self.match_headers_and_values()
		types = dict()
		f2t = defaultdict(list)
		t2f = defaultdict(list)
		for (h, f) in self.fields.items():
			fieldName = h.value
			lht = h.likeliest_type()
			logging.info('Likeliest type for %s header: %s', fieldName, lht)
			if lht is not None: types[fieldName] = lht
			for (t, s) in f.scored_types().items():
				if s < COLUMN_SCORE_THRESHOLD: continue
				f2t[fieldName].append((t, s))
				t2f[t].append((fieldName, s))
		for (h, f) in self.fields.items():
			fieldName = h.value
			ts = sorted(f2t[fieldName], key = itemgetter(1), reverse = True)
			logging.info('Sorted types for {}: {}'. format(fieldName, '; '.join(['{} ({}) '.format(t, s) for (t, s) in ts])))
			for (t, s) in ts:
				betterField = any((p[1] > s and p[0] not in types) for p in t2f[t])
				betterChild = t in SUBTYPING_RELS and any((p[1] * SUBTYPING_RATIO > s and p[0] in SUBTYPING_RELS[t]) for p in ts)
				if not (betterField or betterChild):
					logging.info('Likeliest type for %s values: %s', fieldName, t)
					types[fieldName] = t
					break
			if fieldName not in types: logging.info('Could not infer type for %s values', fieldName)
		return types
def infer_types ( self ) : self . match_headers_and_values ( ) types = dict ( ) f2t = defaultdict ( list ) t2f = defaultdict ( list ) for ( h , f ) in self . fields . items ( ) : fieldName = h . value lht = h . likeliest_type ( ) logging . info ( 'Likeliest type for %s header: %s' , fieldName , lht ) if lht is not None : types [ fieldName ] = lht for ( t , s ) in f . scored_types ( ) . items ( ) : if s < COLUMN_SCORE_THRESHOLD : continue f2t [ fieldName ] . append ( ( t , s ) ) t2f [ t ] . append ( ( fieldName , s ) ) for ( h , f ) in self . fields . items ( ) : fieldName = h . value ts = sorted ( f2t [ fieldName ] , key = itemgetter ( 1 ) , reverse = True ) logging . info ( 'Sorted types for {}: {}' . format ( fieldName , '; ' . join ( [ '{} ({}) ' . format ( t , s ) for ( t , s ) in ts ] ) ) ) for ( t , s ) in ts : betterField = any ( ( p [ 1 ] > s and p [ 0 ] not in types ) for p in t2f [ t ] ) betterChild = t in SUBTYPING_RELS and any ( ( p [ 1 ] * SUBTYPING_RATIO > s and p [ 0 ] in SUBTYPING_RELS [ t ] ) for p in ts ) if not ( betterField or betterChild ) : logging . info ( 'Likeliest type for %s values: %s' , fieldName , t ) types [ fieldName ] = t break if fieldName not in types : logging . info ( 'Could not infer type for %s values' , fieldName ) return types
Returns a dictionary mapping input field name to likeliest type.
Returns a dictionary mapping input field name to likeliest type .
''' Returns a dictionary mapping input field name to likeliest type.
			Fields for which no type has been inferred will be missing from the output dictionary.'''
self