merge_machine/preprocess_fields_v3.py
def normalize_values(self, types):
		''' Generates (field name, list of field values) pairs for each output field. '''
		self.modifiedByColumn.clear()
		self.outputFieldsByColumn.clear()
		for (h, f) in self.fields.items():
			fieldName = h.value
			self.modifiedByColumn[fieldName] = [0] * self.entries
			if fieldName not in types:
				logging.warning('No values to normalize for {}'.format(fieldName))
				continue
			logging.info('Normalizing values for {}'.format(fieldName))
			lvt = types[fieldName]
			ofs = uniq(list(f.normalized_fields(h, lvt)))
			logging.info('Output fields for %s: %s', fieldName, ofs)
			nvs = list(f.normalized_values(h, lvt))
			for of in ofs:
				b = [None] * self.entries
				for i, nc in enumerate(nvs):
					if of not in nc or nc[of] is None: continue
					if b[i] is None:
						b[i] = nc[of]
					else:
						if isinstance(b[i], list):
							if isinstance(nc[of], list): 
								b[i].extend(nc[of])
							else: 
								b[i].append(nc[of])
						else:
							if isinstance(nc[of], list): 
								b[i] = [b[i]] + nc[of]
							else: 
								b[i] = [b[i], nc[of]]
					oldValue = f.cells[i].value
					newValues = list([s for s in (nc[of] if isinstance(nc[of], list) else [nc[of]])])
					isNewValue = len(newValues) > 0 and oldValue not in newValues
					if isNewValue: 
						self.modifiedByColumn[fieldName][i] += 1
				yield (of, b)
			self.outputFieldsByColumn[fieldName] = ofs
 Generates (field name, list of field values) pairs for each output field. 
Generates (field name, list of field values) pairs for each output field.
Generates ( field name list of field values ) pairs for each output field .
def normalize_values(self, types):
		self.modifiedByColumn.clear()
		self.outputFieldsByColumn.clear()
		for (h, f) in self.fields.items():
			fieldName = h.value
			self.modifiedByColumn[fieldName] = [0] * self.entries
			if fieldName not in types:
				logging.warning('No values to normalize for {}'.format(fieldName))
				continue
			logging.info('Normalizing values for {}'.format(fieldName))
			lvt = types[fieldName]
			ofs = uniq(list(f.normalized_fields(h, lvt)))
			logging.info('Output fields for %s: %s', fieldName, ofs)
			nvs = list(f.normalized_values(h, lvt))
			for of in ofs:
				b = [None] * self.entries
				for i, nc in enumerate(nvs):
					if of not in nc or nc[of] is None: continue
					if b[i] is None:
						b[i] = nc[of]
					else:
						if isinstance(b[i], list):
							if isinstance(nc[of], list): 
								b[i].extend(nc[of])
							else: 
								b[i].append(nc[of])
						else:
							if isinstance(nc[of], list): 
								b[i] = [b[i]] + nc[of]
							else: 
								b[i] = [b[i], nc[of]]
					oldValue = f.cells[i].value
					newValues = list([s for s in (nc[of] if isinstance(nc[of], list) else [nc[of]])])
					isNewValue = len(newValues) > 0 and oldValue not in newValues
					if isNewValue: 
						self.modifiedByColumn[fieldName][i] += 1
				yield (of, b)
			self.outputFieldsByColumn[fieldName] = ofs
def normalize_values ( self , types ) : self . modifiedByColumn . clear ( ) self . outputFieldsByColumn . clear ( ) for ( h , f ) in self . fields . items ( ) : fieldName = h . value self . modifiedByColumn [ fieldName ] = [ 0 ] * self . entries if fieldName not in types : logging . warning ( 'No values to normalize for {}' . format ( fieldName ) ) continue logging . info ( 'Normalizing values for {}' . format ( fieldName ) ) lvt = types [ fieldName ] ofs = uniq ( list ( f . normalized_fields ( h , lvt ) ) ) logging . info ( 'Output fields for %s: %s' , fieldName , ofs ) nvs = list ( f . normalized_values ( h , lvt ) ) for of in ofs : b = [ None ] * self . entries for i , nc in enumerate ( nvs ) : if of not in nc or nc [ of ] is None : continue if b [ i ] is None : b [ i ] = nc [ of ] else : if isinstance ( b [ i ] , list ) : if isinstance ( nc [ of ] , list ) : b [ i ] . extend ( nc [ of ] ) else : b [ i ] . append ( nc [ of ] ) else : if isinstance ( nc [ of ] , list ) : b [ i ] = [ b [ i ] ] + nc [ of ] else : b [ i ] = [ b [ i ] , nc [ of ] ] oldValue = f . cells [ i ] . value newValues = list ( [ s for s in ( nc [ of ] if isinstance ( nc [ of ] , list ) else [ nc [ of ] ] ) ] ) isNewValue = len ( newValues ) > 0 and oldValue not in newValues if isNewValue : self . modifiedByColumn [ fieldName ] [ i ] += 1 yield ( of , b ) self . outputFieldsByColumn [ fieldName ] = ofs
Generates (field name, list of field values) pairs for each output field.
Generates ( field name list of field values ) pairs for each output field .
''' Generates (field name, list of field values) pairs for each output field. '''
self
types