merge_machine/preprocess_fields_v3.py
def normalize_and_validate_tokens(phrase,
	keepAcronyms = False, tokenValidator = is_valid_token, phraseValidator = is_valid_phrase, stopWords = None):
	''' Returns a list of normalized, valid tokens for the input phrase (an empty list
		if no valid tokens were found) '''
	if phrase:
		tokens = str.split(case_phrase(phrase, keepAcronyms))
		validTokens = []
		for token in tokens:
			if tokenValidator(token) and (stopWords is None or token not in stopWords): validTokens.append(token)
		if phraseValidator(validTokens): return validTokens
	return []
 Returns a list of normalized, valid tokens for the input phrase (an empty list
		if no valid tokens were found) 
Returns a list of normalized, valid tokens for the input phrase (an empty list
if no valid tokens were found)
Returns a list of normalized valid tokens for the input phrase ( an empty list if no valid tokens were found )
def normalize_and_validate_tokens(phrase,
	keepAcronyms = False, tokenValidator = is_valid_token, phraseValidator = is_valid_phrase, stopWords = None):
	if phrase:
		tokens = str.split(case_phrase(phrase, keepAcronyms))
		validTokens = []
		for token in tokens:
			if tokenValidator(token) and (stopWords is None or token not in stopWords): validTokens.append(token)
		if phraseValidator(validTokens): return validTokens
	return []
def normalize_and_validate_tokens ( phrase , keepAcronyms = False , tokenValidator = is_valid_token , phraseValidator = is_valid_phrase , stopWords = None ) : if phrase : tokens = str . split ( case_phrase ( phrase , keepAcronyms ) ) validTokens = [ ] for token in tokens : if tokenValidator ( token ) and ( stopWords is None or token not in stopWords ) : validTokens . append ( token ) if phraseValidator ( validTokens ) : return validTokens return [ ]
Returns a list of normalized, valid tokens for the input phrase (an empty list
if no valid tokens were found)
Returns a list of normalized valid tokens for the input phrase ( an empty list if no valid tokens were found )
''' Returns a list of normalized, valid tokens for the input phrase (an empty list
		if no valid tokens were found) '''
phrase
keepAcronyms
tokenValidator
phraseValidator
stopWords