streamcorpus_pipeline/text_profile_signature.py
def tps(text, min_token_len=2, quant_rate=0.01):
    '''
    :param text: tag-free UTF-8 string of free text
    :returns string: 32-character md5 hash in hexadecimal

    Python implementation of the TextProfileSignature provided in
    SOLR.  Unlike most other locality sensitive hashes, a TPS can be
    indexed as a searchable property of each document that does not
    require n^2 comparisons to find duplicates.

    http://wiki.apache.org/solr/TextProfileSignature
    '''
    
    counts = Counter(
        ifilter(lambda x: len(x) >= min_token_len, 
                imap(cleanse, text.split())))

    max_freq = counts.most_common(1)[0][1]
    if max_freq <= 1:
        quant = 1
    else:
        quant = max(2, round(max_freq * quant_rate))

    to_hash = []
    for word, count in counts.most_common():
        if count <= quant:
            break
        to_hash += [word, str(int(math.floor(count / quant)))]

    to_hash = u' '.join(to_hash)
    return hashlib.md5(to_hash.encode('utf8')).hexdigest()
def tps(text, min_token_len=2, quant_rate=0.01):
    '''
    :param text: tag-free UTF-8 string of free text
    :returns string: 32-character md5 hash in hexadecimal

    Python implementation of the TextProfileSignature provided in
    SOLR.  Unlike most other locality sensitive hashes, a TPS can be
    indexed as a searchable property of each document that does not
    require n^2 comparisons to find duplicates.

    http://wiki.apache.org/solr/TextProfileSignature
    '''
    
    counts = Counter(
        ifilter(lambda x: len(x) >= min_token_len, 
                imap(cleanse, text.split())))

    max_freq = counts.most_common(1)[0][1]
    if max_freq <= 1:
        quant = 1
    else:
        quant = max(2, round(max_freq * quant_rate))

    to_hash = []
    for word, count in counts.most_common():
        if count <= quant:
            break
        to_hash += [word, str(int(math.floor(count / quant)))]

    to_hash = u' '.join(to_hash)
    return hashlib.md5(to_hash.encode('utf8')).hexdigest()
def tps ( text , min_token_len = 2 , quant_rate = 0.01 ) : counts = Counter ( ifilter ( lambda x : len ( x ) >= min_token_len , imap ( cleanse , text . split ( ) ) ) ) max_freq = counts . most_common ( 1 ) [ 0 ] [ 1 ] if max_freq <= 1 : quant = 1 else : quant = max ( 2 , round ( max_freq * quant_rate ) ) to_hash = [ ] for word , count in counts . most_common ( ) : if count <= quant : break to_hash += [ word , str ( int ( math . floor ( count / quant ) ) ) ] to_hash = u' ' . join ( to_hash ) return hashlib . md5 ( to_hash . encode ( 'utf8' ) ) . hexdigest ( )
:param text: tag-free UTF-8 string of free text
    :returns string: 32-character md5 hash in hexadecimal

    Python implementation of the TextProfileSignature provided in
    SOLR.  Unlike most other locality sensitive hashes, a TPS can be
    indexed as a searchable property of each document that does not
    require n^2 comparisons to find duplicates.

    http://wiki.apache.org/solr/TextProfileSignature
: param text : tag - free UTF - 8 string of free text : returns string : 32 - character md5 hash in hexadecimal