def get_approx_deduplicating_key(text, encoding_scheme=sanscript.DEVANAGARI):
    """
    Given some devanAgarI sanskrit text, this function produces a "key" so
    that
  
    1] The key should be the same for different observed orthographical
    forms of the same text. For example:
  
    ::
  
        - "dharmma" vs "dharma"
        - "rAmaM gacChati" vs "rAma~N gacChati" vs "rAma~N gacChati"
        - "kurvan eva" vs "kurvanneva"
  
    2] The key should be different for different for different texts.
  
    -  "stamba" vs "stambha"
  
    This function attempts to succeed at [1] and [2] almostall the time.
    Longer the text, probability of failing at [2] decreases, while
    probability of failing at [1] increases (albeit very slightly).
  
    Sources of orthographically divergent forms:
  
    -  Phonetically sensible grammar rules
    -  Neglect of sandhi while writing
    -  Punctuation, spaces, avagraha-s.
    -  Regional-language-influenced mistakes (La instead of la.)
  
    Some example applications of this function:
  
    -  Create a database of quotes or words with minimal duplication.
    -  Search a database of quotes or words while being robust to optional
       forms.
  
    Also see equivalent function in the scala indic-transliteration package.
    """
    if encoding_scheme == sanscript.DEVANAGARI:
        key = text
        key = regex.sub("\\P{IsDevanagari}", "", key)
        # Remove spaces
        key = regex.sub("\\s", "", key)
        # Remove punctuations
        key = regex.sub("\\p{P}", "", key)
        # Remove digits, abbreviation sign, svara-s, avagraha
        key = regex.sub("[०-९।॥॰ऽ]|[॑-॔]", "", key)
        # Collapse semi-vowel-anunAsika-s संलग्नम् सल्ँलग्नम् into m
        key = regex.sub("[यरल]्ँ", "म्", key)
        # Collapse all panchama-s into m
        key = regex.sub("[ङञणन]", "म", key)
        # Collapse anusvAra into m
        key = regex.sub("ँ|ं", "म्", key)
        key = regex.sub("ॐ", "ओम्", key)
        key = regex.sub("[ळऴ]", "ल", key)
        # Deal with optional forms where consonants are duplicated - like dharmma
        # Details in https://docs.google.com/spreadsheets/d/1GP8Ps_hmgCGLZPWKIVBCfQB9ZmPQOaCwTrH9OybaWaQ/edit#gid=21
        key = regex.sub("([क-हक़-य़])्\\1+", "\\1", key)
        key = regex.sub("[कग]्ख्", "ख्", key)
        key = regex.sub("[कग]्घ्", "घ्", key)
        key = regex.sub("च्छ्", "छ्", key)
        key = regex.sub("ज्झ्", "झ्", key)
        key = regex.sub("त्थ्", "थ्", key)
        key = regex.sub("द्ध्", "ध्", key)
        key = regex.sub("ड्ढ्", "ढ्", key)
        key = regex.sub("प्फ्", "फ्", key)
        key = regex.sub("ब्भ्", "भ्", key)
        return key
    else:
        logging.warning("got script {} for '{}'".format(encoding_scheme, text))
        return regex.sub("\\s", "", text)
def get_approx_deduplicating_key(text, encoding_scheme=sanscript.DEVANAGARI):
    """
    Given some devanAgarI sanskrit text, this function produces a "key" so
    that
  
    1] The key should be the same for different observed orthographical
    forms of the same text. For example:
  
    ::
  
        - "dharmma" vs "dharma"
        - "rAmaM gacChati" vs "rAma~N gacChati" vs "rAma~N gacChati"
        - "kurvan eva" vs "kurvanneva"
  
    2] The key should be different for different for different texts.
  
    -  "stamba" vs "stambha"
  
    This function attempts to succeed at [1] and [2] almostall the time.
    Longer the text, probability of failing at [2] decreases, while
    probability of failing at [1] increases (albeit very slightly).
  
    Sources of orthographically divergent forms:
  
    -  Phonetically sensible grammar rules
    -  Neglect of sandhi while writing
    -  Punctuation, spaces, avagraha-s.
    -  Regional-language-influenced mistakes (La instead of la.)
  
    Some example applications of this function:
  
    -  Create a database of quotes or words with minimal duplication.
    -  Search a database of quotes or words while being robust to optional
       forms.
  
    Also see equivalent function in the scala indic-transliteration package.
    """
    if encoding_scheme == sanscript.DEVANAGARI:
        key = text
        key = regex.sub("\\P{IsDevanagari}", "", key)
        # Remove spaces
        key = regex.sub("\\s", "", key)
        # Remove punctuations
        key = regex.sub("\\p{P}", "", key)
        # Remove digits, abbreviation sign, svara-s, avagraha
        key = regex.sub("[०-९।॥॰ऽ]|[॑-॔]", "", key)
        # Collapse semi-vowel-anunAsika-s संलग्नम् सल्ँलग्नम् into m
        key = regex.sub("[यरल]्ँ", "म्", key)
        # Collapse all panchama-s into m
        key = regex.sub("[ङञणन]", "म", key)
        # Collapse anusvAra into m
        key = regex.sub("ँ|ं", "म्", key)
        key = regex.sub("ॐ", "ओम्", key)
        key = regex.sub("[ळऴ]", "ल", key)
        # Deal with optional forms where consonants are duplicated - like dharmma
        # Details in https://docs.google.com/spreadsheets/d/1GP8Ps_hmgCGLZPWKIVBCfQB9ZmPQOaCwTrH9OybaWaQ/edit#gid=21
        key = regex.sub("([क-हक़-य़])्\\1+", "\\1", key)
        key = regex.sub("[कग]्ख्", "ख्", key)
        key = regex.sub("[कग]्घ्", "घ्", key)
        key = regex.sub("च्छ्", "छ्", key)
        key = regex.sub("ज्झ्", "झ्", key)
        key = regex.sub("त्थ्", "थ्", key)
        key = regex.sub("द्ध्", "ध्", key)
        key = regex.sub("ड्ढ्", "ढ्", key)
        key = regex.sub("प्फ्", "फ्", key)
        key = regex.sub("ब्भ्", "भ्", key)
        return key
    else:
        logging.warning("got script {} for '{}'".format(encoding_scheme, text))
        return regex.sub("\\s", "", text)
def get_approx_deduplicating_key ( text , encoding_scheme = sanscript . DEVANAGARI ) : if encoding_scheme == sanscript . DEVANAGARI : key = text key = regex . sub ( "\\P{IsDevanagari}" , "" , key ) # Remove spaces key = regex . sub ( "\\s" , "" , key ) # Remove punctuations key = regex . sub ( "\\p{P}" , "" , key ) # Remove digits, abbreviation sign, svara-s, avagraha key = regex . sub ( "[०-९।॥॰ऽ]|[॑-॔]", "", key) # Collapse semi-vowel-anunAsika-s संलग्नम् सल्ँलग्नम् into m key = regex . sub ( "[यरल]्ँ", "म्", ke y # Collapse all panchama-s into m key = regex . sub ( "[ङञणन]", "म", k e ) # Collapse anusvAra into m key = regex . sub ( "ँ|ं", "म ् , key) key = regex . sub ( "ॐ",  " म्", key) key = regex . sub ( "[ळऴ]", "ल "  key) # Deal with optional forms where consonants are duplicated - like dharmma # Details in https://docs.google.com/spreadsheets/d/1GP8Ps_hmgCGLZPWKIVBCfQB9ZmPQOaCwTrH9OybaWaQ/edit#gid=21 key = regex . sub ( "([क-हक़-य़])्\\1+", "\\1", k e ) key = regex . sub ( "[कग]्ख्", "ख्", ke y key = regex . sub ( "[कग]्घ्", "घ्", ke y key = regex . sub ( "च्छ्", "छ्",  k y) key = regex . sub ( "ज्झ्", "झ्",  k y) key = regex . sub ( "त्थ्", "थ्",  k y) key = regex . sub ( "द्ध्", "ध्",  k y) key = regex . sub ( "ड्ढ्", "ढ्",  k y) key = regex . sub ( "प्फ्", "फ्",  k y) key = regex . sub ( "ब्भ्", "भ्",  k y) return key else : logging . warning ( "got script {} for '{}'" . format ( encoding_scheme , text ) ) return regex . sub ( "\\s" , "" , text )
Given some devanAgarI sanskrit text, this function produces a "key" so
    that
  
    1] The key should be the same for different observed orthographical
    forms of the same text. For example:
  
    ::
  
        - "dharmma" vs "dharma"
        - "rAmaM gacChati" vs "rAma~N gacChati" vs "rAma~N gacChati"
        - "kurvan eva" vs "kurvanneva"
  
    2] The key should be different for different for different texts.
  
    -  "stamba" vs "stambha"
  
    This function attempts to succeed at [1] and [2] almostall the time.
    Longer the text, probability of failing at [2] decreases, while
    probability of failing at [1] increases (albeit very slightly).
  
    Sources of orthographically divergent forms:
  
    -  Phonetically sensible grammar rules
    -  Neglect of sandhi while writing
    -  Punctuation, spaces, avagraha-s.
    -  Regional-language-influenced mistakes (La instead of la.)
  
    Some example applications of this function:
  
    -  Create a database of quotes or words with minimal duplication.
    -  Search a database of quotes or words while being robust to optional
       forms.
  
    Also see equivalent function in the scala indic-transliteration package.
Given some devanAgarI sanskrit text this function produces a key so that 1 ] The key should be the same for different observed orthographical forms of the same text . For example : :: - dharmma vs dharma - rAmaM gacChati vs rAma~N gacChati vs rAma~N gacChati - kurvan eva vs kurvanneva 2 ] The key should be different for different for different texts . - stamba vs stambha This function attempts to succeed at [ 1 ] and [ 2 ] almostall the time . Longer the text probability of failing at [ 2 ] decreases while probability of failing at [ 1 ] increases ( albeit very slightly ) . Sources of orthographically divergent forms : - Phonetically sensible grammar rules - Neglect of sandhi while writing - Punctuation spaces avagraha - s . - Regional - language - influenced mistakes ( La instead of la . ) Some example applications of this function : - Create a database of quotes or words with minimal duplication . - Search a database of quotes or words while being robust to optional forms . Also see equivalent function in the scala indic - transliteration package .