# ---
# jupyter:
#   jupytext:
#     text_representation:
#       extension: .py
#       format_name: light
#       format_version: '1.5'
#       jupytext_version: 1.14.4
#   kernelspec:
#     display_name: Python 3
#     language: python
#     name: python3
# ---

import re
from collections import Counter

import warnings
warnings.filterwarnings("ignore")


def words(text):
    return re.findall(r'\w+', text.lower())


words ('EBook of The Adventures of Sherlock Holmes')

dictionary = Counter(words(open('data/big.txt').read()))

dictionary


# Find the correction c, out of all possible candidate corrections, that maximizes the probability that c is the intended correction, given the original word w
#
# Bayes' Theorem
#
# **P(c|w) = P(c) P(w|c) / P(w)**
#
# - **c** - Correction
#
#
# - **w** - Word
#
#
# - **P(c)** - The probability that c appears as a word of English text. <br>For example, occurrences of "the" make up about 7% of English text, so we should have P(the) = 0.07
#
#
# - **P(w|c)** - The probability that w would be typed in a text when the author meant c. <br>For example, P(teh|the) is relatively high, but P(theeexyz|the) would be very low

def P(word, N=sum(dictionary.values())):
    "Probability of `word`"
    return dictionary[word]/N


def correction(word):
    "Most probable spelling correction for word"
    return max(candidates(word), key=P)


# All known words of edit distance 1 are infinitely more probable than known words of edit distance 2, and infinitely less probable than a known word of edit distance 0

def candidates(word):
    "Generate possible spelling correction for words"
    return(known([word]) or known(edits1(word)) or \
          known(edits2(word)) or [word])


# we don't need to multiply by a P(w|c) factor, because every candidate at the chosen priority will have the same probability

def known(words):
    "The subset of words that appear in the dictionary"
    return set(w for w in words if w in dictionary)


# **Sime Edit** - deletion (remove one letter), a transposition (swap two adjacent letters), a replacement (change one letter to another) or an insertion (add a letter)

def edits1(word):
    "All edits that are one edit away from word"
    letters ='abcdefghijklmnopqrstuvwxyz'
    splits = [(word[:i], word[i:]) for i in range(len(word)+1)]
    deletes = [L + R[1:] for L,R in splits if R]
    transposes = [L + R[1] +R[0]+R[2:] for L, R in splits if len(R)>1]
    replaces = [L +c +R[1:] for L,R in splits if R for c in letters]
    inserts = [L+ c+R  for L,R in splits for c in letters]
    return set(deletes + transposes +replaces +inserts)


len(edits1('inention'))

known(edits1('inention'))


def edits2(word):
    "two edits away from word"
    return (e2 for e1 in edits1(word) for e2 in edits1(e1))


len(set(edits2('inention')))

candidates('acient')

known(edits2('inention'))

len(dictionary)

sum(dictionary.values())

P('how')

P('invention')

dictionary.most_common(10)

correction('speling') # Missing letter

correction('korrectud') #Two letter correction

correction('inetion') #Missing two letters

['1','2','3'] or ['4','5']


# ### Test

def unit_tests():
    assert correction('speling') == 'spelling'              # insert
    assert correction('korrectud') == 'corrected'           # replace 2
    assert correction('bycycle') == 'bicycle'               # replace
    assert correction('inconvient') == 'inconvenient'       # insert 2
    assert correction('arrainged') == 'arranged'            # delete
    assert correction('peotry') =='poetry'                  # transpose
    assert correction('peotryy') =='poetry'                 # transpose + delete
    assert correction('word') == 'word'                     # known
    assert correction('quintessential') == 'quintessential' # unknown
    assert words('This is a TEST.') == ['this', 'is', 'a', 'test']
    assert Counter(words('This is a test. 123; A TEST this is.')) == (
           Counter({'123': 1, 'a': 2, 'is': 2, 'test': 2, 'this': 2}))
    assert len(dictionary) == 32198
    assert sum(dictionary.values()) == 1115585
#     assert dictionary.most_common(10) == [
#      ('the', 79808),
#      ('of', 40024),
#      ('and', 38311),
#      ('to', 28765),
#      ('in', 22020),
#      ('a', 21124),
#      ('that', 12512),
#      ('he', 12401),
#      ('was', 11410),
#      ('it', 10681)]
#    assert dictionary['the'] == 79808
    assert P('quintessential') == 0
    assert 0.07 < P('the') < 0.08
    return 'unit_tests pass'


def spelltest(tests, verbose=False):
    "Run correction(wrong) on all (right, wrong) pairs; report results."
    import time
    start = time.clock()
    good, unknown = 0, 0
    n = len(tests)
    for right, wrong in tests:
        w = correction(wrong)
        good += (w == right)
        if w != right:
            unknown += (right not in dictionary)
            if verbose:
                print('correction({}) => {} ({}); expected {} ({})'
                      .format(wrong, w, dictionary[w], right, dictionary[right]))
    dt = time.clock() - start
    print('{:.0%} of {} correct ({:.0%} unknown) at {:.0f} words per second '
          .format(good / n, n, unknown / n, n / dt))


def Testset(lines):
    "Parse 'right: wrong1 wrong2' lines into [('right', 'wrong1'), ('right', 'wrong2')] pairs."
    return [(right, wrong)
            for (right, wrongs) in (line.split(':') for line in lines)
            for wrong in wrongs.split()]


print(unit_tests())
spelltest(Testset(open('data/spell-testset1.txt'))) # Development set
spelltest(Testset(open('data/spell-testset2.txt'))) # Final test set



StarcoderdataJupyterScriptsDedupFiltered