# ---
# jupyter:
#   jupytext:
#     text_representation:
#       extension: .py
#       format_name: light
#       format_version: '1.5'
#       jupytext_version: 1.14.4
#   kernelspec:
#     display_name: Python 3
#     language: python
#     name: python3
# ---

wlist = "book the flight through Houston".split()

Lexicon = {
    "that:0.1 | this:0 | the:0.6 | a:0.3": "Det",
    "book:0.1 | flight:0.3 | meal:0.05 | money:0.05 | food:0.4 | dinner:0.1": "Noun",
    "book:0.3 | include:0.3 | prefer:0.4": "Verb",
    "I:0.4 | she:0.05 | me:0.15 | you:0.4": "Pronoun",
    "Houston:0.6 | NWA:0.4": "Proper-Noun",
    "does:0.6 | can:0.4": "Aux",
    "from:0.3 | to:0.3 | on:0.2 | near:0.15 | through:0.05": "Preposition"
}

Grammar = {
    "S -> NP VP": 0.8,
    "S -> Aux NP VP": 0.15,
    "S -> VP": 0.05,
    "NP -> Pronoun": 0.35,
    "NP -> Proper-Noun": 0.3,
    "NP -> Det Nominal": 0.2,
    "NP -> Nominal": 0.15,
    "Nominal -> Noun": 0.75,
    "Nominal -> Nominal Noun": 0.2,
    "Nominal -> Nominal PP": 0.05,
    "VP -> Verb": 0.35,
    "VP -> Verb NP": 0.2,
    "VP -> Verb NP PP": 0.1,
    "VP -> Verb PP": 0.15,
    "VP -> Verb NP NP": 0.05,
    "VP -> VP PP": 0.15,
    "PP -> Preposition NP": 1,
    }


def Lexicon2CNF(Lexicon: dict) -> dict:
    res = {}
    for key,value in Lexicon.items():
        for item in key.split(" | "):
            w,p = item.split(":")
            res[value + " -> " + w] = p
    return res


def Grammar2CNF(Grammar: dict) -> dict:
    res = {}
    i = 0
    for k,v in Grammar.items():
        l,r = k.split(" -> ")
        rlist = r.split(" ")
        if len(rlist) == 1:
            for wf,n in Lexicon.items():
                if n == r:
                    res[l + " -> " + wf] = v
        elif len(rlist) == 2:
            res[k] = v
        else:
            i += 1
            newr1 = " ".join(rlist[: 2])
            newr2 = "X" + str(i) + " " + rlist[-1]
            res["X" + str(i) + " -> " + newr1] = 1
            res[l + " -> " + newr2] = v
    miduse = res.copy()
#     count = {}
#     for k,v in Grammar.items():
#         l,r = k.split(" -> ")
#         rlist = r.split(" ")
#         if len(rlist) == 1:
#             for kk,vv in miduse.items():
#                 ll,rr = kk.split(" -> ")
#                 if r == ll:
#                     if k in count:
#                         count[k] += 1
#                     else:
#                         count[k] = 1
    for k,v in Grammar.items():
        l,r = k.split(" -> ")
        rlist = r.split(" ")
        if len(rlist) == 1:
            for kk,vv in miduse.items():
                ll,rr = kk.split(" -> ")
                if r == ll:
                    res[l + " -> " + rr] = v
    return res


LexiconCNF = Lexicon2CNF(Lexicon)
GrammarCNF = Grammar2CNF(Grammar)

LexiconCNF

GrammarCNF



StarcoderdataJupyterScriptsDedupFiltered