add dataset

.gitattributes

@@ -33,3 +33,7 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
 *.zip filter=lfs diff=lfs merge=lfs -text
 *.zst filter=lfs diff=lfs merge=lfs -text
 *tfevents* filter=lfs diff=lfs merge=lfs -text
+*.jsonl filter=lfs diff=lfs merge=lfs -text
+*.json filter=lfs diff=lfs merge=lfs -text
+*.csv filter=lfs diff=lfs merge=lfs -text
+*.txt filter=lfs diff=lfs merge=lfs -text

NL2TL-dataset/NLTLsummary.json

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:6291896067a809119e01b43734779333c2d6b7baa36376afe8aa6579b4ba77ee
+size 23025

NL2TL-dataset/collect/Cleaned_ENG.txt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:77f8e6b696147098eb16be8ec7a2891a493c008e7d997f994d1b77fea4255559
+size 447181

NL2TL-dataset/collect/Cleaned_LTL.txt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:3ced5c8ee114bc50e480c44999e991337318749dd7f5988379c0f2cd3c5940d2
+size 252706

NL2TL-dataset/collect/UNCleaned_ENG.txt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:29fe1e57c68d9096705eae98f6456aa24eb86ecc2a43fa7896785a30eba1c3d0
+size 79381

NL2TL-dataset/collect/UNCleaned_LTL.txt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:f903103fcdc5b86bf1aca3568fd5483bfab90412fb1581006c0433c4ac0feecc
+size 34258

NL2TL-dataset/collect/UNCleaned_num.txt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:50067192dcabdc789ff68998a4b67b44d5ea71ad870b2f3029f06a62e93cd1cc
+size 7639

NL2TL-dataset/collect/eng.txt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:834f288fc7c4bf3c27e829db10d72f673b55551f5e50591e9ff05461b78417e7
+size 384436

NL2TL-dataset/collect/eng_gpt_auged.txt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:b5d039a94cfde70475d1c935e80286d0c3bb9c578f01c86cf8f65aed015fdf14
+size 46038

NL2TL-dataset/collect/eng_gpt_auged2.txt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:6484a6ace4d0bdd6e00b337aa2e734d4cd3cfdbcd03f90a960ea3859381ffaad
+size 96837

NL2TL-dataset/collect/eng改过了不太好.txt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e74a45639f6d71263ea70768e40a4d299137fca66de36da7657a15cbba60beb9
+size 379834

NL2TL-dataset/collect/idxsrc_gpt_auged.txt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:ce789edf73241091f48e1e3a2b4eddf06ae0244fc20d4f050cbed474afd79ab6
+size 4425

NL2TL-dataset/collect/idxsrc_gpt_auged2.txt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:a71b50098910d35585cddb30b9fd0187898db7e0ea1ae5cc94a5bf7f100e81a4
+size 9273

NL2TL-dataset/collect/log.jsonl

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:f3a3e9ba897654fe39da61e0fc20687714fb1cd6da68d98eb604731b20d14fce
+size 561367

NL2TL-dataset/collect/ltl copy.txt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:161b2a0b67725db0aa2e803c4576fc47cfb0698f66469c660941a0ddbcaba76c
+size 192556

NL2TL-dataset/collect/ltl.txt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:161b2a0b67725db0aa2e803c4576fc47cfb0698f66469c660941a0ddbcaba76c
+size 192556

NL2TL-dataset/collect/ltl_eng_1.jsonl

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:55879344ea92bf9073b1b605dda98cfe551b9169186dde31592d15c9a24d47e4
+size 1043402

NL2TL-dataset/collect/ltl_eng_mid_order_1.jsonl

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e61fbb4657c7b6d81d26c6a8fa596f03156dba6308b17b54125f79cd3734c058
+size 1069183

NL2TL-dataset/collect/ltl_eng_test-aug.jsonl

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:0982bf8999ef200b1d7fa5c867b70e41caf136b7a4422e73eb303f7ee77b02dd
+size 131838

NL2TL-dataset/collect/ltl_eng_test.jsonl

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:0982bf8999ef200b1d7fa5c867b70e41caf136b7a4422e73eb303f7ee77b02dd
+size 131838

NL2TL-dataset/collect/ltl_eng_test_mid.jsonl

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:bd4e75373318179d3cd3a2bbb330fdfe75376dabe77f475f31d95cf27f4d100c
+size 137907

NL2TL-dataset/collect/ltl_eng_test_mid_ascii.jsonl

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:9185fd98062c160c363f35116a943a0950fdd2151bd7ff61cb9df39921ca73f3
+size 137931

NL2TL-dataset/collect/ltl_eng_test_mid_ascii_gptAuged.jsonl

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:5ca41a9c7ea7dd2151e861ac7754f36eee0e386f7a0dbbf7d2d5cabf927ccb7b
+size 125920

NL2TL-dataset/collect/ltl_eng_train-aug.jsonl

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:4817c9d380816056e05c0bec1f985ac90dbc1068417db5d033d15bfe4273d279
+size 911564

NL2TL-dataset/collect/ltl_eng_train.jsonl

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:4817c9d380816056e05c0bec1f985ac90dbc1068417db5d033d15bfe4273d279
+size 911564

NL2TL-dataset/collect/ltl_eng_train_mid.jsonl

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:0daf22f37a11d38edaacfd837ccabf59fc7395431166bc50b0336e030faf77d1
+size 937958

NL2TL-dataset/collect/ltl_eng_train_mid_ascii.jsonl

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e3674e9aabc015509efadbe1f8374627d5aaaacfb8fe3c717fa1ff852dcb8c4f
+size 937958

NL2TL-dataset/collect/ltl_eng_train_mid_ascii_gptAuged.jsonl

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:cea13abc2204906d232ea88b78830ad6be09e01dbd5b7ad7ec9da9dff2f6c777
+size 1129386

NL2TL-dataset/collect/ltl_mid_order.txt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:646500a7231278e0dd4ca0ed6fc2cc71ba4f3b7284a13a720a689a8e634db78e
+size 225055

NL2TL-dataset/collect/ltl_mid_order_ascii.txt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:f56781a159d3a52b58101038f0d15e1ef054b5f0290dd73228e51f4004243dd6
+size 225073

NL2TL-dataset/collect/ltl_mid_order_ascii_gpt_auged.txt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:f332db20802ecce5a8bd2879316e66061c97d438d18c7cbc6d4899b3bfbcaa44
+size 19575

NL2TL-dataset/collect/ltl_mid_order_ascii_gpt_auged2.txt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:bfca7f2b1129e28f89e4dd68d76406ad26404514d08864aaf1c76f3fa01eb132
+size 40805

NL2TL-dataset/collect/note.txt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:7798d0398769800f3cfc50fc902cec4cd1aa1eff05e3cb41790094c620f61039
+size 59

NL2TL-dataset/collect2/CW_total_3382_for_transfer_word_midfix.jsonl

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:c131c3486f4debdf0ed416248de2892d41627abbf2fcd83c69051b7ba69bfb2b
+size 449106

NL2TL-dataset/collect2/GLTL_train_8923_for_transfer_word_midfix.jsonl

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:cc7aff94ba621fb7def1b174b2f5ee0e1336b572ed65bb7c15f03f1295156930
+size 1839113

NL2TL-dataset/collect2/LTLexplain_0.json

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:08175568f9105c8dd5e8e5413e8a90834ab48b0cd2c8396ca1a71ab97f69ef68
+size 149375

NL2TL-dataset/collect2/LTLexplain_1.json

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:d2a0f1e1bfe11526ca4ad549cf263fcfd60d18d9ee27cf77b889a3d0b8728d37
+size 149123

NL2TL-dataset/collect2/LTLexplain_2.json

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:64ee6dd1b5f76edc89b57cc4384a7dc1ea1af31940016f844c8a22ee0b98c0e4
+size 82749

NL2TL-dataset/collect2/LTLexplain_3.json

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:052761b9c005136e13576c0bb4f279ad5c81e9e15abfd736e344028b32e8af89
+size 73448

NL2TL-dataset/collect2/LTLexplain_4.json

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:d3fa922f8b8402b2f950c18de7e2360a0aa7158297251e0ea1fa2d6c26337ccf
+size 147624

NL2TL-dataset/collect2/LTLsummary.json

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:11079ebb03aa8d76763c690f7c59431f32c25ce385657b3e68daa1b15c7ff734
+size 22393

NL2TL-dataset/collect2/getUniqueLTL.py

@@ -0,0 +1,20 @@
+import json 
+import re,os 
+
+def findUniqueLTL(paths:list):
+    ret={}
+    for path in paths:
+        with open(path,'r') as f:
+            jsonlists=f.readlines()
+            for jsonlist in jsonlists:
+                j=json.loads(jsonlist)
+                ret[j['raw_ltl']]=1
+    return ret
+
+if __name__=='__main__':
+    path=['/home/user/xsj/NL2TL-dataset/collect2/ltl_eng_test_mid_ascii_gptAuged.jsonl','/home/user/xsj/NL2TL-dataset/collect2/ltl_eng_train_mid_ascii_gptAuged.jsonl']
+    LTLs=findUniqueLTL(paths=path)
+    with open(os.path.join('/home/user/xsj/NL2TL-dataset/collect2','NLTLsummary.json'),'w') as f :
+        f.write(json.dumps(LTLs,sort_keys=False,indent=4,separators=(',',':')))
+    
+

NL2TL-dataset/collect2/lifted_data.jsonl

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:c68ab33843fe0485380f33b6e49cb7c9230eae66252d869083584dd7ef048afb
+size 12458149

NL2TL-dataset/collect2/ltl_eng_test_mid_ascii_gptAuged.jsonl

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:41061064f1591c833a29ea73ce1888aaa831b488b0f4f0f2a04994c871a42873
+size 140979

NL2TL-dataset/collect2/ltl_eng_train_mid_ascii_gptAuged.jsonl

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:24f9856d3e7f3d882de35caf5e3d79e70b783630178e1fcbad197b31431b2bfa
+size 1264107

NL2TL-dataset/collect2/navi_total_refined.jsonl

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:11499148f1f053c86476f08ec39f9cb0cb724eb9451d0153e2f34c77a04855b8
+size 3825939

NL2TL-dataset/datasets-Efficient-Eng-2-LTL/augment.ipynb

@@ -0,0 +1,374 @@
+{
+ "cells": [
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "# Augmentation by parapharsing"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "## Init & Load Seed Data"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "import json, openai\n",
+    "from tqdm import tqdm "
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "DOMAIN = \"drone-planning/\"\n",
+    "# DOMAIN = \"clean-up/\"\n",
+    "# DOMAIN = \"pick-and-place/\"\n",
+    "with open(DOMAIN + \"train_seed.jsonl\") as f:\n",
+    "    train_seed = [json.loads(line) for line in f]"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "eng_seeds = {\n",
+    "    seed['natural']: [] for seed in train_seed\n",
+    "}"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "## Augmentation Code\n",
+    "prompting GPT-3 seems to work the best in this case"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "# You need to set your OPENAI API key here\n",
+    "# https://beta.openai.com/account/api-keys\n",
+    "openai.api_key = \"TO_BE_SET\""
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "def normalize(sentence):\n",
+    "    # captialize first letter and add period at the end if not present\n",
+    "    if sentence[0].islower():\n",
+    "        sentence = sentence[0].upper() + sentence[1:]\n",
+    "    if sentence[-1] != '.':\n",
+    "        sentence = sentence + '.'\n",
+    "    return sentence\n",
+    "\n",
+    "def parse_sentences_from_response(response):\n",
+    "    lines = response.split('\\n')\n",
+    "    # assert len(lines) == 5\n",
+    "    assert len(lines) == 10\n",
+    "    lines[0] = \"1.\" + lines[0]\n",
+    "    paraphrases = []\n",
+    "    for idx, line in enumerate(lines):\n",
+    "        assert line.startswith(str(idx+1) + '. ')\n",
+    "        sentence_start_idx = len(str(idx+1) + '. ')\n",
+    "        paraphrases.append(line[sentence_start_idx:])\n",
+    "    for paraphrase in paraphrases:\n",
+    "        if paraphrase[-1] == ' ':\n",
+    "            if paraphrase[-2] == '.':\n",
+    "                paraphrase = paraphrase[:-1]\n",
+    "            else:\n",
+    "                paraphrase = paraphrase[:-2] + '.'\n",
+    "    return paraphrases\n",
+    "\n",
+    "\n",
+    "PROMPT = \"\"\"Rephrase the source sentence in 10 different ways. Make the outputs as diverse as possible.\n",
+    "\n",
+    "Source: \n",
+    "SOURCE-TO-BE-PLACED\n",
+    "\n",
+    "Outputs:\n",
+    "1.\"\"\"\n",
+    "def rephrase_a_sentence(sentence):\n",
+    "    response = openai.Completion.create(\n",
+    "        model=\"text-davinci-002\",\n",
+    "        prompt=PROMPT.replace(\"SOURCE-TO-BE-PLACED\", normalize(sentence)),\n",
+    "        temperature=0.7,\n",
+    "        max_tokens=512,\n",
+    "        top_p=1,\n",
+    "        best_of=1,\n",
+    "        frequency_penalty=0.1,\n",
+    "        presence_penalty=0\n",
+    "        )\n",
+    "    output = response['choices'][0]['text']\n",
+    "    try:\n",
+    "        paraphrases = parse_sentences_from_response(output)\n",
+    "    except:\n",
+    "        print(\"Error in parsing response\")\n",
+    "        print(output)\n",
+    "        return output, \"ERROR\"\n",
+    "    return parse_sentences_from_response(output)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "O = rephrase_a_sentence(\"Go to the red room or go to the green room to finally go to the blue room.\")"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "O"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "## Run Augmentation"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "len(eng_seeds)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "list(eng_seeds.keys())[0]"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "def paraphrase_done(eng_seeds):\n",
+    "    for eng_seed, extended in tqdm(eng_seeds.items()):\n",
+    "        if len(extended) == 0:\n",
+    "            return False\n",
+    "    return True\n",
+    "\n",
+    "while not paraphrase_done(eng_seeds):\n",
+    "    for eng_seed, extended in tqdm(eng_seeds.items()):\n",
+    "        if len(extended) == 0:\n",
+    "            extended += rephrase_a_sentence(eng_seed)"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "eng_seeds"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "### Dump as Training Data"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "train_seed[0]"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "with open(DOMAIN + \"syn-aug.train.jsonl\", 'w') as f:\n",
+    "    for seed in train_seed:\n",
+    "        f.write(json.dumps(seed) + '\\n')\n",
+    "        for aug_eng in eng_seeds[seed['natural']]:\n",
+    "                f.write(json.dumps({\n",
+    "                    'natural': aug_eng,\n",
+    "                    'canonical': seed['canonical'],\n",
+    "                    'formula': seed['formula']\n",
+    "                }) + '\\n')"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "with open(DOMAIN + \"syn.train.jsonl\", 'w') as f:\n",
+    "    for seed in train_seed:\n",
+    "        f.write(json.dumps(seed) + '\\n')"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "### Normalize the natural language form "
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "if DOMAIN == \"clean-up/\":\n",
+    "    # in clean up, golden natural language data comes without period at the end, no capitalization in the beginning\n",
+    "    def clean_up_normalize(sentence):\n",
+    "        if sentence[0].isupper():\n",
+    "            sentence = sentence[0].lower() + sentence[1:]\n",
+    "        if sentence[-1] == '.':\n",
+    "            sentence = sentence[:-1]\n",
+    "        return sentence\n",
+    "\n",
+    "    buffer = []\n",
+    "    with open(DOMAIN + \"syn-aug.train.jsonl\", 'r') as f:\n",
+    "        for l in f.readlines():\n",
+    "            buffer.append(json.loads(l))\n",
+    "    \n",
+    "    with open(DOMAIN + \"syn-aug.train.jsonl\", 'w') as f:\n",
+    "        for dp in buffer:\n",
+    "            f.write(json.dumps({\n",
+    "                'natural': clean_up_normalize(dp['natural']),\n",
+    "                'canonical': dp['canonical'],\n",
+    "                'formula': dp['formula']\n",
+    "            }) + '\\n')\n",
+    "\n",
+    "if DOMAIN == \"pick-and-place/\":\n",
+    "    # in pick and place, golden natural language data comes without period at the end, no capitalization in the beginning\n",
+    "    def clean_up_normalize(sentence):\n",
+    "        if sentence[0].isupper():\n",
+    "            sentence = sentence[0].lower() + sentence[1:]\n",
+    "        if sentence[-1] == '.':\n",
+    "            sentence = sentence[:-1]\n",
+    "        return sentence\n",
+    "\n",
+    "    buffer = []\n",
+    "    with open(DOMAIN + \"syn-aug.train.jsonl\", 'r') as f:\n",
+    "        for l in f.readlines():\n",
+    "            buffer.append(json.loads(l))\n",
+    "    \n",
+    "    with open(DOMAIN + \"syn-aug.train.jsonl\", 'w') as f:\n",
+    "        for dp in buffer:\n",
+    "            f.write(json.dumps({\n",
+    "                'natural': clean_up_normalize(dp['natural']),\n",
+    "                'canonical': dp['canonical'],\n",
+    "                'formula': dp['formula']\n",
+    "            }) + '\\n')"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "if DOMAIN == \"drone-planning/\":\n",
+    "    # in clean up, golden natural language data comes with a \"space + period\" at the end, no capitalization in the beginning\n",
+    "    def clean_up_normalize(sentence):\n",
+    "        if sentence[0].isupper():\n",
+    "            sentence = sentence[0].lower() + sentence[1:]\n",
+    "        while sentence[-1] == ' ' or sentence[-1] == '.' or sentence[-1] == '!':\n",
+    "            sentence = sentence[:-1]\n",
+    "        sentence = sentence + '.'\n",
+    "        sentence = sentence.replace('.', ' .')\n",
+    "        sentence = sentence.replace(',', ' ,')\n",
+    "        return sentence\n",
+    "\n",
+    "    buffer = []\n",
+    "    # with open(DOMAIN + \"syn-aug.train.jsonl\", 'r') as f:\n",
+    "    #     for l in f.readlines():\n",
+    "    #         buffer.append(json.loads(l))\n",
+    "    \n",
+    "    # with open(DOMAIN + \"syn-aug.train.jsonl\", 'w') as f:\n",
+    "    #     for dp in buffer:\n",
+    "    #         f.write(json.dumps({\n",
+    "    #             'natural': clean_up_normalize(dp['natural']),\n",
+    "    #             'canonical': dp['canonical'],\n",
+    "    #             'formula': dp['formula']\n",
+    "    #         }) + '\\n')\n",
+    "    with open(DOMAIN + \"syn.train.jsonl\", 'r') as f:\n",
+    "        for l in f.readlines():\n",
+    "            buffer.append(json.loads(l))\n",
+    "    \n",
+    "    with open(DOMAIN + \"syn.train.jsonl\", 'w') as f:\n",
+    "        for dp in buffer:\n",
+    "            f.write(json.dumps({\n",
+    "                'natural': clean_up_normalize(dp['natural']),\n",
+    "                'canonical': dp['canonical'],\n",
+    "                'formula': dp['formula']\n",
+    "            }) + '\\n')"
+   ]
+  }
+ ],
+ "metadata": {
+  "kernelspec": {
+   "display_name": "GPML",
+   "language": "python",
+   "name": "python3"
+  },
+  "language_info": {
+   "codemirror_mode": {
+    "name": "ipython",
+    "version": 3
+   },
+   "file_extension": ".py",
+   "mimetype": "text/x-python",
+   "name": "python",
+   "nbconvert_exporter": "python",
+   "pygments_lexer": "ipython3",
+   "version": "3.7.13"
+  },
+  "orig_nbformat": 4,
+  "vscode": {
+   "interpreter": {
+    "hash": "75567ad983eac98a78c1e40a895e8d82557b42cf9969286235abec07ddbf9e7d"
+   }
+  }
+ },
+ "nbformat": 4,
+ "nbformat_minor": 2
+}

NL2TL-dataset/datasets-Efficient-Eng-2-LTL/clean-up/canonical-lifted.json

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:5dfa917383f4e67d55a159050ffa60c83dea987236ef516edf7d2e45e16689d3
+size 7890

NL2TL-dataset/datasets-Efficient-Eng-2-LTL/clean-up/canonical.json

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:7554f76c648d84596c8a77458bd615550be66d0903143146a8497797205315f4
+size 12442

NL2TL-dataset/datasets-Efficient-Eng-2-LTL/clean-up/golden-lifted.jsonl

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:24cc3906e8d32f3a1bad606ac8d262608065aa6ffbe3065fdf98b4b1fb754bc7
+size 599574