Rename gingerit.py to gramformer.py

gingerit.py

@@ -1,77 +0,0 @@
-
-import requests
-import cloudscraper
-
-URL = "https://services.gingersoftware.com/Ginger/correct/jsonSecured/GingerTheTextFull"  # noqa
-API_KEY = "6ae0c3a0-afdc-4532-a810-82ded0054236"
-
-class GingerIt(object):
-    def __init__(self):
-        self.url = URL
-        self.api_key = API_KEY
-        self.api_version = "2.0"
-        self.lang = "US"
-
-    def parse(self, text, verify=True):
-        session = cloudscraper.create_scraper()
-        try:
-            request = session.get(
-                self.url,
-                params={
-                    "lang": self.lang,
-                    "apiKey": self.api_key,
-                    "clientVersion": self.api_version,
-                    "text": text,
-                },
-                verify=verify,
-            )
-            
-            # Print the raw response text for debugging
-            print("Response Text:", request.text)
-            
-            # Try parsing the response as JSON
-            try:
-                data = request.json()
-            except ValueError:
-                print("Failed to parse JSON response. Returning original text.")
-                return {"text": text, "result": text, "corrections": []}
-            
-            # Process and return the corrected data
-            return self._process_data(text, data)
-        
-        except requests.exceptions.RequestException as e:
-            print(f"An error occurred during the API request: {e}")
-            return {"text": text, "result": text, "corrections": []}
-
-    @staticmethod
-    def _change_char(original_text, from_position, to_position, change_with):
-        return "{}{}{}".format(
-            original_text[:from_position], change_with, original_text[to_position + 1 :]
-        )
-
-    def _process_data(self, text, data):
-        result = text
-        corrections = []
-
-        if "Corrections" not in data:
-            print("No corrections found in the API response.")
-            return {"text": text, "result": text, "corrections": corrections}
-
-        for suggestion in reversed(data["Corrections"]):
-            start = suggestion["From"]
-            end = suggestion["To"]
-
-            if suggestion["Suggestions"]:
-                suggest = suggestion["Suggestions"][0]
-                result = self._change_char(result, start, end, suggest["Text"])
-
-                corrections.append(
-                    {
-                        "start": start,
-                        "text": text[start : end + 1],
-                        "correct": suggest.get("Text", None),
-                        "definition": suggest.get("Definition", None),
-                    }
-                )
-
-        return {"text": text, "result": result, "corrections": corrections}

gramformer.py

@@ -0,0 +1,128 @@
+class Gramformer:
+
+  def __init__(self, models=1, use_gpu=False):
+    from transformers import AutoTokenizer
+    from transformers import AutoModelForSeq2SeqLM
+    #from lm_scorer.models.auto import AutoLMScorer as LMScorer
+    import errant
+    self.annotator = errant.load('en')
+    
+    if use_gpu:
+        device= "cuda:0"
+    else:
+        device = "cpu"
+    batch_size = 1    
+    #self.scorer = LMScorer.from_pretrained("gpt2", device=device, batch_size=batch_size)    
+    self.device    = device
+    correction_model_tag = "prithivida/grammar_error_correcter_v1"
+    self.model_loaded = False
+
+    if models == 1:
+        self.correction_tokenizer = AutoTokenizer.from_pretrained(correction_model_tag, use_auth_token=False)
+        self.correction_model     = AutoModelForSeq2SeqLM.from_pretrained(correction_model_tag, use_auth_token=False)
+        self.correction_model     = self.correction_model.to(device)
+        self.model_loaded = True
+        print("[Gramformer] Grammar error correct/highlight model loaded..")
+    elif models == 2:
+        # TODO
+        print("TO BE IMPLEMENTED!!!")
+
+  def correct(self, input_sentence, max_candidates=1):
+      if self.model_loaded:
+        correction_prefix = "gec: "
+        input_sentence = correction_prefix + input_sentence
+        input_ids = self.correction_tokenizer.encode(input_sentence, return_tensors='pt')
+        input_ids = input_ids.to(self.device)
+
+        preds = self.correction_model.generate(
+            input_ids,
+            do_sample=True, 
+            max_length=128, 
+#             top_k=50, 
+#             top_p=0.95, 
+            num_beams=7,
+            early_stopping=True,
+            num_return_sequences=max_candidates)
+
+        corrected = set()
+        for pred in preds:  
+          corrected.add(self.correction_tokenizer.decode(pred, skip_special_tokens=True).strip())
+
+        #corrected = list(corrected)
+        #scores = self.scorer.sentence_score(corrected, log=True)
+        #ranked_corrected = [(c,s) for c, s in zip(corrected, scores)]
+        #ranked_corrected.sort(key = lambda x:x[1], reverse=True)
+        return corrected
+      else:
+        print("Model is not loaded")  
+        return None
+
+  def highlight(self, orig, cor):
+      edits = self._get_edits(orig, cor)
+      orig_tokens = orig.split()
+
+      ignore_indexes = []
+
+      for edit in edits:
+          edit_type = edit[0]
+          edit_str_start = edit[1]
+          edit_spos = edit[2]
+          edit_epos = edit[3]
+          edit_str_end = edit[4]
+
+          # if no_of_tokens(edit_str_start) > 1 ==> excluding the first token, mark all other tokens for deletion
+          for i in range(edit_spos+1, edit_epos):
+            ignore_indexes.append(i)
+
+          if edit_str_start == "":
+              if edit_spos - 1 >= 0:
+                  new_edit_str = orig_tokens[edit_spos - 1]
+                  edit_spos -= 1
+              else:
+                  new_edit_str = orig_tokens[edit_spos + 1]
+                  edit_spos += 1
+              if edit_type == "PUNCT":
+                st = "<a type='" + edit_type + "' edit='" + \
+                    edit_str_end + "'>" + new_edit_str + "</a>"
+              else:
+                st = "<a type='" + edit_type + "' edit='" + new_edit_str + \
+                    " " + edit_str_end + "'>" + new_edit_str + "</a>"
+              orig_tokens[edit_spos] = st
+          elif edit_str_end == "":
+            st = "<d type='" + edit_type + "' edit=''>" + edit_str_start + "</d>"
+            orig_tokens[edit_spos] = st
+          else:
+            st = "<c type='" + edit_type + "' edit='" + \
+                edit_str_end + "'>" + edit_str_start + "</c>"
+            orig_tokens[edit_spos] = st
+
+      for i in sorted(ignore_indexes, reverse=True):
+        del(orig_tokens[i])
+
+      return(" ".join(orig_tokens))
+
+  def detect(self, input_sentence):
+        # TO BE IMPLEMENTED
+        pass
+
+  def _get_edits(self, orig, cor):
+        orig = self.annotator.parse(orig)
+        cor = self.annotator.parse(cor)
+        alignment = self.annotator.align(orig, cor)
+        edits = self.annotator.merge(alignment)
+
+        if len(edits) == 0:  
+            return []
+
+        edit_annotations = []
+        for e in edits:
+            e = self.annotator.classify(e)
+            edit_annotations.append((e.type[2:], e.o_str, e.o_start, e.o_end,  e.c_str, e.c_start, e.c_end))
+                
+        if len(edit_annotations) > 0:
+            return edit_annotations
+        else:    
+            return []
+
+  def get_edits(self, orig, cor):
+      return self._get_edits(orig, cor)