Merge branch 'staging'

mathtext_fastapi/conversation_manager.py

@@ -39,6 +39,7 @@ def create_text_message(message_text, whatsapp_id):
         "preview_url": False,
         "recipient_type": "individual",
         "to": whatsapp_id,
+        # FIXME: Better to use "message_type" (but be careful with refactor)
         "type": "text",
         "text": {
             "body": message_text
@@ -136,6 +137,9 @@ def manage_math_quiz_fsm(user_message, contact_uuid, type):
 
     # Make a completely new entry
     if fsm_check.data == []:
+        # FIXME: Try not to use the Python reserved keyword `type` as a variable name
+        #        It's better to use `kind` or `convo_type` or `convo_name`
+        #        And the variable `type` is not defined here so I don't understand how this is working at all.
         if type == 'addition':
             math_quiz_state_machine = MathQuizFSM()
         else:

mathtext_fastapi/nlu.py

@@ -2,23 +2,32 @@ from collections.abc import Mapping
 from logging import getLogger
 import datetime as dt
 from dateutil.parser import isoparse
-import re
 
 from fuzzywuzzy import fuzz
 from mathtext_fastapi.logging import prepare_message_data_for_logging
 from mathtext.sentiment import sentiment
-from mathtext.text2int import text2int
-from mathtext_fastapi.intent_classification import create_intent_classification_model, retrieve_intent_classification_model, predict_message_intent
+from mathtext.text2int import text2int, TOKENS2INT_ERROR_INT
+from mathtext_fastapi.intent_classification import predict_message_intent
 
 log = getLogger(__name__)
 
-ERROR_CODE = 32202
+PAYLOAD_VALUE_TYPES = {
+    'author_id': str,
+    'author_type': str,
+    'contact_uuid': str,
+    'message_body': str,
+    'message_direction': str,
+    'message_id': str,
+    'message_inserted_at': str,
+    'message_updated_at': str,
+    'message_inserted_at': str,
+    }
 
 
-def build_nlu_response_object(type, data, confidence):
+def build_nlu_response_object(nlu_type, data, confidence):
     """ Turns nlu results into an object to send back to Turn.io
     Inputs
-    - type: str - the type of nlu run (integer or sentiment-analysis)
+    - nlu_type: str - the type of nlu run (integer or sentiment-analysis)
     - data: str/int - the student message
     - confidence: - the nlu confidence score (sentiment) or '' (integer)
 
@@ -28,7 +37,11 @@ def build_nlu_response_object(type, data, confidence):
     >>> build_nlu_response_object('sentiment', 'POSITIVE', 0.99)
     {'type': 'sentiment', 'data': 'POSITIVE', 'confidence': 0.99}
     """
-    return {'type': type, 'data': data, 'confidence': confidence}
+    return {
+        'type': nlu_type,
+        'data': data,
+        'confidence': confidence
+        }
 
 
 # def test_for_float_or_int(message_data, message_text):
@@ -144,7 +157,22 @@ def payload_is_valid(payload_object):
     """
     >>> payload_is_valid({'author_id': '+5555555', 'author_type': 'OWNER', 'contact_uuid': '3246-43ad-faf7qw-zsdhg-dgGdg', 'message_body': 'thirty one', 'message_direction': 'inbound', 'message_id': 'SDFGGwafada-DFASHA4aDGA', 'message_inserted_at': '2022-07-05T04:00:34.03352Z', 'message_updated_at': '2023-04-06T10:08:23.745072Z'})
     True
+
+    >>> message: {'author_id': '@event.message._vnd.v1.chat.owner', 'author_type': '@event.message._vnd.v1.author.type', 'contact_uuid': '@event.message._vnd.v1.chat.contact_uuid', 'message_body': '@event.message.text.body', 'message_direction': '@event.message._vnd.v1.direction', 'message_id': '@event.message.id', 'message_inserted_at': '@event.message._vnd.v1.chat.inserted_at', 'message_updated_at': '@event.message._vnd.v1.chat.updated_at'}
+    False
     """
+    try:
+        isinstance(
+            isoparse(payload_object.get('message_inserted_at')),
+            dt.datetime
+        )
+        isinstance(
+            isoparse(payload_object.get('message_updated_at')),
+            dt.datetime
+        )
+    except ValueError:
+        return False
+
     return (
         isinstance(payload_object, Mapping) and
         isinstance(payload_object.get('author_id'), str) and
@@ -155,17 +183,40 @@ def payload_is_valid(payload_object):
         isinstance(payload_object.get('message_id'), str) and
         isinstance(payload_object.get('message_inserted_at'), str) and
         isinstance(payload_object.get('message_updated_at'), str) and
-        isinstance(payload_object.get('message_inserted_at'), str) and
-        isinstance(payload_object.get('message_updated_at'), str) and
-        isinstance(
-            isoparse(payload_object.get('message_inserted_at')),
+        isinstance(payload_object.get('message_inserted_at'), str)       
+    )
+
+
+def log_payload_errors(payload_object):
+    errors = []
+    try:
+        assert isinstance(payload_object, Mapping)
+    except Exception as e:
+        log.error(f'Invalid HTTP request payload object: {e}')
+        errors.append(e)
+    for k, typ in PAYLOAD_VALUE_TYPES.items():
+        try:
+            assert isinstance(payload_object.get(k), typ)
+        except Exception as e:
+            log.error(f'Invalid HTTP request payload object: {e}')
+            errors.append(e)
+    try:
+        assert isinstance(
+            dt.datetime.fromisoformat(payload_object.get('message_inserted_at')),
             dt.datetime
-        ) and 
+        )
+    except Exception as e:
+        log.error(f'Invalid HTTP request payload object: {e}')
+        errors.append(e)
+    try: 
         isinstance(
-            isoparse(payload_object.get('message_updated_at')),
+            dt.datetime.fromisoformat(payload_object.get('message_updated_at')),
             dt.datetime
-        )        
-    )
+        )
+    except Exception as e:
+        log.error(f'Invalid HTTP request payload object: {e}')
+        errors.append(e)
+    return errors
 
 
 def evaluate_message_with_nlu(message_data):
@@ -182,14 +233,15 @@ def evaluate_message_with_nlu(message_data):
     log.info(f'Starting evaluate message: {message_data}')
 
     if not payload_is_valid(message_data):
-        return {'type': 'error', 'data': ERROR_CODE, 'confidence': 0}
+        log_payload_errors(message_data)
+        return {'type': 'error', 'data': TOKENS2INT_ERROR_INT, 'confidence': 0}
 
     try:
         message_text = str(message_data.get('message_body', ''))
     except:
         log.error(f'Invalid request payload: {message_data}')
         # use python logging system to do this//
-        return {'type': 'error', 'data': ERROR_CODE, 'confidence': 0}
+        return {'type': 'error', 'data': TOKENS2INT_ERROR_INT, 'confidence': 0}
 
     # Run intent classification only for keywords
     intent_api_response = run_intent_classification(message_text)
@@ -199,7 +251,7 @@ def evaluate_message_with_nlu(message_data):
 
     number_api_resp = text2int(message_text.lower())
 
-    if number_api_resp == ERROR_CODE:
+    if number_api_resp == TOKENS2INT_ERROR_INT:
         # Run intent classification with logistic regression model
         predicted_label = predict_message_intent(message_text)
         if predicted_label['confidence'] > 0.01:

scripts/bump_version.py

@@ -0,0 +1,36 @@
+#!/usr/bin/env python
+from pathlib import Path
+import re
+import shutil
+
+BASE_DIR = Path(__file__).parent.parent
+PYPROJECT_PATH = BASE_DIR / 'pyproject.toml'
+PATTERN = re.compile(r'(version\s*=\s*)[\'"]?(\d(\.\d+)+)[\'"]?\s*')
+
+if __name__ == '__main__':
+    verline = None
+    with PYPROJECT_PATH.open() as fin:
+        lines = []
+        verline = None
+        for line in fin:
+            lines.append(line)
+            if verline:
+                continue
+            match = PATTERN.match(line)
+            if match:
+                print(f'Found match.groups(): {dict(list(enumerate(match.groups())))}')
+                ver = [int(x) for x in match.groups()[1].split('.')]
+                print(f'             Old ver: {ver}')
+                ver[-1] += 1
+                print(f'             New ver: {ver}')
+                ver = '.'.join([str(x) for x in ver])
+                print(f'         New ver str: {ver}')
+                verline = f'version = "{ver}"\n'
+                print(f'        New ver line: {verline}')
+                lines[-1] = verline
+                print(f'        New ver line: {lines[-1]}')
+
+    if verline:
+        shutil.copy(PYPROJECT_PATH, PYPROJECT_PATH.with_suffix('.toml.bak'))
+        with PYPROJECT_PATH.open('w') as fout:
+            fout.writelines(lines)

scripts/cleanpyc.sh

@@ -0,0 +1,2 @@
+#!usr/bin/env bash
+find . | grep -E "(/__pycache__$|\.pyc$|\.pyo$)" | xargs rm -rf

scripts/make_request.py

@@ -48,18 +48,21 @@ def run_simulated_request(endpoint, sample_answer, context=None):
     print(f"Case: {sample_answer}")
     b_string = add_message_text_to_sample_object(sample_answer)
 
-    if endpoint == 'sentiment-analysis' or endpoint == 'text2int' or endpoint =='intent-classification':
-        request = requests.post(
-            url=f'http://localhost:7860/{endpoint}',
-            json={'content': sample_answer}
-        ).json()
-    else:
-        request = requests.post(
-            url=f'http://localhost:7860/{endpoint}',
-            data=b_string
-        ).json()
-
-    print(request)
+    print("BSTRING")
+    print(b_string)
+
+    # if endpoint == 'sentiment-analysis' or endpoint == 'text2int' or endpoint =='intent-classification':
+    #     request = requests.post(
+    #         url=f'http://localhost:7860/{endpoint}',
+    #         json={'content': sample_answer}
+    #     ).json()
+    # else:
+    #     request = requests.post(
+    #         url=f'http://localhost:7860/{endpoint}',
+    #         data=b_string
+    #     ).json()
+
+    # print(request)
 
 
 # run_simulated_request('intent-classification', 'exit')

scripts/pin_requirements.py

@@ -0,0 +1,62 @@
+""" Parse requirements.txt and pyproject.toml and move versions to pyproject.toml """
+from pathlib import Path
+import re
+import sys
+import toml
+
+def get_requirement_versions(path='requirements.txt'):
+    """ Read requirements.txt file and return dict of package versions """
+    path = Path(path or '')
+    if path.is_dir():
+        path = next(iter(path.glob('**/requirements.txt')))
+    reqdict = {}
+    text = Path(path).open().read()
+    for line in text.splitlines():
+        if line.strip():
+            match = re.match(r'([-_a-zA-Z0-9]+)\s*([ >=<~^,.rabc0-9]+)\s*', line)
+            if match:
+                name, ver = match.groups()
+                reqdict[name] = ver
+    return reqdict  
+
+
+def normalize_name(name):
+    return str(name).strip().replace('_', '-').replace(' ', '-').lower()
+
+
+def pin_versions(pyproject='pyproject.toml', reqdict=None, overwrite=False):
+    if not reqdict or isinstance(reqdict, (str, Path)):
+        reqdict = get_requirement_versions(path=reqdict)
+    reqdict = {
+        normalize_name(k): v for (k, v) in 
+        reqdict.items()
+        }
+    
+    pyproj = toml.load(pyproject)
+    depdict = pyproj.get('tool', {}).get('poetry', {}).get('dependencies', {})
+    depdict = {
+        normalize_name(k): v for (k, v) in 
+        depdict.items()
+        }
+    
+    for name, spec in reqdict.items():
+        if name in depdict:
+            ver = depdict[name]
+            if isinstance(ver, str) and (overwrite or ver == '*'):
+                depdict[name] = spec
+
+    pyproj['tool']['poetry']['dependencies'] = depdict
+    overwrite = overwrite or (input(f'Overwrite {pyproject}?')[0].lower() == 'y')
+    if overwrite:
+        with open(pyproject, 'w') as stream:
+            toml.dump(pyproj, stream)
+    return pyproj
+
+
+if __name__ == '__main__':
+    path = 'requirements.txt'
+    if sys.argv[1:]:
+        path = sys.argv[1]
+    pyproj = pin_versions(reqdict=path)
+    print(toml.dumps(pyproj))
+