Language Translator

.gitattributes

@@ -1,35 +0,0 @@
-*.7z filter=lfs diff=lfs merge=lfs -text
-*.arrow filter=lfs diff=lfs merge=lfs -text
-*.bin filter=lfs diff=lfs merge=lfs -text
-*.bz2 filter=lfs diff=lfs merge=lfs -text
-*.ckpt filter=lfs diff=lfs merge=lfs -text
-*.ftz filter=lfs diff=lfs merge=lfs -text
-*.gz filter=lfs diff=lfs merge=lfs -text
-*.h5 filter=lfs diff=lfs merge=lfs -text
-*.joblib filter=lfs diff=lfs merge=lfs -text
-*.lfs.* filter=lfs diff=lfs merge=lfs -text
-*.mlmodel filter=lfs diff=lfs merge=lfs -text
-*.model filter=lfs diff=lfs merge=lfs -text
-*.msgpack filter=lfs diff=lfs merge=lfs -text
-*.npy filter=lfs diff=lfs merge=lfs -text
-*.npz filter=lfs diff=lfs merge=lfs -text
-*.onnx filter=lfs diff=lfs merge=lfs -text
-*.ot filter=lfs diff=lfs merge=lfs -text
-*.parquet filter=lfs diff=lfs merge=lfs -text
-*.pb filter=lfs diff=lfs merge=lfs -text
-*.pickle filter=lfs diff=lfs merge=lfs -text
-*.pkl filter=lfs diff=lfs merge=lfs -text
-*.pt filter=lfs diff=lfs merge=lfs -text
-*.pth filter=lfs diff=lfs merge=lfs -text
-*.rar filter=lfs diff=lfs merge=lfs -text
-*.safetensors filter=lfs diff=lfs merge=lfs -text
-saved_model/**/* filter=lfs diff=lfs merge=lfs -text
-*.tar.* filter=lfs diff=lfs merge=lfs -text
-*.tar filter=lfs diff=lfs merge=lfs -text
-*.tflite filter=lfs diff=lfs merge=lfs -text
-*.tgz filter=lfs diff=lfs merge=lfs -text
-*.wasm filter=lfs diff=lfs merge=lfs -text
-*.xz filter=lfs diff=lfs merge=lfs -text
-*.zip filter=lfs diff=lfs merge=lfs -text
-*.zst filter=lfs diff=lfs merge=lfs -text
-*tfevents* filter=lfs diff=lfs merge=lfs -text

.gitignore

@@ -0,0 +1,3 @@
+.idea
+.DS_Store
+__pycache__

LanguageTranslator/model/openai_model.py

@@ -0,0 +1,55 @@
+import openai
+import time
+import math
+
+class Model:
+    def make_text_prompt(self, text: str, target_language: str) -> str:
+        # Create a text prompt for translation
+        return f"Translate to {target_language}: {text}"
+
+    def translate_prompt(self, content, target_language: str) -> str:
+        # Generate a translation prompt based on the content type
+        if isinstance(content, (str, list)):
+            return self.make_text_prompt(content, target_language)
+
+    def make_request(self, prompt):
+        # Subclasses must implement this method to send the request
+        raise NotImplementedError("Subclasses must implement the make_request method")
+
+
+class OpenAIModel(Model):
+    def __init__(self, model_name: str, api_key: str):
+        self.model_name = model_name
+        openai.api_key = api_key
+
+    def make_translation_request(self, prompt):
+        attempts = 0
+        while attempts < 3:
+            try:
+                if self.model_name == "gpt-3.5-turbo":
+                    response = openai.ChatCompletion.create(
+                        model=self.model_name,
+                        messages=[
+                            {"role": "user", "content": prompt}
+                        ]
+                    )
+                    translation = response.choices[0].message['content'].strip()
+                else:
+                    response = openai.ChatCompletion.create(
+                        model=self.model_name,
+                        prompt=prompt,
+                        max_tokens=150,
+                        temperature=0
+                    )
+                    translation = response.choices[0].text.strip()
+
+                return translation, True
+            except openai.error.RateLimitError:
+                attempts += 1
+                if attempts < 3:
+                    wait_time = math.pow(2, attempts)  # exponential backoff
+                    print("Rate limit reached. Waiting for 60 seconds before retrying.")
+                    time.sleep(60)
+                else:
+                    raise Exception("Rate limit reached. Maximum attempts exceeded.")
+        return "", False

LanguageTranslator/test.json

@@ -0,0 +1,4 @@
+{
+    "text": "I am iron man, I love you",
+    "dest_language": "vi"
+}

LanguageTranslator/utils/argument.py

@@ -0,0 +1,20 @@
+import argparse
+
+
+class ArgumentParser:
+    def __init__(self):
+        self.parser = argparse.ArgumentParser(description='Translate English to Vietnamese.')
+        self.parser.add_argument('--model_type', type=str, required=False, default='OpenAIModel',
+                                 help='The type of translation model to use. Choose between "GLMModel" and "OpenAIModel".')
+        self.parser.add_argument('--text', nargs='+', type=str, help='Input text(s) for translation.')
+        self.parser.add_argument('--dest_language', type=str, help='Target language for translation.')
+        self.parser.add_argument('--timeout', type=int, help='Timeout for the API request in seconds.')
+        self.parser.add_argument('--openai_model', type=str, required=False,default='gpt-3.5-turbo',
+                                 help='The model name of OpenAI Model. Required if model_type is "OpenAIModel".')
+        self.parser.add_argument('--openai_api_key', type=str, required=False,default='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',
+                                 help='The API key for OpenAIModel. Required if model_type is "OpenAIModel".')
+        self.parser.add_argument('--json', type=str, help='Path to a JSON file for input')
+
+    def parse_arguments(self):
+        args = self.parser.parse_args()
+        return args

README.md

@@ -1,12 +1,36 @@
----
-title: Language Translation
-emoji: 🌖
-colorFrom: pink
-colorTo: indigo
-sdk: gradio
-sdk_version: 4.2.0
-app_file: app.py
-pinned: false
----
+# Language Translator 
+## Introduction 
+Open AI Translator is the Power tool to translate English to Vietnamese Language. Using the Large Language Model (LLM) like GPT-3.5-turbo, ... for translation
 
-Check out the configuration reference at https://huggingface.co/docs/hub/spaces-config-reference
+The project is built in Python Which include API by FastAPI and User Interface by Gradio 
+
+## Getting Started
+### Environment Setup
+1. Clone the repository: 
+   ```commandline
+   git clone [email protected]:TinVo0908/Language-Translator.git'
+   ```
+2. The project is require: Python 3.9+ 
+3. Install dependencies by:
+    ```
+   pip install -r requirements.txt
+   ```
+### How to Use 
+1. For running with command line in terminal: 
+   For input is text and dest_language 
+    ```commandline
+    python test_local_translator.py --text 'your input text' --dest_language 'vi'
+    ```
+   For input is json file 
+    ```commandline
+    python test_local_translator.py --json 'your json file' 
+    ```
+   Noted: You can set your api key with argument --openai_api_key and set the model name with --openai_model or you can use api key and model with my default.
+2. For running API: 
+   ```commandline
+   uvicorn api:app --reload
+   ```
+3. For running User Interface
+   ```commandline
+   python app.py 
+   ```

api.py

@@ -0,0 +1,62 @@
+from fastapi import FastAPI, UploadFile, Query, HTTPException
+from starlette.responses import RedirectResponse
+from typing import Union, List
+from pydantic import BaseModel
+from translator import ServerTranslator
+import json
+import uvicorn
+
+app = FastAPI()
+
+
+# Define a data model for the input
+class TranslationInput(BaseModel):
+    text: str
+    dest_language: str
+
+
+class TranslationResult(BaseModel):
+    text: Union[str, List[str]]
+    language_translation: str
+
+
+@app.get("/", include_in_schema=False)
+async def index():
+    return RedirectResponse(url="/docs")
+
+
+@app.post("/translate", response_model=TranslationResult)
+async def run_translation_manual(
+        text: str = Query(..., description="Input text to translate"),
+        dest_language: str = Query(..., description="Destination language")):
+    # Splitting the input text
+    text = text.split(',')
+    # Creating and processing the translator
+    processing_language = ServerTranslator.language_translator(
+        text=text,
+        dest_language=dest_language,
+    )
+    # Getting the translated result
+    result_response = processing_language.translate()
+    return result_response
+
+
+@app.post("/translate_json", response_model=TranslationResult)
+async def run_translation_auto(json_file: UploadFile):
+    try:
+        # Reading the JSON content from the file
+        json_content = await json_file.read()
+        json_data = json.loads(json_content.decode("utf-8"))
+        # Creating and processing the translator
+        processing_language = ServerTranslator.language_translator(
+            json_data
+        )
+        # Getting the translated result
+        result_response = processing_language.translate()
+        return result_response
+    except json.JSONDecodeError:
+        raise HTTPException(status_code=400, detail="Invalid JSON input")
+
+
+if __name__ == "__main__":
+    uvicorn.run(app, host="0.0.0.0", port=8000)

app.py

@@ -0,0 +1,62 @@
+import gradio as gr
+from translator import ServerTranslator
+import json
+
+
+def clear_all(input_json, input_text, input_dest_lang, translated_text):
+    return None, "", "", ""
+
+
+def translate_text_json(input_json):
+    # Translation code for JSON input
+    try:
+        json_file_path = input_json.name
+        with open(json_file_path, 'r') as f:
+            file_content = f.read()
+            json_input = json.loads(file_content)
+            translation = ServerTranslator.language_translator(inputs_data=json_input).translate()
+            translate_text = translation['text']
+        return translate_text
+    except Exception as e:
+        translate_text = f"Error: {str(e)}"
+
+
+def translate_text_text(input_text, input_dest_lang):
+    # Translation code for text input
+    try:
+        translation = ServerTranslator.language_translator(text=input_text, dest_language=input_dest_lang).translate()
+        translate_text = translation['text']
+        return translate_text
+    except Exception as e:
+        translate_text = f"Error: {str(e)}"
+
+
+with gr.Blocks() as demo:
+    input_json = gr.components.File(label="Upload JSON file")
+    input_dest_lang = gr.components.Textbox(placeholder='Example input: vi',label="Destination Language")
+    input_text = gr.components.Textbox(placeholder='Example inputs: I love you, I love you than myself', label="Enter Text")
+    translated_text = gr.components.Textbox(placeholder='Example outputs: Anh yêu em, Anh yêu em hơn bản thân mình',
+                                            label="Translated Text")
+
+    with gr.Column():
+        submit_json = gr.Button("Submit Json")
+        submit_text = gr.Button("Submit Text")
+        submit_json.click(
+            translate_text_json,
+            [input_json],  # Pass all three inputs
+            [translated_text]
+        )
+        with gr.Column():
+            submit_text.click(
+                translate_text_text,
+                [input_text, input_dest_lang],  # Pass all three inputs
+                [translated_text]
+            )
+        clear = gr.Button("Clear")
+        clear.click(
+            clear_all,
+            [input_json, input_text, input_dest_lang, translated_text],
+            [input_json, input_text, input_dest_lang, translated_text]
+        )
+
+demo.launch(share=True)

requirements.txt

@@ -0,0 +1,143 @@
+aiofiles==23.2.1
+aiohttp==3.8.6
+aiosignal==1.3.1
+altair==5.1.2
+annotated-types==0.6.0
+anyio==3.7.1
+async-timeout==4.0.3
+attrs==23.1.0
+backoff==2.2.1
+bcrypt==4.0.1
+beautifulsoup4==4.12.2
+cachetools==5.3.2
+certifi==2023.7.22
+chardet==5.2.0
+charset-normalizer==3.3.2
+chroma-hnswlib==0.7.3
+chromadb==0.4.16
+click==8.1.7
+colorama==0.4.6
+coloredlogs==15.0.1
+contourpy==1.2.0
+cycler==0.12.1
+dataclasses-json==0.6.2
+Deprecated==1.2.14
+emoji==2.8.0
+exceptiongroup==1.1.3
+faiss-cpu==1.7.4
+fastapi==0.104.1
+ffmpy==0.3.1
+filelock==3.13.1
+filetype==1.2.0
+flatbuffers==23.5.26
+fonttools==4.44.0
+fpdf==1.7.2
+frozenlist==1.4.0
+fsspec==2023.10.0
+google-auth==2.23.4
+googleapis-common-protos==1.61.0
+gradio==3.45.2
+gradio_client==0.5.3
+grpcio==1.59.2
+h11==0.14.0
+httpcore==1.0.2
+httptools==0.6.1
+httpx==0.25.1
+huggingface-hub==0.17.3
+humanfriendly==10.0
+idna==3.4
+importlib-metadata==6.8.0
+importlib-resources==6.1.1
+install==1.3.5
+Jinja2==3.1.2
+joblib==1.3.2
+jsonpatch==1.33
+jsonpointer==2.4
+jsonschema==4.19.2
+jsonschema-specifications==2023.7.1
+kiwisolver==1.4.5
+kubernetes==28.1.0
+langchain==0.0.334
+langdetect==1.0.9
+langsmith==0.0.63
+lxml==4.9.3
+markdown-it-py==3.0.0
+MarkupSafe==2.1.3
+marshmallow==3.20.1
+matplotlib==3.8.1
+mdurl==0.1.2
+monotonic==1.6
+mpmath==1.3.0
+multidict==6.0.4
+mypy-extensions==1.0.0
+nltk==3.8.1
+numpy==1.26.1
+oauthlib==3.2.2
+onnxruntime==1.16.2
+openai==0.27.3
+opentelemetry-api==1.21.0
+opentelemetry-exporter-otlp-proto-common==1.21.0
+opentelemetry-exporter-otlp-proto-grpc==1.21.0
+opentelemetry-proto==1.21.0
+opentelemetry-sdk==1.21.0
+opentelemetry-semantic-conventions==0.42b0
+orjson==3.9.10
+overrides==7.4.0
+packaging==23.2
+pandas==2.1.2
+Pillow==10.1.0
+posthog==3.0.2
+protobuf==4.25.0
+pulsar-client==3.3.0
+pyasn1==0.5.0
+pyasn1-modules==0.3.0
+pydantic==2.4.2
+pydantic_core==2.10.1
+pydub==0.25.1
+Pygments==2.16.1
+pyparsing==3.1.1
+PyPika==0.48.9
+python-dateutil==2.8.2
+python-dotenv==1.0.0
+python-iso639==2023.6.15
+python-magic==0.4.27
+python-multipart==0.0.6
+pytz==2023.3.post1
+PyYAML==6.0.1
+rapidfuzz==3.5.2
+referencing==0.30.2
+regex==2023.10.3
+requests==2.31.0
+requests-oauthlib==1.3.1
+rich==13.6.0
+rpds-py==0.12.0
+rsa==4.9
+semantic-version==2.10.0
+shellingham==1.5.4
+six==1.16.0
+sniffio==1.3.0
+soupsieve==2.5
+SQLAlchemy==2.0.23
+starlette==0.27.0
+sympy==1.12
+tabulate==0.9.0
+tenacity==8.2.3
+tiktoken==0.5.1
+tokenizers==0.14.1
+tomlkit==0.12.0
+toolz==0.12.0
+tqdm==4.66.1
+typer==0.9.0
+typing-inspect==0.9.0
+typing_extensions==4.8.0
+tzdata==2023.3
+unstructured==0.10.29
+urllib3==1.26.18
+uvicorn==0.24.0.post1
+uvloop==0.19.0
+watchfiles==0.21.0
+websocket-client==1.6.4
+websockets==11.0.3
+wrapt==1.16.0
+yarl==1.9.2
+zipp==3.17.0

test_local_translator.py

@@ -0,0 +1,3 @@
+from translator import LocalTranslator
+process = LocalTranslator.language_translator()
+process.translate()

translator.py

@@ -0,0 +1,159 @@
+import json
+import time
+from LanguageTranslator.utils.argument import ArgumentParser
+from LanguageTranslator.model.openai_model import OpenAIModel
+from fastapi import HTTPException
+
+
+class LocalTranslator:
+    def __init__(self, open_ai, inputs_data):
+        # Initialize the LocalTranslator with OpenAI and inputs data
+        self.open_ai = open_ai
+        self.inputs_data = inputs_data
+
+    def translate(self):
+        # Translates the provided text into the destination language
+        translations = []
+        result_response = {
+            'text': '',
+            'language_translation': ''
+        }
+
+        if 'text' not in self.inputs_data or 'dest_language' not in self.inputs_data:
+            print("Required fields 'text' and 'dest_language' are missing in input data.")
+            return translations, ''
+
+        text = self.inputs_data['text']
+        dest_language = self.inputs_data['dest_language']
+
+        if not isinstance(dest_language, str):
+            print("Destination language is not a single text.")
+            return translations, dest_language
+
+        if isinstance(text, str):
+            text = [text]
+
+        if not isinstance(text, list):
+            print("Input data is not a text or a list of text")
+            return translations, dest_language
+
+        start_time = time.time()
+        for t in text:
+            prompt = self.open_ai.translate_prompt(t, dest_language)
+            translation, _ = self.open_ai.make_translation_request(prompt)
+            translations.append(translation)
+        end_time = time.time()
+
+        execution_time = end_time - start_time
+        result_response['text'] = translations if len(translations) > 1 else translations[0]
+        result_response['language_translation'] = dest_language
+        print(f"Translation completed in {execution_time:.2f} seconds 🎉🎉🎉\nTranslated: {result_response}")
+        return result_response
+
+    @classmethod
+    def load_parser(cls):
+        # Loads the parser object for argument parsing
+        argument_parser = ArgumentParser()
+        args = argument_parser.parse_arguments()
+        return args
+
+    @classmethod
+    def language_translator(cls):
+        # Builds the language translator using the provided arguments
+        args = cls.load_parser()
+        if args.model_type == "OpenAIModel":
+            if args.openai_model and args.openai_api_key:
+                model_name = args.openai_model
+                api_key = args.openai_api_key
+            else:
+                raise ValueError("Invalid OpenAI model or API key")
+            model = OpenAIModel(model_name=model_name, api_key=api_key)
+        else:
+            raise ValueError("Invalid OpenAIModel specified.")
+
+        if args.json:
+            with open(args.json, 'r') as j:
+                input_data = json.load(j)
+        else:
+            input_data = {
+                "text": args.text,
+                "dest_language": args.dest_language
+            }
+
+        return cls(open_ai=model, inputs_data=input_data)
+
+
+class ServerTranslator:
+    def __init__(self, open_ai, inputs_data):
+        # Initialize the ServerTranslator with OpenAI and inputs data
+        self.open_ai = open_ai
+        self.inputs_data = inputs_data
+
+    def translate(self):
+        # Translates the given text into the destination language
+        translations = []
+        result_response = {
+            'text': '',
+            'language_translation': ''
+        }
+
+        if 'text' not in self.inputs_data or 'dest_language' not in self.inputs_data:
+            raise HTTPException(status_code=400,
+                                detail="Required fields 'text' and 'dest_language' are missing in input data.")
+
+        text = self.inputs_data['text']
+        dest_language = self.inputs_data['dest_language']
+
+        if not isinstance(dest_language, str):
+            raise HTTPException(status_code=400, detail="Destination language is not a single text.")
+
+        if isinstance(text, str):
+            text = text.split(',')
+            text = [text]
+
+        if not isinstance(text, list):
+            raise HTTPException(status_code=400, detail="Input data is not a text or a list of text.")
+
+        start_time = time.time()
+        for t in text:
+            prompt = self.open_ai.translate_prompt(t, dest_language)
+            translation, _ = self.open_ai.make_translation_request(prompt)
+            translations.append(translation)
+        end_time = time.time()
+        execution_time = end_time - start_time
+        result_response['text'] = translations if len(translations) > 1 else translations[0]
+        result_response['language_translation'] = dest_language
+        print(f"Translation completed in {execution_time:.2f} seconds 🎉🎉🎉\nTranslated: {result_response}")
+        return result_response
+
+    @classmethod
+    def language_translator(cls, inputs_data=None, text=None, dest_language=None,
+                            model_type='OpenAIModel', openai_model='gpt-3.5-turbo',
+                            openai_api_key='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'):
+        # Builds the language translator using the provided arguments
+        if model_type == "OpenAIModel":
+            if openai_model and openai_api_key:
+                model_name = openai_model
+                api_key = openai_api_key
+            else:
+                raise HTTPException(status_code=400, detail="Invalid OpenAI model or API key")
+            model = OpenAIModel(model_name=model_name, api_key=api_key)
+        else:
+            raise HTTPException(status_code=400, detail="Invalid OpenAIModel specified.")
+
+        if inputs_data:
+            input_data = {
+                "text": inputs_data['text'].split(','),
+                "dest_language": inputs_data['dest_language']
+            }
+        else:
+            input_data = {
+                "text": text,
+                "dest_language": dest_language
+            }
+        return cls(open_ai=model, inputs_data=input_data)
+
+
+# if __name__ == "__main__":
+#     process = LocalTranslator.language_translator()
+#     process.translate()