Switch to blocks, attempt

.stignore

@@ -0,0 +1,101 @@
+.git
+# Byte-compiled / optimized / DLL files
+__pycache__
+*.py[cod]
+*$py.class
+
+# C extensions
+*.so
+
+# Distribution / packaging
+.Python
+build
+develop-eggs
+dist
+downloads
+eggs
+.eggs
+lib
+lib64
+parts
+sdist
+var
+wheels
+pip-wheel-metadata
+share/python-wheels
+*.egg-info
+.installed.cfg
+*.egg
+MANIFEST
+
+# PyInstaller
+#  Usually these files are written by a python script from a template
+#  before PyInstaller builds the exe, so as to inject date/other infos into it.
+*.manifest
+*.spec
+
+# Installer logs
+pip-log.txt
+pip-delete-this-directory.txt
+
+# Translations
+*.mo
+*.pot
+
+# Django stuff:
+*.log
+local_settings.py
+db.sqlite3
+
+# Flask stuff:
+instance
+.webassets-cache
+
+# Scrapy stuff:
+.scrapy
+
+# Sphinx documentation
+docs/_build
+
+# PyBuilder
+target
+
+# Jupyter Notebook
+.ipynb_checkpoints
+
+# IPython
+profile_default
+ipython_config.py
+
+# pyenv
+.python-version
+
+# celery beat schedule file
+celerybeat-schedule
+
+# SageMath parsed files
+*.sage.py
+
+# Environments
+.env
+.venv
+env
+venv
+ENV
+env.bak
+venv.bak
+
+# Spyder project settings
+.spyderproject
+.spyproject
+
+# Rope project settings
+.ropeproject
+
+# mypy
+.mypy_cache
+.dmypy.json
+dmypy.json
+
+# Pyre type checker
+.pyre

install-sw.sh

@@ -0,0 +1,23 @@
+# pip install pipx
+# pipx install poetry
+# pipx ensurepath
+# source ~/.bashrc
+
+# curl -sSL https://install.python-poetry.org | python3 -
+# -C- continue -S show error -o output
+curl -sSL -C- -o install-poetry.py  https://install.python-poetry.org
+python install-poetry.py
+rm install-poetry.py
+echo export PATH=~/.local/bin:$PATH > ~/.bashrc
+source ~/.bashrc
+# ~/.local/bin/poetry install
+
+wget -c https://deb.nodesource.com/setup_12.x
+bash setup_12.x
+apt-get install -y nodejs
+npm install -g npm@latest
+npm install -g nodemon
+rm setup_12.x
+
+# apt upate  # alerady done in apt-get install -y nodejs
+apt install byobu -y > /dev/null 2>&1

install-sw1.sh

@@ -0,0 +1,25 @@
+# pip install pipx
+# pipx install poetry
+# pipx ensurepath
+# source ~/.bashrc
+
+# curl -sSL https://install.python-poetry.org | python3 -
+# -C- continue -S show error -o output
+curl -sSL -C- -o install-poetry.py  https://install.python-poetry.org
+python install-poetry.py
+rm install-poetry.py
+echo export PATH=~/.local/bin:$PATH > ~/.bashrc
+source ~/.bashrc
+# ~/.local/bin/poetry install
+
+wget -c https://deb.nodesource.com/setup_12.x
+bash setup_12.x
+apt-get install -y nodejs
+npm install -g npm@latest
+npm install -g nodemon
+rm setup_12.x
+
+# apt update  # alerady done in apt-get install -y nodejs
+apt install byobu -y > /dev/null 2>&1
+byobu-enable
+byobu

okteto.yml

@@ -0,0 +1,44 @@
+name: gradio-cmat
+
+# The build section defines how to build the images of
+# your development environment
+# More info: https://www.okteto.com/docs/reference/manifest/#build
+# build:
+#   my-service:
+#     context: .
+
+# The deploy section defines how to deploy your development environment
+# More info: https://www.okteto.com/docs/reference/manifest/#deploy
+# deploy:
+  # commands:
+    # - name: Deploy
+      # command: echo 'Replace this line with the proper 'helm'
+
+#  or 'kubectl' commands to deploy your development environment'
+
+# The dependencies section defines other git repositories to be
+# deployed as part of your development environment
+# More info: https://www.okteto.com/docs/reference/manifest/#dependencies
+# dependencies:
+#   - https://github.com/okteto/sample
+# The dev section defines how to activate a development container
+# More info: https://www.okteto.com/docs/reference/manifest/#dev
+dev:
+  gradio-cmat:
+    # image: okteto/dev:latest
+    # image: python:3.8.13-bullseye
+    # image: simbachain/poetry-3.8
+    image: python:3.8
+    command: bash
+    workdir: /usr/src/app
+    sync:
+      - .:/usr/src/app
+    environment:
+      - name=$USER
+    forward:
+      - 7861:7861
+      - 7860:7860
+      - 8501:8501
+    reverse:
+      - 9000:9000
+    autocreate: true

poetry.toml

@@ -0,0 +1,3 @@
+[virtualenvs]
+create = true
+in-project = true

requirements.txt

@@ -3,10 +3,8 @@ install
 transformers
 sentencepiece
 sklearn
-git+https://github.com/ffreemt/fast-langid
 git+https://github.com/ffreemt/align-model-pool
 sentence-transformers
-sentence_splitter
 logzero
 icecream
 alive-progress
@@ -14,10 +12,11 @@ more_itertools
 #
 openpyxl
 # --- seg_text
-Morfessor
-pyicu
-pycld2
-tqdm
-polyglot
-sentence_splitter
-pyfunctional
+# Morfessor
+# pyicu
+# pycld2
+# tqdm
+# polyglot
+# sentence_splitter
+# pyfunctional
+# git+https://github.com/ffreemt/fast-langid

ubee/__main__.py

@@ -1,20 +1,18 @@
 """Gen ubee main."""
-# pylint: disable=unused-import, wrong-import-position, wrong-import-order, too-many-locals, broad-except
+# pylint: disable=unused-import, wrong-import-position, wrong-import-order, too-many-locals, broad-except, line-too-long
 
-from typing import Tuple, Optional
-
-from pathlib import Path
 import sys
-from random import shuffle
-
 from itertools import zip_longest
+from pathlib import Path
+from random import shuffle
 from textwrap import dedent
+from typing import Optional, Tuple
 
 import gradio as gr
-
-import pandas as pd
-from icecream import install as ic_install, ic
 import logzero
+import pandas as pd
+from icecream import ic
+from icecream import install as ic_install
 from logzero import logger
 
 # for embeddable python

ubee/seg_text.py

@@ -9,15 +9,14 @@ else use polyglot.text.Text
 """
 # pylint: disable=
 
+import re
 from typing import List, Optional, Union
 
-import re
-from tqdm.auto import tqdm
+from logzero import logger
 from polyglot.detect.base import logger as polyglot_logger
 from polyglot.text import Detector, Text
 from sentence_splitter import split_text_into_sentences
-
-from logzero import logger
+from tqdm.auto import tqdm
 
 # turn of polyglot.text.Detector warning
 polyglot_logger.setLevel("ERROR")

ubee/ubee.py

@@ -1,11 +1,12 @@
 """Align via ubee,"""
 # pylint: disable=
-from typing import Iterable, List, Tuple
 from itertools import zip_longest
+from typing import Iterable, List, Tuple
 
+from icecream import ic
 from logzero import logger
+
 from ubee.uclas import uclas
-from icecream import ic
 
 
 def ubee(
@@ -17,7 +18,7 @@ def ubee(
 
     Args:
         sents_zh: list of text, can be any langauge supported by clas-l-user
-        sents_zh: ditto
+        sents_en: ditto
     Returns:
         three tuples of aligned blocked
         leftovers (unaligned)

ubee/uclas.py

@@ -2,16 +2,16 @@
 # pylint: disable=invalid-name
 
 from typing import List, Tuple, Union
-import numpy as np
-from sklearn.metrics.pairwise import cosine_similarity
-from joblib import Memory
-
-from model_pool import fetch_check_aux  # pylint: disable=import-error
-from model_pool.model_s import load_model_s  # pylint: disable=import-error
-from model_pool.load_model import load_model  # pylint: disable=import-error
 
 import logzero
+import numpy as np
+from joblib import Memory
 from logzero import logger
+# set PYTHONPATH=..\align-model-pool  # in win10
+from model_pool.fetch_check_aux import fetch_check_aux
+from model_pool.load_model import load_model
+from model_pool.model_s import load_model_s
+from sklearn.metrics.pairwise import cosine_similarity
 
 logzero.loglevel(20)