initial commit

.gitattributes

@@ -33,3 +33,19 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
 *.zip filter=lfs diff=lfs merge=lfs -text
 *.zst filter=lfs diff=lfs merge=lfs -text
 *tfevents* filter=lfs diff=lfs merge=lfs -text
+3.jpg filter=lfs diff=lfs merge=lfs -text
+GrayImages/1.jpg filter=lfs diff=lfs merge=lfs -text
+GrayImages/10.jpg filter=lfs diff=lfs merge=lfs -text
+GrayImages/12.jpg filter=lfs diff=lfs merge=lfs -text
+GrayImages/7.jpg filter=lfs diff=lfs merge=lfs -text
+GrayImages/8.jpg filter=lfs diff=lfs merge=lfs -text
+GrayImages/9.jpg filter=lfs diff=lfs merge=lfs -text
+GrayImages/parcel_img13.png filter=lfs diff=lfs merge=lfs -text
+GrayImages/parcel_img3.png filter=lfs diff=lfs merge=lfs -text
+grey_images/1.jpg filter=lfs diff=lfs merge=lfs -text
+grey_images/10.jpg filter=lfs diff=lfs merge=lfs -text
+grey_images/12.jpg filter=lfs diff=lfs merge=lfs -text
+grey_images/7.jpg filter=lfs diff=lfs merge=lfs -text
+grey_images/8.jpg filter=lfs diff=lfs merge=lfs -text
+grey_images/9.jpg filter=lfs diff=lfs merge=lfs -text
+grey_images/parcel_img3.png filter=lfs diff=lfs merge=lfs -text

App.py

@@ -0,0 +1,67 @@
+import streamlit as st
+import cv2
+from pipeline import main
+from pathlib import Path
+import pandas as pd
+import os
+from dotenv import load_dotenv
+from pathlib import Path
+env_path = Path('.') / '.env'
+load_dotenv(dotenv_path=env_path)
+
+path = {
+        'SEG_MODEL_PATH': str(os.getenv('SEG_MODEL_PATH')),
+        'MAIN_FLOW_GRAY_IMG_DIR_PATH': str(os.getenv('MAIN_FLOW_GRAY_IMG_DIR_PATH')),
+        'MAIN_FLOW_INFERENCE_FOLDER': str(os.getenv('MAIN_FLOW_INFERENCE_FOLDER')),
+        }
+
+with st.sidebar:
+    st.title("Shipping Label Extraction")
+    data = st.file_uploader(label='Upload Image of Parcel',type=['png','jpg','jpeg'])
+
+
+if data:
+    Path('grey_images').mkdir(parents=True, exist_ok=True)
+
+    with open(os.path.join('grey_images',data.name),'wb') as f:
+        f.write(data.getvalue())
+        
+    img = cv2.imread(os.path.join('grey_images',data.name),0)
+    
+    if img.shape[0] > 1500:
+        height, width = img.shape 
+        img = img[height//4:-height//4, width//4:-width//4]
+    
+    cv2.imwrite(os.path.join('grey_images',data.name), img)
+
+    #call main function
+    Output_dict= main(os.path.join('grey_images',data.name))
+    df = pd.DataFrame(Output_dict)
+
+    col1,col2 = st.columns(2)
+
+    with col1:
+        st.markdown("<h3 style='text-align: center;'>Grey Image</h1>", unsafe_allow_html=True) 
+        st.image(os.path.join('grey_images',data.name))
+
+
+        st.markdown("<h3 style='text-align: center;'>Enhanced Image</h1>", unsafe_allow_html=True) 
+        st.image(os.path.join('runs', 'segment', path['MAIN_FLOW_INFERENCE_FOLDER'], 'enhanced', data.name))
+
+    with col2:
+        st.markdown("<h3 style='text-align: center;'>Detected Image</h1>", unsafe_allow_html=True) 
+        st.image(os.path.join('runs', 'segment',path['MAIN_FLOW_INFERENCE_FOLDER'],data.name))
+
+
+        st.markdown("<h3 style='text-align: center;'>Rotated Image</h1>", unsafe_allow_html=True) 
+        st.image(os.path.join('runs', 'segment', path['MAIN_FLOW_INFERENCE_FOLDER'], 'rotated_image', data.name))
+
+
+    ocr_data = ""
+    with open(os.path.join('runs', 'segment', path['MAIN_FLOW_INFERENCE_FOLDER'], 'ocr_label_data', data.name.split('.')[0]+'.txt'),'r+') as f :
+        ocr_data = f.read()
+    st.header("OCR Text Output")
+    st.text(ocr_data)
+
+    st.header("NER Output")
+    st.table(df)

NER/final_model_2.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:432a72a5a4c668835a579e9f9fcad6137e0fd6ec72056eaf98a6373abac2852f
+size 414142321

NER/ner_inference.py

@@ -0,0 +1,96 @@
+
+from flair.data import Sentence
+from flair.models import SequenceTagger
+# import pathlib
+# temp = pathlib.PosixPath
+# pathlib.PosixPath = pathlib.WindowsPath
+import json
+import os
+from dotenv import load_dotenv
+from pathlib import Path
+env_path = Path('.') / '.env'
+load_dotenv(dotenv_path=env_path)
+import re    
+import string
+import nltk
+from nltk.corpus import stopwords
+
+nltk.download('stopwords')
+nltk.download('punkt')
+
+path = {
+        'NER_MODEL_PATH': str(os.getenv('NER_MODEL_PATH')),
+       }
+model = SequenceTagger.load(path['NER_MODEL_PATH'])
+
+
+def inference(sent):
+
+    sent = clean_sent(sent)
+    sentence = Sentence(sent)
+
+    res = model.predict(sentence)
+    res = sentence.to_dict()
+
+    label_name = {'GCNUM':[],'TRACK-ID':[],'Company':"" }
+    for label in res['entities']:
+        sorted_labels = sorted(label['labels'], key=lambda k: k['confidence']) 
+        if sorted_labels[0]['value'] not in label_name:
+            label_name[sorted_labels[0]['value']].append(label['text'])
+        else:
+            label_name[sorted_labels[0]['value']].append(label['text'])
+
+    # RE part 
+    if label_name['GCNUM'] is None or len(label_name['GCNUM']) == 0:
+        pattern = r'[G,O,Q][C,O,Q]\d{5}'
+        result = re.findall(pattern, sent, re.IGNORECASE)
+        if result is not None and len(result) > 0:
+            label_name['GCNUM'].append(result[0])
+    
+    label_name['GCNUM'] = label_name['GCNUM']
+    label_name['TRACK-ID'] = label_name['TRACK-ID']
+    #getting companie name
+    #1. get name directly from sentence
+    label_name['Company'] = get_company_name(sent.lower())
+
+    #2. track id from sentence
+
+    return label_name
+
+def get_company_name(sent):
+    patterns = []
+    patterns.append({'company': "FeDex", 'pattern' : re.compile(r"\b{}\b".format("fedex")),"track_pattern":re.compile(r"\b{}\b".format("[0-9]{12}"))})
+    patterns.append({'company':'UPS', 'pattern':re.compile(r"\b{}\b".format("ups")),"track_pattern":re.compile(r"\b{}\b".format("1Z"))})
+    patterns.append({'company':'USPS', 'pattern':re.compile(r"\b{}\b".format("usps")),"track_pattern":re.compile(r"\b{}\b".format("42033155"))})
+    patterns.append({'company':'onTrack', 'pattern':re.compile(r"\b{}\b".format("ontrack")),"track_pattern":re.compile(r"\b{}\b".format("BG[0-9]{5}"))})
+    patterns.append({'company':'Lasership', 'pattern':re.compile(r"\b{}\b".format('1LS')),"track_pattern":re.compile(r"\b{}\b".format('1LS'))})
+    patterns.append({'company':'Amazon', 'pattern':re.compile(r"\b{}\b".format("dmi6 | dm16")),"track_pattern":re.compile(r"\b{}\b".format("T[B,8]A"))})
+
+    for pattern in patterns:
+        if pattern['pattern'].search(sent) is not None:
+            return pattern['company']
+    
+    for pattern in patterns:
+        if pattern['track_pattern'].search(sent) is not None:
+            return pattern['company']
+    return None
+
+
+def clean_sent(sent):
+
+    # Remove punctuation
+    sent = sent.translate(str.maketrans('', '', string.punctuation))
+
+    # Tokenize the sentence
+    tokens = nltk.word_tokenize(sent)
+
+    # Remove stop words
+    stop_words = set(stopwords.words('english'))
+    tokens = [token for token in tokens if token.lower() not in stop_words]
+
+    # Join the tokens back into a sentence
+    sent = ' '.join(tokens)
+    return sent
+
+
+# print(inference("CVG2 5.1Lbs 02123 DMIGE Rasheba PierreGC12100 4654SW75TH AVE 33155-4433MIAMIFLUnited TBA305477063112 DMI6 CYCLE 1 MIA1 MIA2 &MI6 DMIG"))

OCR/label_ocr.py

@@ -0,0 +1,37 @@
+#import libraries
+import numpy as np
+import os
+import cv2
+from dotenv import load_dotenv
+from pathlib import Path
+env_path = Path('.') / '.env'
+
+from paddleocr import PaddleOCR
+ocr = PaddleOCR(use_angle_cls=True, lang='en') # need to run only once to download and load model into memory
+
+load_dotenv(dotenv_path=env_path)
+path = {
+        'ROTATED_IMAGE_FOLDER_PATH': str(os.getenv('ROTATED_IMAGE_FOLDER_PATH')),
+        }
+# Traves rotated images  
+for img_name in os.listdir(path["ROTATED_IMAGE_FOLDER_PATH"]):
+
+    # perform ocr 
+    file_name = img_name.split(".")[0]
+    result = ocr.ocr(os.path.join(path["ROTATED_IMAGE_FOLDER_PATH"],img_name), cls=True)
+    ocr_output_paddle = []
+    for i in result:
+        ocr_output_paddle.append(" ".join([line[1][0] for line in i]))
+
+    #store ocr in OCR_LABEL_DATA folder
+    if result is not None:
+        try: 
+            Path('runs').mkdir(parents=True, exist_ok=True)
+            Path(os.path.join('runs', 'segment')).mkdir(parents=True, exist_ok=True)
+            Path(os.path.join('runs', 'segment', 'inference')).mkdir(parents=True, exist_ok=True)
+            Path(os.path.join('runs', 'segment', 'inference', 'ocr_label_data')).mkdir(parents=True, exist_ok=True)
+        except OSError as error:  
+            print(error)
+
+    with open(os.path.join('runs', 'segment', 'inference', 'ocr_label_data', img_name) +'.txt',"w+") as f:
+        f.write("\n".join(ocr_output_paddle))

OCR/label_rotation.py

@@ -0,0 +1,47 @@
+
+import cv2
+import os 
+from rotation_functions import rotate,hoffman_transformation,pytesseractRotate
+from dotenv import load_dotenv
+from pathlib import Path
+env_path = Path('.') / '.env'
+load_dotenv(dotenv_path=env_path)
+
+path = {
+        'ENHANCED_IMAGE_FOLDER_PATH': str(os.getenv('ENHANCED_IMAGE_FOLDER_PATH')),
+        }
+
+import numpy as np
+
+for img_name in os.listdir(path['ENHANCED_IMAGE_FOLDER_PATH']):
+    img = cv2.imread(os.path.join(path['ENHANCED_IMAGE_FOLDER_PATH'],img_name))
+    original_img = img
+
+    # scale and dialate the image for better result
+    img  = cv2.resize(img,None,fx=2.7, fy=3)
+    kernel = np.ones((2,2),np.uint8)
+    img = cv2.dilate(img,kernel)
+    sharpen_kernel = np.array([[-1,-1,-1], [-1,9,-1], [-1,-1,-1]])
+    img = cv2.filter2D(img, -1, sharpen_kernel)
+
+    #apply hoffman transformation
+    rotated_image,angle = hoffman_transformation(img, True)
+
+    original_img  = rotate(original_img,angle)
+
+    # apply tesseract ocd
+    rotated_image = pytesseractRotate(rotated_image,original_img,1)
+
+
+      # save to file
+    if rotated_image is not None:
+        try: 
+            Path('runs').mkdir(parents=True, exist_ok=True)
+            Path(os.path.join('runs', 'segment')).mkdir(parents=True, exist_ok=True)
+            Path(os.path.join('runs', 'segment', 'inference')).mkdir(parents=True, exist_ok=True)
+            Path(os.path.join('runs', 'segment', 'inference', 'rotated_image')).mkdir(parents=True, exist_ok=True)
+        except OSError as error:  
+            print(error)
+            pass
+        
+        cv2.imwrite(os.path.join('runs', 'segment', 'inference', 'rotated_image', img_name), rotated_image)

OCR/rotation_functions.py

@@ -0,0 +1,194 @@
+# method 4 mix two methods 
+import cv2
+import numpy as np
+import os 
+import re
+import pytesseract 
+from dotenv import load_dotenv
+from pathlib import Path
+env_path = Path('.') / '.env'
+
+load_dotenv(dotenv_path=env_path)
+path = {
+        'TESSERACT_PATH': str(os.getenv('TESSERACT_PATH')),
+        }
+pytesseract.pytesseract.tesseract_cmd = (path['TESSERACT_PATH'])
+
+
+
+def hoffman_transformation(image, verbose=False):
+    """
+    this function performs hoffman transformation method which fixes the rotation of image in 4 angles 0,90,270,360.
+    Args:
+        image (ndarray): gets image and perform hoffman tarnsformation
+        verbose (bool, optional): for seeing image transformation using matplotlib plots. Defaults to False.
+
+    Returns:
+        rotated_image: returns rotated image which can be only 4 angles rotated label 
+    """
+# Define our parameters for Canny
+    low_threshold = 50
+    high_threshold = 100
+    kernel = np.ones((8,8),dtype=np.uint8)
+    eroded_image = cv2.erode(image,kernel=kernel)
+    eroded_image = cv2.dilate(eroded_image,kernel)
+
+    # perform canny edge detection 
+    edges = cv2.Canny(eroded_image, low_threshold, high_threshold)
+  
+    edges = cv2.erode(edges,(50,50))
+ 
+# Define the Hough transform parameters
+# Make a blank the same size as our image to draw on
+    rho = 1
+    theta = np.pi/180
+    threshold = 60
+    min_line_length = 10
+    max_line_gap = 5
+    line_image = np.copy(image) #creating an image copy to draw lines on
+
+    # Run Hough on the edge-detected image
+    lines = cv2.HoughLinesP(edges, rho, theta, threshold, np.array([]),
+                            min_line_length, max_line_gap)
+    
+    # Iterate over the output "lines" and draw lines on the image copy
+    angles_count = {}
+    final_angle = 0
+    if lines is not None:
+        for line in lines:
+          if line is not None:
+                for x1,y1,x2,y2 in line:
+                    cv2.line(line_image,(x1,y1),(x2,y2),(255,0,0),5)
+                    
+                    angle = 0
+                    if abs(x1-x2) < 0.000001:
+                        angle = np.pi/2
+                    else:
+                        angle = (y1-y2)/(x1-x2)
+                        angle = np.arctan(angle)
+                    angle = angle*180/np.pi
+                    angle = np.round(angle)
+                    if angle%10 < 5:
+                        angle = angle- angle%10
+                    else:
+                        angle = angle + 10 - angle%10
+                    if angle in angles_count:
+                        angles_count[angle] += 1
+                    else:
+                        angles_count[angle] = 1
+                    
+                    final_angle = max(angles_count, key=angles_count.get)
+                    
+
+    line_image = cv2.putText(line_image, str(final_angle), (20,30), cv2.FONT_HERSHEY_COMPLEX, 1, (0,255,0), 3, cv2.LINE_8, False)
+
+    angle= 360
+    angle-= final_angle
+    angle = -(90 + angle) if angle < -45 else -angle
+
+    # rotate image at final_angle using rotation matrix and warpAffine transformation
+    h, w = image.shape[:2]
+    (c_x, c_y) = (w // 2, h // 2)
+    matrix = cv2.getRotationMatrix2D((c_x, c_y), angle, 1.0)
+    
+    cos = np.abs(matrix[0, 0])
+    sin = np.abs(matrix[0, 1])
+    
+    n_w = int((h * sin) + (w * cos))
+    n_h = int((h * cos) + (w * sin))
+    
+    matrix[0, 2] += (n_w / 2) - c_x
+    matrix[1, 2] += (n_h / 2) - c_y
+    
+    rotated_image =  cv2.warpAffine(image, matrix, (n_w, n_h), borderValue=(255, 255, 255))
+    return rotated_image,angle
+
+
+
+def rotate(
+        image: np.ndarray, angle: float
+) -> np.ndarray:
+    """ this function rotates the image at given angle and returns the rotated image
+
+    Args:
+        image (np.ndarray): _description_
+        angle (float): _description_
+
+    Returns:
+        np.ndarray: _description_
+    """
+    h, w = image.shape[:2]
+    (c_x, c_y) = (w // 2, h // 2)
+    matrix = cv2.getRotationMatrix2D((c_x, c_y), angle, 1.0)
+    
+    cos = np.abs(matrix[0, 0])
+    sin = np.abs(matrix[0, 1])
+    
+    n_w = int((h * sin) + (w * cos))
+    n_h = int((h * cos) + (w * sin))
+    
+    matrix[0, 2] += (n_w / 2) - c_x
+    matrix[1, 2] += (n_h / 2) - c_y
+    
+    return cv2.warpAffine(image, matrix, (n_w, n_h), borderValue=(255, 255, 255))
+
+
+def pytesseractRotate(image,original_image, grid=3):
+    """ this function takes one image and apply pytesseract osd method and gives orientation and script details and returns 0 degree oriented parcel image.
+
+    Args:
+        image (ndarray): takes image and perform osd
+        original_image (ndarray): _description_
+        grid (int, optional): _description_. Defaults to 3.
+
+    Returns:
+       rotated_image (ndarray): 
+    """
+    h, w = image.shape[:2]
+    
+    images_list = []
+    angles_list = {}
+    for i in range(1, grid+1):
+        for j in range(1, grid+1):
+            tx, ty = (w//grid)*(j-1), (h//grid)*(i-1)
+            bx, by = (w//grid)*j, (h//grid)*i
+            
+            
+            img = image[ty:by, tx: bx]
+            images_list.append(img)
+        
+    for i in range(len(images_list)):
+        try:
+            result = pytesseract.image_to_osd(images_list[i], config="osd --psm 0  -c min_characters_to_try=200", output_type='dict')
+            pytesseract_angle = result['rotate']
+            orientation_conf = result['orientation_conf']
+            script = result['script']
+            script_conf = result['script_conf']
+            
+            script_list = ['Latin','Cyrillic']
+            
+            if script in script_list and script_conf > 0:
+                if pytesseract_angle in angles_list:
+                    angles_list[pytesseract_angle].append(orientation_conf)
+                else:
+                    angles_list[pytesseract_angle] = [orientation_conf]
+                
+        except Exception as error:
+            print(error)
+    
+    confidence_list = []
+    for key in angles_list.keys():
+        mean = sum(angles_list[key])/len(angles_list[key])
+        confidence_list.append((len(angles_list[key]), mean, key))
+    
+    confidence_list = sorted(confidence_list)
+
+    final_angle = 360
+    if len(confidence_list) > 0:        
+        final_angle -= confidence_list[-1][-1]
+    else:
+        final_angle -= 0
+        
+    rotated_image = rotate(original_image, final_angle)
+
+    return rotated_image

ObjectDetection/best_nano_det.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:bac3e136b414633b08317ed286f32ba889ca260823e1ddb820a2580df6eb84b1
+size 6245593

ObjectDetection/best_nano_seg.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:0f62babdbb36264497df9deeab00157570b160f32241e1cdffe9e8228b5357f8
+size 6792867

ObjectDetection/functions.py

@@ -0,0 +1,124 @@
+import torch
+import cv2
+import numpy as np
+
+def cropBlackBackground(img):
+    """Removes black background of labbel image and returns portion which contains label only.
+    It will need gray image.
+
+    Args:
+        img (ndarray): Numpy array representation of image
+
+    Returns:
+        img (ndarray): Numpy array representation of cropped image
+    """
+    try:
+        _, binary = cv2.threshold(img, 1, 255, cv2.THRESH_BINARY)
+        contours, _ = cv2.findContours(binary, cv2.RETR_EXTERNAL, cv2.CHAIN_APPROX_SIMPLE)
+        
+        max_area_indx = 0
+        max_area = 0
+        for indx, contour in enumerate(contours):
+            area = cv2.contourArea(contour)
+            if area > max_area:
+                max_area_indx = indx
+        
+        x, y, w, h = cv2.boundingRect(contours[max_area_indx])
+        img = img[y:y+h, x:x+w]
+        return img
+    except Exception as e:
+        print(e)
+        return None
+
+def enhanceImage(img, block_size: int = 19, constant: int = 5, adaptive_thresold_type = "GAUSSIAN", need_to_sharp: bool = True):
+    """Enhance image by appling adaptive thresolding and filter2D
+
+    Args:
+        img (ndarray): Numpy array representation of image
+        block_size (int, optional): Block size for adaptive thresolding. Defaults to 25.
+        constant (int, optional): Constant for adaptive thresolding. Defaults to 10.
+        adaptive_thresold_type (str, optional): "GAUSSIAN" or "MEAN. Defaults to "GAUSSIAN".
+        need_to_sharp (bool, optional): Defaults to True.
+
+    Returns:
+        img (ndarray): Numpy array representation of enhanced image
+    """
+    try:
+        if block_size < 2:
+            block_size = 2
+        block_size = block_size + 1 if block_size%2 == 0 else block_size
+        
+        final_img = img
+        if adaptive_thresold_type == "MEAN":
+            final_img = cv2.adaptiveThreshold(img,255,cv2.ADAPTIVE_THRESH_MEAN_C,cv2.THRESH_BINARY,block_size,constant)
+        else:
+            final_img = cv2.adaptiveThreshold(img,255,cv2.ADAPTIVE_THRESH_GAUSSIAN_C,cv2.THRESH_BINARY,block_size,constant)
+
+        if need_to_sharp:
+            kernel = np.array([[-1, -1, -1], [-1, 9, -1], [-1, -1, -1]])
+            final_img = cv2.filter2D(final_img, -1, kernel)
+            
+        return final_img
+    except Exception as e:
+        print(e)
+        return None
+
+def generateMask(res, original_img):
+    """_summary_
+
+    Args:
+        res: Resulf of yolo image segmentation for single mask
+        original_img (ndarray): Numpy array representation of original image
+
+    Returns:
+        tupple (ndarray, ndarray): (crop_img, mask)
+    """
+    
+    try:
+        height,width  = original_img.shape
+        masks = res.masks.data
+        boxes = res.boxes.data
+        
+        #get index of box which has maximum confidence
+        max_conf_index = res.boxes.conf.argmax()
+
+        # extract classes
+        clss = boxes[:, 5]
+        # get indices of ress where class is 0 
+        label_indices = torch.where(clss == 0)
+        # use these indices to extract the relevant masks
+        label_masks = masks[label_indices]
+
+        
+        # get maximum confidence label's mask
+        max_conf_label_mask = torch.empty(size=(1,label_masks[max_conf_index].shape[0],label_masks[max_conf_index].shape[1]))
+        max_conf_label_mask[0]= label_masks[max_conf_index]
+
+        # scale for visualizing ress
+        label_mask = torch.any(max_conf_label_mask, dim=0).int() * 255
+
+        #final mask 
+        final_mask = label_mask.cpu().numpy()
+        height_mask,width_mask =final_mask.shape
+        fy = height/height_mask
+        fx = width/width_mask
+        final_mask = cv2.resize(final_mask,(0,0),fx =fx ,fy = fy,interpolation=cv2.INTER_NEAREST)
+
+        original_img =  original_img.astype(np.uint8)
+        final_mask = final_mask.astype(np.uint8)
+        
+        # Expand boundries
+        kernel = np.ones((40,40), np.uint8)
+        expanded_mask =  cv2.dilate(final_mask, kernel)
+
+        #crop_img
+        crop_img = cv2.bitwise_and(original_img,original_img,mask=expanded_mask)
+
+
+        return crop_img, expanded_mask
+
+    except Exception as e:
+        print(e)
+        return None, None
+    
+

ObjectDetection/object_detection.py

@@ -0,0 +1,65 @@
+import cv2
+from ultralytics import YOLO 
+import os
+from dotenv import load_dotenv
+from pathlib import Path
+env_path = Path('.') / '.env'
+load_dotenv(dotenv_path=env_path)
+from functions import cropBlackBackground, enhanceImage, generateMask 
+
+path = {
+        'SEG_MODEL_PATH': str(os.getenv('SEG_MODEL_PATH')),
+        'DET_MODEL_PATH': str(os.getenv('DET_MODEL_PATH')),
+        'IMG_DIR_PATH': str(os.getenv('IMG_DIR_PATH')),
+        'INFERENCE_FOLDER': str(os.getenv('INFERENCE_FOLDER')),
+        }
+
+#import models
+seg_model = YOLO(path['SEG_MODEL_PATH'])
+det_model = YOLO(path['DET_MODEL_PATH'])
+
+CONF = 0.7  
+
+
+# do inference for detection and store croped images in folder
+for img in os.listdir(path['IMG_DIR_PATH']):
+    img_file = cv2.imread(os.path.join(path['IMG_DIR_PATH'],img),0)
+    cv2.imwrite(os.path.join(path['IMG_DIR_PATH'],img),img_file)
+    
+    det_model(os.path.join(path['IMG_DIR_PATH'],img),conf = CONF, save=True , save_crop=True , name=path['INFERENCE_FOLDER'],exist_ok = True)
+
+#do inference for image segmentation and store image in folder
+for img in os.listdir(path['IMG_DIR_PATH']):
+    img_file = cv2.imread(os.path.join(path['IMG_DIR_PATH'],img),0)
+    cv2.imwrite(os.path.join(path['IMG_DIR_PATH'],img),img_file)
+    
+    result = seg_model(os.path.join(path['IMG_DIR_PATH'],img),save = True,name = path['INFERENCE_FOLDER'],exist_ok = True)
+    
+    original_img = cv2.imread(os.path.join(path['IMG_DIR_PATH'],img))
+    
+    for res in result:
+        crop_img, mask = generateMask(res, original_img)
+        
+        image = None
+        if crop_img is not None:
+            # Convert to gray scale image
+            crop_img = cv2.cvtColor(crop_img, cv2.COLOR_BGR2GRAY)
+            crop_img = cropBlackBackground(crop_img)
+            image = enhanceImage(crop_img)
+        
+        # save to file
+        if image is not None:
+            try: 
+                Path('runs').mkdir(parents=True, exist_ok=True)
+                Path(os.path.join('runs', 'segment')).mkdir(parents=True, exist_ok=True)
+                Path(os.path.join('runs', 'segment', 'inference')).mkdir(parents=True, exist_ok=True)
+                Path(os.path.join('runs', 'segment', 'inference', 'crops_seg')).mkdir(parents=True, exist_ok=True)
+                Path(os.path.join('runs', 'segment', 'inference', 'enhanced')).mkdir(parents=True, exist_ok=True)
+                Path(os.path.join('runs', 'segment', 'inference', 'masks')).mkdir(parents=True, exist_ok=True)
+            except OSError as error:  
+                print(error)
+                pass
+            
+            cv2.imwrite(os.path.join('runs', 'segment', 'inference', 'masks', img), mask)
+            cv2.imwrite(os.path.join('runs', 'segment', 'inference', 'crops_seg', img), crop_img )
+            cv2.imwrite(os.path.join('runs', 'segment', 'inference', 'enhanced', img), image )

app.py

@@ -0,0 +1,155 @@
+import streamlit as st
+import cv2
+from pipeline import main
+from pathlib import Path
+import pandas as pd
+import os
+from dotenv import load_dotenv
+from pathlib import Path
+from pipeline_functions import object_detection, crop_image, enhance_image, morphological_transform, hoffman_transform, pytesseract_rotate, ocr,ner
+
+env_path = Path('.') / '.env'
+load_dotenv(dotenv_path=env_path)
+
+path = {
+        'SEG_MODEL_PATH': str(os.getenv('SEG_MODEL_PATH')),
+        'MAIN_FLOW_GRAY_IMG_DIR_PATH': str(os.getenv('MAIN_FLOW_GRAY_IMG_DIR_PATH')),
+        'MAIN_FLOW_INFERENCE_FOLDER': str(os.getenv('MAIN_FLOW_INFERENCE_FOLDER')),
+        }
+
+with st.sidebar:
+    st.title("Shipping Label Extraction")
+    data = st.file_uploader(label='Upload Image of Parcel',type=['png','jpg','jpeg'])
+
+
+if data:
+    Path('grey_images').mkdir(parents=True, exist_ok=True)
+
+    with open(os.path.join('grey_images',data.name),'wb') as f:
+        f.write(data.getvalue())
+        
+    img = cv2.imread(os.path.join('grey_images',data.name),0)
+    
+    if img.shape[0] > 1500:
+        height, width = img.shape 
+        img = img[height//4:-height//4, width//4:-width//4]
+    
+    cv2.imwrite(os.path.join('grey_images',data.name), img)
+
+    #call main function
+    # main(os.path.join('grey_images',data.name))
+    file_path = os.path.join('grey_images',data.name)
+    img_name = os.path.basename(file_path)
+
+
+    col1,col2 = st.columns(2)
+
+    with col1:
+        st.markdown("<h3 style='text-align: center;'>Grey Image</h1>", unsafe_allow_html=True) 
+        st.image(os.path.join('grey_images',data.name))
+
+        # Object detection and enhance image
+        seg_result, img_file = object_detection(file_path)
+        croped_img = crop_image(seg_result, img_file, img_name)
+        image = enhance_image(croped_img, img_name)
+
+        st.markdown("<h3 style='text-align: center;'>Enhanced Image</h1>", unsafe_allow_html=True) 
+        st.image(os.path.join('runs', 'segment', path['MAIN_FLOW_INFERENCE_FOLDER'], 'enhanced', data.name))
+
+    
+    with col2:
+        st.markdown("<h3 style='text-align: center;'>Detected Image</h1>", unsafe_allow_html=True) 
+        st.image(os.path.join('runs', 'segment',path['MAIN_FLOW_INFERENCE_FOLDER'],data.name))
+
+        # Rotation
+        processed_img = morphological_transform(image)
+        rotated_image, image = hoffman_transform(processed_img, image)
+        img_name = pytesseract_rotate(rotated_image, image, img_name)
+
+        st.markdown("<h3 style='text-align: center;'>Rotated Image</h1>", unsafe_allow_html=True) 
+        st.image(os.path.join('runs', 'segment', path['MAIN_FLOW_INFERENCE_FOLDER'], 'rotated_image', data.name))
+
+    # Apply OCR and NER
+    file_name = ocr(img_name)
+    Output_dict = ner(file_name)
+    # df = pd.DataFrame(Output_dict)
+
+    ocr_data = ""
+    with open(os.path.join('runs', 'segment', path['MAIN_FLOW_INFERENCE_FOLDER'], 'ocr_label_data', data.name.split('.')[0]+'.txt'),'r+') as f :
+        ocr_data = f.read()
+    st.header("OCR Text Output")
+    st.text(ocr_data)
+
+    st.header("NER Output")
+    
+    new_df = pd.DataFrame()
+    new_df['Entity'] = list(Output_dict.keys())
+    
+    # print(df)
+    new_df['Value'] = list(Output_dict.values())
+    new_df['Value'] = new_df['Value'].astype('str')
+    st.table(new_df)
+
+else:
+    img_name = '3.jpg'
+    img = cv2.imread(img_name,0)
+    
+    if img.shape[0] > 1500:
+        height, width = img.shape 
+        img = img[height//4:-height//4, width//4:-width//4]
+    
+    cv2.imwrite(os.path.join('grey_images',img_name), img)
+
+    #call main function
+    # main(os.path.join('grey_images',img_name))
+    file_path = os.path.join('grey_images',img_name)
+    img_name = os.path.basename(file_path)
+
+
+    col1,col2 = st.columns(2)
+
+    with col1:
+        st.markdown("<h3 style='text-align: center;'>Grey Image</h1>", unsafe_allow_html=True) 
+        st.image(os.path.join('grey_images',img_name))
+
+        # Object detection and enhance image
+        seg_result, img_file = object_detection(file_path)
+        croped_img = crop_image(seg_result, img_file, img_name)
+        image = enhance_image(croped_img, img_name)
+
+        st.markdown("<h3 style='text-align: center;'>Enhanced Image</h1>", unsafe_allow_html=True) 
+        st.image(os.path.join('runs', 'segment', path['MAIN_FLOW_INFERENCE_FOLDER'], 'enhanced', img_name))
+
+    
+    with col2:
+        st.markdown("<h3 style='text-align: center;'>Detected Image</h1>", unsafe_allow_html=True) 
+        st.image(os.path.join('runs', 'segment',path['MAIN_FLOW_INFERENCE_FOLDER'],img_name))
+
+        # Rotation
+        processed_img = morphological_transform(image)
+        rotated_image, image = hoffman_transform(processed_img, image)
+        img_name = pytesseract_rotate(rotated_image, image, img_name)
+
+        st.markdown("<h3 style='text-align: center;'>Rotated Image</h1>", unsafe_allow_html=True) 
+        st.image(os.path.join('runs', 'segment', path['MAIN_FLOW_INFERENCE_FOLDER'], 'rotated_image', img_name))
+
+    # Apply OCR and NER
+    file_name = ocr(img_name)
+    Output_dict = ner(file_name)
+    # df = pd.DataFrame(Output_dict)
+
+    ocr_data = ""
+    with open(os.path.join('runs', 'segment', path['MAIN_FLOW_INFERENCE_FOLDER'], 'ocr_label_data', img_name.split('.')[0]+'.txt'),'r+') as f :
+        ocr_data = f.read()
+    st.header("OCR Text Output")
+    st.text(ocr_data)
+
+    st.header("NER Output")
+    
+    new_df = pd.DataFrame()
+    new_df['Entity'] = list(Output_dict.keys())
+    
+    # print(df)
+    new_df['Value'] = list(Output_dict.values())
+    new_df['Value'] = new_df['Value'].astype('str')
+    st.table(new_df)

notebooks/readme.md

@@ -0,0 +1 @@
+

pipeline.py

@@ -0,0 +1,14 @@
+from pipeline_functions import object_detection, crop_image, enhance_image, morphological_transform, hoffman_transform, pytesseract_rotate, ocr,ner
+import os
+
+def main(path):
+    img_name = os.path.basename(path)
+    seg_result, img_file = object_detection(path)
+    croped_img = crop_image(seg_result, img_file, img_name)
+    image = enhance_image(croped_img, img_name)
+    processed_img = morphological_transform(image)
+    rotated_image, image = hoffman_transform(processed_img, image)
+    img_name = pytesseract_rotate(rotated_image, image, img_name)
+    file_name = ocr(img_name)
+    Output_dict = ner(file_name)
+    return Output_dict

pipeline_functions.py

@@ -0,0 +1,157 @@
+import cv2
+from ultralytics import YOLO 
+import numpy as np
+from paddleocr import PaddleOCR
+
+
+from ObjectDetection.functions import generateMask, cropBlackBackground, enhanceImage
+from OCR.rotation_functions import hoffman_transformation, rotate, pytesseractRotate
+from NER.ner_inference import inference
+
+import os
+from dotenv import load_dotenv
+from pathlib import Path
+env_path = Path('.') / '.env'
+load_dotenv(dotenv_path=env_path)
+
+path = {
+        'SEG_MODEL_PATH': str(os.getenv('SEG_MODEL_PATH')),
+        'MAIN_FLOW_GRAY_IMG_DIR_PATH': str(os.getenv('MAIN_FLOW_GRAY_IMG_DIR_PATH')),
+        'MAIN_FLOW_INFERENCE_FOLDER': str(os.getenv('MAIN_FLOW_INFERENCE_FOLDER')),
+        }
+seg_model = YOLO(path['SEG_MODEL_PATH'])
+
+CONF = 0.7
+
+
+
+
+def object_detection(file):
+    print("**************************** PERFORMING_OBJECT_DETECTION **************************** ")
+    img_file = cv2.imread(file,0)
+    img_name = os.path.basename(file)
+
+    Path(os.path.join(path['MAIN_FLOW_GRAY_IMG_DIR_PATH'])).mkdir(parents=True, exist_ok=True)
+    cv2.imwrite(os.path.join(path['MAIN_FLOW_GRAY_IMG_DIR_PATH'],img_name),img_file)
+    result = seg_model(os.path.join(path['MAIN_FLOW_GRAY_IMG_DIR_PATH'],img_name),conf = CONF,save = True,name = path['MAIN_FLOW_INFERENCE_FOLDER'],exist_ok = True)
+    
+    return result, img_file
+    
+    
+def crop_image(seg_result, img_file, img_name):
+    print("**************************** CROPPING_IMAGE **************************** ")
+    for res in seg_result:
+        
+        croped_img, mask = generateMask(res, img_file)
+
+        if croped_img is not None:
+            croped_img = cropBlackBackground(croped_img)
+
+            # save to file
+            try: 
+                Path('runs').mkdir(parents=True, exist_ok=True)
+                Path(os.path.join('runs', 'segment')).mkdir(parents=True, exist_ok=True)
+                Path(os.path.join('runs', 'segment', path['MAIN_FLOW_INFERENCE_FOLDER'])).mkdir(parents=True, exist_ok=True)
+                Path(os.path.join('runs', 'segment', path['MAIN_FLOW_INFERENCE_FOLDER'], 'crops_seg')).mkdir(parents=True, exist_ok=True)
+                Path(os.path.join('runs', 'segment', path['MAIN_FLOW_INFERENCE_FOLDER'], 'masks')).mkdir(parents=True, exist_ok=True)
+            except OSError as error:  
+                print(error)
+                pass
+                
+            cv2.imwrite(os.path.join('runs', 'segment', path['MAIN_FLOW_INFERENCE_FOLDER'], 'masks', img_name), mask)
+            cv2.imwrite(os.path.join('runs', 'segment', path['MAIN_FLOW_INFERENCE_FOLDER'], 'crops_seg', img_name), croped_img )
+            return croped_img
+    return img_file 
+
+def enhance_image(croped_img, img_name):
+    print("**************************** ENHANCE_IMAGE **************************** ")
+    image = None
+    if croped_img is not None:
+        image = enhanceImage(croped_img)
+        
+        if image is not None:
+            try: 
+                Path('runs').mkdir(parents=True, exist_ok=True)
+                Path(os.path.join('runs', 'segment')).mkdir(parents=True, exist_ok=True)
+                Path(os.path.join('runs', 'segment', path['MAIN_FLOW_INFERENCE_FOLDER'])).mkdir(parents=True, exist_ok=True)
+                Path(os.path.join('runs', 'segment', path['MAIN_FLOW_INFERENCE_FOLDER'], 'enhanced')).mkdir(parents=True, exist_ok=True)
+            except OSError as error:  
+                print(error)
+                pass
+                
+        cv2.imwrite(os.path.join('runs', 'segment', path['MAIN_FLOW_INFERENCE_FOLDER'], 'enhanced', img_name), image )
+    
+    return image
+
+def morphological_transform(image):
+
+    print("**************************** APPLY_MORPHOLOGICAL_TRANSFORM **************************** ")
+    processed_img = cv2.resize(image,None,fx=2.7, fy=3)
+    kernel = np.ones((2,2),np.uint8)
+    processed_img = cv2.dilate(processed_img,kernel)
+    sharpen_kernel = np.array([[-1,-1,-1], [-1,9,-1], [-1,-1,-1]])
+    processed_img = cv2.filter2D(processed_img, -1, sharpen_kernel)
+    
+    return processed_img
+
+def hoffman_transform(processed_img, original_img):
+    print("**************************** APPLY_HOFFMAN_TRANSFORM **************************** ")
+    rotated_image,angle = hoffman_transformation(processed_img, True)
+    original_img  = rotate(original_img,angle)
+    
+    return rotated_image, original_img
+
+def pytesseract_rotate(rotated_image, original_img, img_name):
+    print("**************************** APPLY_PYTESSERACT_ROTATION **************************** ")
+    rotated_image = pytesseractRotate(rotated_image,original_img,1)
+
+    if rotated_image is not None:
+        try: 
+            Path('runs').mkdir(parents=True, exist_ok=True)
+            Path(os.path.join('runs', 'segment')).mkdir(parents=True, exist_ok=True)
+            Path(os.path.join('runs', 'segment', path['MAIN_FLOW_INFERENCE_FOLDER'])).mkdir(parents=True, exist_ok=True)
+            Path(os.path.join('runs', 'segment', path['MAIN_FLOW_INFERENCE_FOLDER'], 'rotated_image')).mkdir(parents=True, exist_ok=True)
+        except OSError as error:  
+            print(error)
+            pass
+        
+        cv2.imwrite(os.path.join('runs', 'segment', path['MAIN_FLOW_INFERENCE_FOLDER'], 'rotated_image', img_name), rotated_image)
+        
+    return img_name
+
+def ocr(img_name):
+    print("**************************** APPLY_OCR **************************** ")
+    ocr = PaddleOCR(use_angle_cls=True, lang='en') # need to run only once to download and load model into memory
+    result = ocr.ocr(os.path.join('runs', 'segment', path['MAIN_FLOW_INFERENCE_FOLDER'], 'rotated_image', img_name), cls=True)
+    
+    ocr_output_paddle = []
+    if result is not None:
+        try:
+            for i in result:
+                ocr_output_paddle.append(" ".join([line[1][0] for line in i]))
+        except:
+            pass
+        try: 
+            Path('runs').mkdir(parents=True, exist_ok=True)
+            Path(os.path.join('runs', 'segment')).mkdir(parents=True, exist_ok=True)
+            Path(os.path.join('runs', 'segment', path['MAIN_FLOW_INFERENCE_FOLDER'])).mkdir(parents=True, exist_ok=True)
+            Path(os.path.join('runs', 'segment', path['MAIN_FLOW_INFERENCE_FOLDER'], 'ocr_label_data')).mkdir(parents=True, exist_ok=True)
+        except OSError as error:  
+            print(error)
+
+    file_name = img_name.split('.')[0] +'.txt'
+    with open(os.path.join('runs', 'segment', path['MAIN_FLOW_INFERENCE_FOLDER'], 'ocr_label_data',file_name) ,"w+") as f:
+        f.write("\n".join(ocr_output_paddle))
+    
+    return file_name
+def ner(file_name):
+    print("**************************** APPLY_NER **************************** ")
+    # print(file_name)
+    ocr_file = os.path.join('runs', 'segment',path['MAIN_FLOW_INFERENCE_FOLDER'],'ocr_label_data',file_name)
+    with open(ocr_file,'r+') as f:
+        sent = f.read()
+    print(sent)
+    output_dict = inference(sent)
+    
+    return output_dict
+ 

requirements.txt

@@ -0,0 +1,188 @@
+accelerate==0.28.0
+altair==5.2.0
+anyio==4.3.0
+astor==0.8.1
+asttokens==2.4.1
+attrdict==2.0.1
+attrs==23.2.0
+Babel==2.14.0
+bce-python-sdk==0.9.5
+beautifulsoup4==4.12.3
+blinker==1.7.0
+boto3==1.34.73
+botocore==1.34.73
+bpemb==0.3.5
+cachetools==5.3.3
+certifi==2024.2.2
+charset-normalizer==3.3.2
+click==8.1.7
+colorama==0.4.6
+comm==0.2.2
+conllu==4.5.3
+contourpy==1.2.0
+cssselect==1.2.0
+cssutils==2.9.0
+cycler==0.12.1
+Cython==3.0.9
+debugpy==1.8.1
+decorator==5.1.1
+Deprecated==1.2.14
+et-xmlfile==1.1.0
+exceptiongroup==1.2.0
+executing==2.0.1
+filelock==3.13.3
+fire==0.6.0
+flair==0.13.1
+Flask==3.0.2
+flask-babel==4.0.0
+fonttools==4.50.0
+fsspec==2024.3.1
+ftfy==6.2.0
+future==1.0.0
+gdown==5.1.0
+gensim==4.3.2
+gitdb==4.0.11
+GitPython==3.1.42
+h11==0.14.0
+httpcore==1.0.4
+httpx==0.27.0
+huggingface-hub==0.22.2
+idna==3.6
+imageio==2.34.0
+imgaug==0.4.0
+importlib_metadata==7.1.0
+importlib_resources==6.4.0
+ipykernel==6.29.4
+ipython==8.18.1
+itsdangerous==2.1.2
+Janome==0.5.0
+jedi==0.19.1
+Jinja2==3.1.3
+jmespath==1.0.1
+joblib==1.3.2
+jsonschema==4.21.1
+jsonschema-specifications==2023.12.1
+jupyter_client==8.6.1
+jupyter_core==5.7.2
+kiwisolver==1.4.5
+langdetect==1.0.9
+lazy_loader==0.3
+lmdb==1.4.1
+lxml==5.1.0
+markdown-it-py==3.0.0
+MarkupSafe==2.1.5
+matplotlib==3.8.3
+matplotlib-inline==0.1.6
+mdurl==0.1.2
+more-itertools==10.2.0
+mpld3==0.5.10
+mpmath==1.3.0
+nest-asyncio==1.6.0
+networkx==3.2.1
+nltk==3.8.1
+numpy==1.26.4
+nvidia-cublas-cu12==12.1.3.1
+nvidia-cuda-cupti-cu12==12.1.105
+nvidia-cuda-nvrtc-cu12==12.1.105
+nvidia-cuda-runtime-cu12==12.1.105
+nvidia-cudnn-cu12==8.9.2.26
+nvidia-cufft-cu12==11.0.2.54
+nvidia-curand-cu12==10.3.2.106
+nvidia-cusolver-cu12==11.4.5.107
+nvidia-cusparse-cu12==12.1.0.106
+nvidia-nccl-cu12==2.19.3
+nvidia-nvjitlink-cu12==12.4.99
+nvidia-nvtx-cu12==12.1.105
+opencv-contrib-python==4.6.0.66
+opencv-python==4.6.0.66
+opencv-python-headless==4.9.0.80
+openpyxl==3.1.2
+opt-einsum==3.3.0
+packaging==23.2
+paddleocr==2.7.2
+paddlepaddle==2.6.1
+pandas==2.2.1
+parso==0.8.3
+pdf2docx==0.5.8
+pexpect==4.9.0
+pillow==10.2.0
+platformdirs==4.2.0
+pptree==3.1
+premailer==3.10.0
+prompt-toolkit==3.0.43
+protobuf==3.20.2
+psutil==5.9.8
+ptyprocess==0.7.0
+pure-eval==0.2.2
+py-cpuinfo==9.0.0
+pyarrow==15.0.2
+pyclipper==1.3.0.post5
+pycryptodome==3.20.0
+pydeck==0.8.1b0
+Pygments==2.17.2
+PyMuPDF==1.24.0
+PyMuPDFb==1.24.0
+pyparsing==3.1.2
+PySocks==1.7.1
+pytesseract==0.3.10
+python-dateutil==2.9.0.post0
+python-docx==1.1.0
+python-dotenv==1.0.1
+pytorch_revgrad==0.2.0
+pytz==2024.1
+PyYAML==6.0.1
+pyzmq==25.1.2
+rapidfuzz==3.7.0
+rarfile==4.1
+referencing==0.34.0
+regex==2023.12.25
+requests==2.31.0
+rich==13.7.1
+rpds-py==0.18.0
+s3transfer==0.10.1
+safetensors==0.4.2
+scikit-image==0.22.0
+scikit-learn==1.4.1.post1
+scipy==1.12.0
+seaborn==0.13.2
+segtok==1.5.11
+semver==3.0.2
+sentencepiece==0.1.99
+shapely==2.0.3
+six==1.16.0
+smart-open==7.0.4
+smmap==5.0.1
+sniffio==1.3.1
+soupsieve==2.5
+sqlitedict==2.1.0
+stack-data==0.6.3
+streamlit==1.32.2
+sympy==1.12
+tabulate==0.9.0
+tenacity==8.2.3
+termcolor==2.4.0
+thop==0.1.1.post2209072238
+threadpoolctl==3.4.0
+tifffile==2024.2.12
+tokenizers==0.15.2
+toml==0.10.2
+toolz==0.12.1
+torch==2.2.1
+torchvision==0.17.1
+tornado==6.4
+tqdm==4.66.2
+traitlets==5.14.2
+transformer-smaller-training-vocab==0.3.3
+transformers==4.39.2
+triton==2.2.0
+typing_extensions==4.10.0
+tzdata==2024.1
+ultralytics==8.1.34
+urllib3==1.26.18
+visualdl==2.5.3
+watchdog==4.0.0
+wcwidth==0.2.13
+Werkzeug==3.0.1
+Wikipedia-API==0.6.0
+wrapt==1.16.0
+zipp==3.18.1