Upload 7 files

.gitattributes

@@ -33,3 +33,5 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
 *.zip filter=lfs diff=lfs merge=lfs -text
 *.zst filter=lfs diff=lfs merge=lfs -text
 *tfevents* filter=lfs diff=lfs merge=lfs -text
+model_farm_deeplabv3_resnet50_qcs6490_tflite_fp32_aidlite/cpp/deeplab2.png filter=lfs diff=lfs merge=lfs -text
+model_farm_deeplabv3_resnet50_qcs6490_tflite_fp32_aidlite/python/deeplab2.png filter=lfs diff=lfs merge=lfs -text

model_farm_deeplabv3_resnet50_qcs6490_tflite_fp32_aidlite/README.md

@@ -0,0 +1,55 @@
+## Model Information
+### Source model
+- Input shape: 520x520
+- Number of parameters: 40.06M
+- Model size: 160.16M
+- Output shape: 1x21x520x520
+
+Source model repository: [deeplabv3](https://github.com/pytorch/vision/blob/main/torchvision/models/segmentation/deeplabv3.py)
+
+### Converted model
+
+- Precision: FP32
+- Backend: TFLITE
+- Target Device: FV01 QCS6490
+
+## Inference with AidLite SDK
+
+### SDK installation
+Model Farm uses AidLite SDK as the model inference SDK. For details, please refer to the [AidLite Developer Documentation](https://v2.docs.aidlux.com/en/sdk-api/aidlite-sdk/)
+
+- install AidLite SDK
+
+```bash
+# Install the appropriate version of the aidlite sdk
+sudo aid-pkg update
+sudo aid-pkg install aidlite-sdk
+# Download the qnn version that matches the above backend. Eg Install QNN2.23 Aidlite: sudo aid-pkg install aidlite-qnn223
+sudo aid-pkg install aidlite-{QNN VERSION}
+```
+
+- Verify AidLite SDK
+
+```bash
+# aidlite sdk c++ check
+python3 -c "import aidlite ; print(aidlite.get_library_version())"
+
+# aidlite sdk python check
+python3 -c "import aidlite ; print(aidlite.get_py_library_version())"
+```
+
+### Run Demo
+#### python
+```bash
+cd deeplabv3_resnet50/model_farm_deeplabv3_resnet50_qcs6490_qnn2.16_fp32_aidlite
+python3  python/run_test.py --target_model ./models/cutoff_deeplabv3_resnet50_fp32.tflite --imgs ./python/deeplab2.png  --invoke_nums 10
+```
+
+#### c++
+```bash
+cd deeplabv3_resnet50/model_farm_deeplabv3_resnet50_qcs6490_qnn2.16_fp32_aidlite/cpp
+mkdir build && cd build 
+cmake ..
+make
+./run_test
+```

model_farm_deeplabv3_resnet50_qcs6490_tflite_fp32_aidlite/cpp/CMakeLists.txt

@@ -0,0 +1,32 @@
+cmake_minimum_required (VERSION 3.5)
+project("run_test")
+
+find_package(OpenCV REQUIRED)
+
+message(STATUS "oPENCV Library status:")
+message(STATUS ">version:${OpenCV_VERSION}")
+message(STATUS "Include:${OpenCV_INCLUDE_DIRS}")
+
+set(CMAKE_CXX_FLAGS "-Wno-error=deprecated-declarations -Wno-deprecated-declarations")
+
+include_directories(
+    /usr/local/include
+    /usr/include/opencv4
+)
+
+link_directories(
+    /usr/local/lib/
+)
+
+file(GLOB SRC_LISTS 
+    ${CMAKE_CURRENT_SOURCE_DIR}/run_test.cpp  
+)
+
+add_executable(run_test ${SRC_LISTS})
+
+target_link_libraries(run_test
+    aidlite
+	${OpenCV_LIBS}
+    pthread
+    jsoncpp
+)

model_farm_deeplabv3_resnet50_qcs6490_tflite_fp32_aidlite/cpp/run_test.cpp

@@ -0,0 +1,264 @@
+#include <iostream>
+#include <fstream>
+#include <opencv2/opencv.hpp>
+#include <aidlux/aidlite/aidlite.hpp>
+#include <vector>
+#include <numeric>
+#include <cmath>
+#include <cfloat>
+#include <jsoncpp/json/json.h>
+
+using namespace cv;
+using namespace std;
+using namespace Aidlux::Aidlite;
+
+
+struct Args {
+    std::string target_model = "../../models/cutoff_deeplabv3_resnet50_fp32.tflite";
+    std::string imgs = "../deeplab2.png";
+    int invoke_nums = 1;
+    std::string model_type = "TFLITE";
+};
+
+
+Args parse_args(int argc, char* argv[]) {
+    Args args;
+    for (int i = 1; i < argc; ++i) {
+        std::string arg = argv[i];
+        if (arg == "--target_model" && i + 1 < argc) {
+            args.target_model = argv[++i];
+        } else if (arg == "--imgs" && i + 1 < argc) {
+            args.imgs = argv[++i];
+        } else if (arg == "--invoke_nums" && i + 1 < argc) {
+            args.invoke_nums = std::stoi(argv[++i]);
+        } else if (arg == "--model_type" && i + 1 < argc) {
+            args.model_type = argv[++i];
+        }
+    }
+    return args;
+}
+
+std::string to_lower(const std::string& str) {
+    std::string lower_str = str;
+    std::transform(lower_str.begin(), lower_str.end(), lower_str.begin(), [](unsigned char c) {
+        return std::tolower(c);
+    });
+    return lower_str;
+}
+
+static const cv::Vec3b PASCAL_VOC_COLORS[21] = {
+    {  0,   0,   0}, {128,   0,   0}, {  0, 128,   0}, {128, 128,   0},
+    {  0,   0, 128}, {128,   0, 128}, {  0, 128, 128}, {128, 128, 128},
+    { 64,   0,   0}, {192,   0,   0}, { 64, 128,   0}, {192, 128,   0},
+    { 64,   0, 128}, {192,   0, 128}, { 64, 128, 128}, {192, 128, 128},
+    {  0,  64,   0}, {128,  64,   0}, {  0, 192,   0}, {128, 192,   0},
+    {  0,  64, 128}
+};
+
+// simple decodeSegMap: classIdx CV_8UC1 → color CV_8UC3
+cv::Mat decodeSegMap(const cv::Mat& classIdx)
+{
+    CV_Assert(classIdx.type() == CV_8UC1);
+    cv::Mat color(classIdx.size(), CV_8UC3);
+    for(int y = 0; y < classIdx.rows; ++y) {
+        const uchar* p = classIdx.ptr<uchar>(y);
+        cv::Vec3b* q = color.ptr<cv::Vec3b>(y);
+        for(int x = 0; x < classIdx.cols; ++x) {
+            q[x] = PASCAL_VOC_COLORS[p[x]];
+        }
+    }
+    return color;
+}
+
+int transpose(float* src, unsigned int* src_dims, unsigned int* tsp_dims, float* dest){
+
+    int current_coordinate[4] = {0, 0, 0, 0};
+    for(int a = 0; a < src_dims[0]; ++a){
+        current_coordinate[0] = a;
+        for(int b = 0; b < src_dims[1]; ++b){
+            current_coordinate[1] = b;
+            for(int c = 0; c < src_dims[2]; ++c){
+                current_coordinate[2] = c;
+                for(int d = 0; d < src_dims[3]; ++d){
+                    current_coordinate[3] = d;
+
+                    int old_index = current_coordinate[0]*src_dims[1]*src_dims[2]*src_dims[3] + 
+                                    current_coordinate[1]*src_dims[2]*src_dims[3] + 
+                                    current_coordinate[2]*src_dims[3] + 
+                                    current_coordinate[3];
+
+                    int new_index = current_coordinate[tsp_dims[0]]*src_dims[tsp_dims[1]]*src_dims[tsp_dims[2]]*src_dims[tsp_dims[3]] + 
+                                    current_coordinate[tsp_dims[1]]*src_dims[tsp_dims[2]]*src_dims[tsp_dims[3]] + 
+                                    current_coordinate[tsp_dims[2]]*src_dims[tsp_dims[3]] + 
+                                    current_coordinate[tsp_dims[3]];
+
+                    dest[new_index] = src[old_index];
+                }
+            }
+        }
+    }
+
+    return EXIT_SUCCESS;
+}
+
+
+
+int invoke(const Args& args) {
+    std::cout << "Start main ... ... Model Path: " << args.target_model << "\n"
+              << "Image Path: " << args.imgs << "\n"
+              << "Inference Nums: " << args.invoke_nums << "\n"
+              << "Model Type: " << args.model_type << "\n";
+    Model* model = Model::create_instance(args.target_model);
+    if(model == nullptr){
+        printf("Create model failed !\n");
+        return EXIT_FAILURE;
+    }
+    Config* config = Config::create_instance();
+    if(config == nullptr){
+        printf("Create config failed !\n");
+        return EXIT_FAILURE;
+    }
+    config->implement_type = ImplementType::TYPE_LOCAL;
+    std::string model_type_lower = to_lower(args.model_type);
+    if (model_type_lower == "qnn"){
+        config->framework_type = FrameworkType::TYPE_QNN;
+        config->accelerate_type = AccelerateType::TYPE_DSP;
+        config->is_quantify_model = 1;
+    } else if (model_type_lower == "snpe2" || model_type_lower == "snpe") {
+        config->framework_type = FrameworkType::TYPE_SNPE2;
+        config->accelerate_type = AccelerateType::TYPE_DSP;
+        config->is_quantify_model = 1;
+    } else if (model_type_lower == "tflite") {
+        config->framework_type = FrameworkType::TYPE_TFLITE;
+        config->accelerate_type = AccelerateType::TYPE_GPU;
+    } else {
+        printf("Unsupported model type: %s\n", args.model_type.c_str());
+        return EXIT_FAILURE;
+    }
+
+    unsigned int model_h = 520;
+    unsigned int model_w = 520;
+    std::vector<std::vector<uint32_t>> input_shapes = {{1,model_h,model_w,3}};
+    std::vector<std::vector<uint32_t>> output_shapes = {{1,520,520,21}};
+    model->set_model_properties(input_shapes, Aidlux::Aidlite::DataType::TYPE_FLOAT32, output_shapes, Aidlux::Aidlite::DataType::TYPE_FLOAT32);
+    std::unique_ptr<Interpreter> fast_interpreter = InterpreterBuilder::build_interpretper_from_model_and_config(model, config);
+    if(fast_interpreter == nullptr){
+        printf("build_interpretper_from_model_and_config failed !\n");
+        return EXIT_FAILURE;
+    }
+    int result = fast_interpreter->init();
+    if(result != EXIT_SUCCESS){
+        printf("interpreter->init() failed !\n");
+        return EXIT_FAILURE;
+    }
+    // load model
+    fast_interpreter->load_model();
+    if(result != EXIT_SUCCESS){
+        printf("interpreter->load_model() failed !\n");
+        return EXIT_FAILURE;
+    }
+    printf("detect model load success!\n");
+
+    cv::Mat frame = cv::imread(args.imgs);
+    if (frame.empty()) {
+        printf("detect image load failed!\n");
+        return 1;
+    }
+    printf("img_src cols: %d, img_src rows: %d\n", frame.cols, frame.rows);
+    cv::Mat input_data;
+    cv::Mat frame_clone = frame.clone();
+    cv::cvtColor(frame_clone, frame_clone, cv::COLOR_BGR2RGB);
+    cv::resize(frame_clone, frame_clone, cv::Size(model_w, model_h));
+    frame_clone.convertTo(input_data, CV_32FC3, 1.0f / 255.0f);
+
+
+    float *outdata0 = nullptr;
+    std::vector<float> invoke_time;
+    for (int i = 0; i < args.invoke_nums; ++i) {
+        result = fast_interpreter->set_input_tensor(0, input_data.data);
+        if(result != EXIT_SUCCESS){
+            printf("interpreter->set_input_tensor() failed !\n");
+            return EXIT_FAILURE;
+        }
+        auto t1 = std::chrono::high_resolution_clock::now();
+        result = fast_interpreter->invoke();
+        auto t2 = std::chrono::high_resolution_clock::now();
+        std::chrono::duration<double> cost_time = t2 - t1;
+        invoke_time.push_back(cost_time.count() * 1000);
+        if(result != EXIT_SUCCESS){
+            printf("interpreter->invoke() failed !\n");
+            return EXIT_FAILURE;
+        }
+        uint32_t out_data_0 = 0;
+        result = fast_interpreter->get_output_tensor(0, (void**)&outdata0, &out_data_0);
+        if(result != EXIT_SUCCESS){
+            printf("interpreter->get_output_tensor() 1 failed !\n");
+            return EXIT_FAILURE;
+        }
+
+    }
+
+    float max_invoke_time = *std::max_element(invoke_time.begin(), invoke_time.end());
+    float min_invoke_time = *std::min_element(invoke_time.begin(), invoke_time.end());
+    float mean_invoke_time = std::accumulate(invoke_time.begin(), invoke_time.end(), 0.0f) / args.invoke_nums;
+    float var_invoketime = 0.0f;
+    for (auto time : invoke_time) {
+        var_invoketime += (time - mean_invoke_time) * (time - mean_invoke_time);
+    }
+    var_invoketime /= args.invoke_nums;
+    printf("=======================================\n");
+    printf("QNN inference %d times :\n --mean_invoke_time is %f \n --max_invoke_time is %f \n --min_invoke_time is %f \n --var_invoketime is %f\n", 
+        args.invoke_nums, mean_invoke_time, max_invoke_time, min_invoke_time, var_invoketime);
+    printf("=======================================\n");
+
+    //  post process
+    
+    unsigned int src_dims[4] = {1, 520,520,21};
+    unsigned int tsp_dims[4] = {0,3,1,2};
+    unsigned int stride_data_num = 1*520*520*21;
+    float* format_data = new float[stride_data_num];
+    transpose(outdata0, src_dims, tsp_dims, format_data);
+
+    
+    cv::Mat prediction(model_h, model_w, CV_8UC1);
+    for(int y = 0; y < model_h; ++y) {
+        for(int x = 0; x < model_w; ++x) {
+            float maxVal = -FLT_MAX;
+            int   maxIdx = 0;
+            for(int c = 0; c < 21; ++c) {
+                int idx = c*model_h*model_w + y*model_w + x;
+                float v = format_data[idx];
+                if(v > maxVal) {
+                    maxVal = v;
+                    maxIdx = c;
+                }
+            }
+            prediction.at<uchar>(y,x) = static_cast<uchar>(maxIdx);
+        }
+    }
+
+    // 2) decode to color map
+    cv::Mat colorMap = decodeSegMap(prediction);
+    // 3) blend with original frame (assumes frame is same size as out_h×out_w)
+    cv::Mat resizedFrame;
+    cv::resize(frame, resizedFrame, cv::Size(model_w, model_h), 0, 0, cv::INTER_LINEAR);
+
+    cv::Mat mask_img;
+    cv::addWeighted(resizedFrame, 0.5, colorMap, 0.5, 0.0, mask_img);
+
+    // 4) resize back to original resolution and save
+    cv::resize(mask_img, mask_img, cv::Size(model_w, model_h), 0, 0, cv::INTER_NEAREST);
+    cv::imwrite("./original_mask_result_TFlite_fp32.jpg", mask_img);
+
+
+
+
+    fast_interpreter->destory();
+    return 0;
+}
+
+
+int main(int argc, char* argv[]) {
+    Args args = parse_args(argc, argv);
+    return invoke(args);
+}

model_farm_deeplabv3_resnet50_qcs6490_tflite_fp32_aidlite/models/cutoff_deeplabv3_resnet50_fp32.tflite

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:5207ca983775ed595b4d128f1c56b2e48cee0df9c0bfd4d7998875f78143bb42
+size 158483120

model_farm_deeplabv3_resnet50_qcs6490_tflite_fp32_aidlite/python/run_test.py

@@ -0,0 +1,141 @@
+import onnxruntime
+import cv2
+import numpy as np
+import aidlite
+import argparse
+import time
+
+# Define the helper function
+def decode_segmap(image, nc=21):
+    label_colors = np.array([(0, 0, 0),  # 0=background
+                # 1=aeroplane, 2=bicycle, 3=bird, 4=boat, 5=bottle
+                (128, 0, 0), (0, 128, 0), (128, 128, 0), (0, 0, 128), (128, 0, 128),
+                # 6=bus, 7=car, 8=cat, 9=chair, 10=cow
+                (0, 128, 128), (128, 128, 128), (64, 0, 0), (192, 0, 0), (64, 128, 0),
+                # 11=dining table, 12=dog, 13=horse, 14=motorbike, 15=person
+                (192, 128, 0), (64, 0, 128), (192, 0, 128), (64, 128, 128), (192, 128, 128),
+                # 16=potted plant, 17=sheep, 18=sofa, 19=train, 20=tv/monitor
+                (0, 64, 0), (128, 64, 0), (0, 192, 0), (128, 192, 0), (0, 64, 128)])
+    r = np.zeros_like(image).astype(np.uint8)
+    g = np.zeros_like(image).astype(np.uint8)
+    b = np.zeros_like(image).astype(np.uint8)
+    for l in range(0, nc):
+        idx = image == l
+        r[idx] = label_colors[l, 0]
+        g[idx] = label_colors[l, 1]
+        b[idx] = label_colors[l, 2]
+    rgb = np.stack([r, g, b], axis=2)
+    return rgb
+
+def run(args):   
+    print("Start main ... ...")
+    # aidlite.set_log_level(aidlite.LogLevel.INFO)
+    # aidlite.log_to_stderr()
+    # print(f"Aidlite library version : {aidlite.get_library_version()}")
+    # print(f"Aidlite python library version : {aidlite.get_py_library_version()}")
+    config = aidlite.Config.create_instance()
+    if config is None:
+        print("Create config failed !")
+        return False
+    
+    config.implement_type = aidlite.ImplementType.TYPE_LOCAL
+    if args.model_type.lower()=="qnn":
+        config.framework_type = aidlite.FrameworkType.TYPE_QNN
+        config.accelerate_type = aidlite.AccelerateType.TYPE_DSP
+        config.is_quantify_model = 1
+    elif args.model_type.lower()=="snpe2" or args.model_type.lower()=="snpe":
+        config.framework_type = aidlite.FrameworkType.TYPE_SNPE2
+        config.accelerate_type = aidlite.AccelerateType.TYPE_DSP
+        config.is_quantify_model = 1
+    elif args.model_type.lower()=="tflite":
+        config.framework_type = aidlite.FrameworkType.TYPE_TFLITE
+        config.accelerate_type = aidlite.AccelerateType.TYPE_GPU
+        config.is_quantify_model = 0
+        
+    
+
+    model = aidlite.Model.create_instance(args.target_model)
+    if model is None:
+        print("Create model failed !")
+        return False
+    
+    input_shapes=[[1,520,520,3]]
+    output_shapes=[[1,520,520,21]]
+    model.set_model_properties(input_shapes, aidlite.DataType.TYPE_FLOAT32,
+                               output_shapes, aidlite.DataType.TYPE_FLOAT32)
+    
+    interpreter = aidlite.InterpreterBuilder.build_interpretper_from_model_and_config(model, config)
+    if interpreter is None:
+        print("build_interpretper_from_model_and_config failed !")
+        return None
+    result = interpreter.init()
+    if result != 0:
+        print(f"interpreter init failed !")
+        return False
+    result = interpreter.load_model()
+    if result != 0:
+        print("interpreter load model failed !")
+        return False
+    print("detect model load success!")
+
+    frame = cv2.imread(args.imgs)
+    w,h = frame.shape[1],frame.shape[0]
+    frame = cv2.resize(frame,(520, 520))
+    new_img = cv2.cvtColor(frame, cv2.COLOR_BGR2RGB)
+    img = new_img/255.0
+    img_input = img.astype(np.float32)
+
+    invoke_time=[]
+    for i in range(args.invoke_nums):
+        result = interpreter.set_input_tensor(0, img_input.data)
+        if result != 0:
+            print("interpreter set_input_tensor() failed")
+        
+        t1=time.time()
+        result = interpreter.invoke()
+        cost_time = (time.time()-t1)*1000
+        invoke_time.append(cost_time)
+        
+        if result != 0:
+            print("interpreter set_input_tensor() failed")
+        
+        qnn_out = interpreter.get_output_tensor(0)
+
+    result = interpreter.destory()
+    ## time 统计
+    max_invoke_time = max(invoke_time)
+    min_invoke_time = min(invoke_time)
+    mean_invoke_time = sum(invoke_time)/args.invoke_nums
+    var_invoketime=np.var(invoke_time)
+    print("=======================================")
+    print(f"QNN inference {args.invoke_nums} times :\n --mean_invoke_time is {mean_invoke_time} \n --max_invoke_time is {max_invoke_time} \n --min_invoke_time is {min_invoke_time} \n --var_invoketime is {var_invoketime}")
+    print("=======================================")
+    
+    # 后处理
+    qnn_out1 = qnn_out.reshape(output_shapes[0]).transpose(0,3,1,2)
+    prediction = np.argmax(qnn_out1[0],axis=0)
+    # print("prediction :",prediction.shape,prediction)
+    prediction = prediction.astype(np.uint8)
+    test = decode_segmap(prediction,args.cls_num)
+
+    mask_img = cv2.addWeighted(frame,0.5,test,0.5,0)
+    mask_img = cv2.resize(mask_img,(w,h),interpolation=cv2.INTER_NEAREST)
+
+    cv2.imwrite(f"./python/original_mask_result_TFlite_fp32.jpg",mask_img)
+    print("Image segmentation has completed!")
+
+
+
+def parser_args():
+    parser = argparse.ArgumentParser(description="Run model benchmarks")
+    parser.add_argument('--target_model',type=str,default='./models/cutoff_deeplabv3_resnet50_fp32.tflite',help="inference model path")
+    parser.add_argument('--imgs',type=str,default='./python/deeplab2.png',help="Predict images path")
+    parser.add_argument('--cls_num',type=int,default=21,help="The number of targets detected")
+    parser.add_argument('--invoke_nums',type=int,default=10,help="Inference nums")
+    parser.add_argument('--model_type',type=str,default='TFLITE',help="run backend")
+    args = parser.parse_args()
+    return args
+    
+if __name__ == "__main__":
+    args = parser_args()
+    run(args)