text: /**
   * Copyright 2021-2022 Huawei Technologies Co., Ltd
   *
   * Licensed under the Apache License, Version 2.0 (the "License");
   * you may not use this file except in compliance with the License.
   * You may obtain a copy of the License at
   *
   * http://www.apache.org/licenses/LICENSE-2.0
   *
   * Unless required by applicable law or agreed to in writing, software
   * distributed under the License is distributed on an "AS IS" BASIS,
   * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
   * See the License for the specific language governing permissions and
   * limitations under the License.
   */
  #include "common/common.h"
  #include "minddata/dataset/core/global_context.h"
  #include "minddata/dataset/engine/serdes.h"
  #include "minddata/dataset/include/dataset/datasets.h"
  #include "minddata/dataset/include/dataset/vision.h"
  #include "minddata/dataset/include/dataset/transforms.h"
  #include "minddata/dataset/kernels/ir/data/transforms_ir.h"
  
  using namespace mindspore::dataset;
  using mindspore::dataset::DatasetNode;
  
  using mindspore::dataset::ShuffleMode;
  using mindspore::dataset::Tensor;
  
  class MindDataTestDeserialize : public UT::DatasetOpTesting {
   protected:
  };
  
  void compare_dataset(std::shared_ptr<DatasetNode> ds) {
    nlohmann::json out_json;
    ASSERT_OK(Serdes::SaveToJSON(ds, "dataset_pipeline.json", &out_json));
    // output the deserialized out_json to ds1 and then out_json1
    std::shared_ptr<DatasetNode> ds1;
    ASSERT_OK(Serdes::Deserialize("dataset_pipeline.json", &ds1));
    EXPECT_NE(ds1, nullptr);
  
    // check original and deserialized dataset are the same
    nlohmann::json out_json1;
    ASSERT_OK(Serdes::SaveToJSON(ds1, "dataset_pipeline_1.json", &out_json1));
    std::stringstream json_ss;
    json_ss << out_json;
    std::stringstream json_ss1;
    json_ss1 << out_json1;
    EXPECT_EQ(json_ss.str(), json_ss1.str());
    return;
  }
  
  // test mnist dataset, and special cases of tensor operations (no input or tensor operation input)
  TEST_F(MindDataTestDeserialize, TestDeserializeMnist) {
    MS_LOG(INFO) << "Doing MindDataTestDeserialize-Minist.";
    std::string data_dir = "./data/dataset/testMnistData";
    std::string usage = "all";
    std::shared_ptr<SamplerObj> sampler = std::make_shared<RandomSamplerObj>(true, 100);
    std::shared_ptr<DatasetNode> ds = std::make_shared<MnistNode>(data_dir, usage, sampler, nullptr);
    std::shared_ptr<TensorOperation> operation0 = std::make_shared<vision::EqualizeOperation>();
    std::shared_ptr<TensorOperation> operation1 = std::make_shared<vision::BoundingBoxAugmentOperation>(operation0, 0.5);
    std::shared_ptr<TensorOperation> operation2 = std::make_shared<vision::HorizontalFlipOperation>();
    std::shared_ptr<TensorOperation> operation3 = std::make_shared<vision::HwcToChwOperation>();
    std::shared_ptr<TensorOperation> operation4 = std::make_shared<vision::RgbaToBgrOperation>();
    std::shared_ptr<TensorOperation> operation5 = std::make_shared<vision::RgbaToRgbOperation>();
    std::shared_ptr<TensorOperation> operation6 = std::make_shared<vision::SwapRedBlueOperation>();
    std::vector<std::vector<std::pair<std::shared_ptr<TensorOperation>, double>>> policy;
    std::vector<std::pair<std::shared_ptr<TensorOperation>, double>> sub_policy;
    sub_policy.push_back(std::make_pair(operation1, 0.4));
    policy.push_back(sub_policy);
    std::shared_ptr<TensorOperation> operation7 = std::make_shared<vision::RandomSelectSubpolicyOperation>(policy);
    std::vector<std::shared_ptr<TensorOperation>> transforms;
    transforms.push_back(operation2);
    transforms.push_back(operation3);
    transforms.push_back(operation4);
    std::shared_ptr<TensorOperation> operation8 = std::make_shared<vision::UniformAugOperation>(transforms, 3);
    transforms.push_back(operation5);
    transforms.push_back(operation6);
    transforms.push_back(operation7);
    transforms.push_back(operation8);
    ds = std::make_shared<MapNode>(ds, transforms);
    ds = std::make_shared<BatchNode>(ds, 10, true);
    compare_dataset(ds);
  }
  
  // test celeba dataset and part of the tensor operation
  TEST_F(MindDataTestDeserialize, TestDeserializeCelebA) {
    MS_LOG(INFO) << "Doing MindDataTestDeserialize-CelebA.";
    std::string data_dir = "./data/dataset/testCelebAData/";
    std::string usage = "all";
    std::shared_ptr<SamplerObj> sampler = std::make_shared<DistributedSamplerObj>(1, 0, true, 2, 1, 1, true);
    bool decode = true;
    std::set<std::string> extensions = {};
    std::shared_ptr<DatasetCache> cache = nullptr;
    std::shared_ptr<DatasetNode> ds = std::make_shared<CelebANode>(data_dir, usage, sampler, decode, extensions, cache);
    std::vector<int32_t> size = {80, 80};
    std::vector<int32_t> size1 = {80, 80};
    std::vector<int32_t> coordinates = {5, 5};
    std::vector<int32_t> padding = {20, 20, 20, 20};
    std::vector<uint8_t> fill_value = {20, 20, 20};
    std::vector<uint32_t> ignore = {20, 20, 20, 20};
    std::vector<float> mean = {2.0, 2.0, 2.0, 2.0};
    std::vector<float> std = {0.5, 0.5, 0.5, 0.5};
    std::vector<float> translation = {0.5, 0.5};
    std::vector<float> shear = {0.5, 0.5};
    std::vector<float> sigma = {0.5, 0.5};
    InterpolationMode interpolation = InterpolationMode::kLinear;
    std::shared_ptr<TensorOperation> operation0 =
      std::make_shared<vision::AffineOperation>(0.0, translation, 0.5, shear, interpolation, fill_value);
    std::shared_ptr<TensorOperation> operation1 = std::make_shared<vision::AutoContrastOperation>(0.5, ignore);
    std::shared_ptr<TensorOperation> operation2 = std::make_shared<vision::CenterCropOperation>(size);
    std::shared_ptr<TensorOperation> operation3 =
      std::make_shared<vision::CutMixBatchOperation>(ImageBatchFormat::kNHWC, 0.1, 0.1);
    std::shared_ptr<TensorOperation> operation4 = std::make_shared<vision::CutOutOperation>(1, 1, true);
    std::shared_ptr<TensorOperation> operation5 = std::make_shared<vision::DecodeOperation>(true);
    std::shared_ptr<TensorOperation> operation6 = std::make_shared<vision::GaussianBlurOperation>(coordinates, sigma);
    std::shared_ptr<TensorOperation> operation7 = std::make_shared<vision::MixUpBatchOperation>(1.0);
    std::shared_ptr<TensorOperation> operation8 = std::make_shared<vision::NormalizeOperation>(mean, std, true);
    std::shared_ptr<TensorOperation> operation9 = std::make_shared<vision::NormalizePadOperation>(mean, std, "float",
                                                                                                  true);
    std::shared_ptr<TensorOperation> operation10 =
      std::make_shared<vision::PadOperation>(padding, fill_value, BorderType::kConstant);
    std::shared_ptr<TensorOperation> operation11 = std::make_shared<vision::RescaleOperation>(1.0, 0.5);
    std::shared_ptr<TensorOperation> operation12 = std::make_shared<vision::ResizePreserveAROperation>(10, 10, 0);
    std::shared_ptr<TensorOperation> operation13 = std::make_shared<vision::ResizeWithBBoxOperation>(size, interpolation);
    std::shared_ptr<TensorOperation> operation14 = std::make_shared<vision::ResizeOperation>(size, interpolation);
    std::vector<std::shared_ptr<TensorOperation>> operations;
    operations.push_back(operation0);
    operations.push_back(operation1);
    operations.push_back(operation2);
    operations.push_back(operation3);
    operations.push_back(operation4);
    operations.push_back(operation5);
    operations.push_back(operation6);
    operations.push_back(operation7);
    operations.push_back(operation8);
    operations.push_back(operation9);
    operations.push_back(operation10);
    operations.push_back(operation11);
    operations.push_back(operation12);
    operations.push_back(operation13);
    operations.push_back(operation14);
    ds = std::make_shared<RepeatNode>(ds, 2);
    ds = std::make_shared<MapNode>(ds, operations);
    compare_dataset(ds);
  }
  
  // test cifar10 dataset and random tensor operations
  TEST_F(MindDataTestDeserialize, TestDeserializeCifar10) {
    MS_LOG(INFO) << "Doing MindDataTestDeserialize-Cifar10.";
    std::string data_dir = "./data/dataset/testCifar10Data";
    std::string usage = "all";
    std::shared_ptr<DatasetCache> cache = nullptr;
    std::shared_ptr<SamplerObj> sampler = std::make_shared<SequentialSamplerObj>(0, 10);
    std::shared_ptr<DatasetNode> ds = std::make_shared<Cifar10Node>(data_dir, usage, sampler, cache);
    std::vector<float> center = {50.0, 50.0};
    std::vector<uint8_t> threshold = {5, 5};
    std::vector<uint8_t> fill_value = {150, 150, 150};
    std::vector<uint8_t> bit_range = {5, 15};
    std::vector<float> degrees = {0.0, 0.0};
    std::vector<float> scale = {0.5, 0.5};
    std::vector<float> ratio = {0.5, 0.5};
    std::vector<int32_t> size = {224, 224};
    std::vector<int32_t> padding = {20, 20, 20, 20};
    std::vector<float_t> translate_range = {0.0, 0.0, 0.0, 0.0};
    std::vector<float_t> scale_range = {1.0, 1.0};
    std::vector<float_t> shear_ranges = {0.0, 0.0, 0.0, 0.0};
    InterpolationMode interpolation = InterpolationMode::kLinear;
    std::shared_ptr<TensorOperation> operation1 = std::make_shared<vision::RandomRotationOperation>(
      degrees, InterpolationMode::kNearestNeighbour, true, center, fill_value);
    std::shared_ptr<TensorOperation> operation2 = std::make_shared<vision::RandomAffineOperation>(
      degrees, translate_range, scale_range, shear_ranges, interpolation, fill_value);
    std::shared_ptr<TensorOperation> operation3 = std::make_shared<vision::RandomColorOperation>(0.5, 10.5);
    std::shared_ptr<TensorOperation> operation4 =
      std::make_shared<vision::RandomCropDecodeResizeOperation>(size, scale, ratio, interpolation, 2);
    std::shared_ptr<TensorOperation> operation5 =
      std::make_shared<vision::RandomCropWithBBoxOperation>(size, padding, true, fill_value, BorderType::kConstant);
    std::shared_ptr<TensorOperation> operation6 = std::make_shared<vision::RandomHorizontalFlipOperation>(0.1);
    std::shared_ptr<TensorOperation> operation7 = std::make_shared<vision::RandomHorizontalFlipWithBBoxOperation>(0.1);
    std::shared_ptr<TensorOperation> operation8 = std::make_shared<vision::RandomPosterizeOperation>(bit_range);
    std::shared_ptr<TensorOperation> operation9 = std::make_shared<vision::RandomResizeOperation>(size);
    std::shared_ptr<TensorOperation> operation10 = std::make_shared<vision::RandomResizeWithBBoxOperation>(size);
    std::shared_ptr<TensorOperation> operation11 =
      std::make_shared<vision::RandomResizedCropOperation>(size, scale, ratio, interpolation, 2);
    std::shared_ptr<TensorOperation> operation12 =
      std::make_shared<vision::RandomResizedCropWithBBoxOperation>(size, scale, ratio, interpolation, 2);
    std::shared_ptr<TensorOperation> operation13 =
      std::make_shared<vision::RandomRotationOperation>(degrees, interpolation, true, center, fill_value);
    std::shared_ptr<TensorOperation> operation14 = std::make_shared<vision::RandomSharpnessOperation>(degrees);
    std::shared_ptr<TensorOperation> operation15 = std::make_shared<vision::RandomSolarizeOperation>(threshold);
    std::shared_ptr<TensorOperation> operation16 = std::make_shared<vision::RandomVerticalFlipOperation>(0.1);
    std::shared_ptr<TensorOperation> operation17 = std::make_shared<vision::RandomVerticalFlipWithBBoxOperation>(0.1);
    std::vector<std::shared_ptr<TensorOperation>> operations;
    operations.push_back(operation1);
    operations.push_back(operation2);
    operations.push_back(operation3);
    operations.push_back(operation4);
    operations.push_back(operation5);
    operations.push_back(operation6);
    operations.push_back(operation7);
    operations.push_back(operation8);
    operations.push_back(operation9);
    operations.push_back(operation10);
    operations.push_back(operation11);
    operations.push_back(operation12);
    operations.push_back(operation13);
    operations.push_back(operation14);
    operations.push_back(operation15);
    operations.push_back(operation16);
    operations.push_back(operation17);
    ds = std::make_shared<MapNode>(ds, operations);
    ds = std::make_shared<BatchNode>(ds, 1, true);
    ds = std::make_shared<SkipNode>(ds, 1);
    compare_dataset(ds);
  }
  
  TEST_F(MindDataTestDeserialize, TestDeserializeCifar100) {
    MS_LOG(INFO) << "Doing MindDataTestDeserialize-Cifar100.";
    std::string data_dir = "./data/dataset/testCifar100Data";
    std::string usage = "all";
    std::shared_ptr<DatasetCache> cache = nullptr;
    std::shared_ptr<SamplerObj> sampler = std::make_shared<SequentialSamplerObj>(0, 10);
    std::shared_ptr<DatasetNode> ds = std::make_shared<Cifar100Node>(data_dir, usage, sampler, cache);
    ds = std::make_shared<TakeNode>(ds, 6);
    std::shared_ptr<TensorOperation> operation = std::make_shared<vision::HorizontalFlipOperation>();
    std::vector<std::shared_ptr<TensorOperation>> ops = {operation};
    ds = std::make_shared<MapNode>(ds, ops);
    std::vector<std::shared_ptr<TensorOperation>> operations;
    std::vector<int32_t> size = {32, 32};
    std::vector<int32_t> padding = {4, 4, 4, 4};
    bool pad_if_needed = false;
    std::vector<uint8_t> fill_value = {4, 4, 4};
    InterpolationMode interpolation = InterpolationMode::kLinear;
    std::shared_ptr<TensorOperation> operation1 =
      std::make_shared<vision::RandomCropOperation>(size, padding, pad_if_needed, fill_value, BorderType::kConstant);
    size = {224, 224};
    std::shared_ptr<TensorOperation> operation2 = std::make_shared<vision::ResizeOperation>(size, interpolation);
    std::shared_ptr<TensorOperation> operation3 = std::make_shared<vision::RescaleOperation>(0.5, 0.0);
    std::vector<float> mean = {0.49, 0.48, 0.46};
    std::vector<float> std = {0.20, 0.199, 0.201};
    std::shared_ptr<TensorOperation> operation4 = std::make_shared<vision::NormalizeOperation>(mean, std, true);
    operations.push_back(operation1);
    operations.push_back(operation2);
    operations.push_back(operation3);
    operations.push_back(operation4);
    ds = std::make_shared<MapNode>(ds, operations);
    ds = std::make_shared<BatchNode>(ds, 3, true);
    ds = std::make_shared<RepeatNode>(ds, 1);
    compare_dataset(ds);
  }
  
  TEST_F(MindDataTestDeserialize, TestDeserializeCSV) {
    MS_LOG(INFO) << "Doing MindDataTestDeserialize-CSV.";
    std::string data_file = "./data/dataset/testCSV/1.csv";
    std::vector<std::string> dataset_files = {data_file};
    char field_delim = ',';
    std::vector<std::string> column_names = {"col1", "col2", "col3", "col4"};
    std::vector<std::string> columns = {"col1", "col4", "col2"};
    std::vector<std::shared_ptr<CsvBase>> column_defaults = {};
    std::shared_ptr<DatasetCache> cache = nullptr;
    std::shared_ptr<DatasetNode> ds = std::make_shared<CSVNode>(dataset_files, field_delim, column_defaults, column_names,
                                                                3, ShuffleMode::kGlobal, 1, 0, cache);
    ds = std::make_shared<ProjectNode>(ds, columns);
    compare_dataset(ds);
  }
  
  TEST_F(MindDataTestDeserialize, TestDeserializeImageFolder) {
    MS_LOG(INFO) << "Doing MindDataTestDeserialize-ImageFolder.";
    std::string dataset_dir = "./data/dataset/testPK/data";
    std::shared_ptr<SamplerObj> child_sampler = std::make_shared<PKSamplerObj>(3, true, 1);
    std::vector<double> weights = {1.0, 0.1, 0.02, 0.3, 0.4, 0.05, 1.2, 0.13, 0.14, 0.015, 0.16, 1.1};
    std::set<std::string> extensions = {};
    std::shared_ptr<DatasetCache> cache = nullptr;
    std::map<std::string, int32_t> class_indexing = {};
    std::shared_ptr<SamplerObj> sampler = std::make_shared<WeightedRandomSamplerObj>(weights, 11);
    sampler->AddChildSampler(child_sampler);
    std::shared_ptr<DatasetNode> ds =
      std::make_shared<ImageFolderNode>(dataset_dir, false, sampler, false, extensions, class_indexing, cache);
    ds = std::make_shared<RepeatNode>(ds, 1);
    std::vector<int32_t> size = {224, 224};
    std::vector<float> scale = {0.5, 0.5};
    std::vector<float> ratio = {0.5, 0.5};
    std::vector<float> center = {50.0, 50.0};
    std::vector<uint8_t> fill_value = {150, 150, 150};
    InterpolationMode interpolation = InterpolationMode::kLinear;
    std::shared_ptr<TensorOperation> operation1 = std::make_shared<vision::SoftDvppDecodeResizeJpegOperation>(size);
    std::vector<std::shared_ptr<TensorOperation>> ops = {operation1};
    ds = std::make_shared<MapNode>(ds, ops);
    std::vector<std::shared_ptr<TensorOperation>> operations;
    std::shared_ptr<TensorOperation> operation2 =
      std::make_shared<vision::SoftDvppDecodeRandomCropResizeJpegOperation>(size, scale, ratio, 2);
    std::shared_ptr<TensorOperation> operation3 =
      std::make_shared<vision::RotateOperation>(0.5, interpolation, true, center, fill_value);
    operations.push_back(operation2);
    operations.push_back(operation3);
    ds = std::make_shared<MapNode>(ds, operations);
    ds = std::make_shared<BatchNode>(ds, 2, true);
    compare_dataset(ds);
  }
  
  TEST_F(MindDataTestDeserialize, TestDeserializeManifest) {
    MS_LOG(INFO) << "Doing MindDataTestDeserialize-Manifest.";
    std::string data_file = "./data/dataset/testManifestData/cpp.json";
    // Add SkipFirstEpochSampler to pipeline - for recovery test coverage, since users cannot add this sampler explicitly
    std::shared_ptr<SamplerObj> sampler = std::make_shared<SequentialSamplerObj>(0, 10);
    std::shared_ptr<SamplerObj> child_sampler = std::make_shared<SkipFirstEpochSamplerObj>(0);
    sampler->AddChildSampler(child_sampler);
    std::map<std::string, int32_t> class_indexing = {};
    std::shared_ptr<DatasetCache> cache = nullptr;
    std::shared_ptr<DatasetNode> ds =
      std::make_shared<ManifestNode>(data_file, "train", sampler, class_indexing, false, cache);
    std::vector<int32_t> coordinates = {50, 50};
    std::vector<int32_t> size = {224, 224};
    std::shared_ptr<TensorOperation> operation1 = std::make_shared<vision::CropOperation>(coordinates, size);
    std::shared_ptr<TensorOperation> operation2 = std::make_shared<vision::RgbToBgrOperation>();
    std::shared_ptr<TensorOperation> operation3 = std::make_shared<vision::RgbToGrayOperation>();
    std::shared_ptr<TensorOperation> operation4 =
      std::make_shared<vision::SlicePatchesOperation>(5, 5, SliceMode::kDrop, 1);
    std::shared_ptr<TensorOperation> operation5 = std::make_shared<vision::VerticalFlipOperation>();
    std::vector<std::shared_ptr<TensorOperation>> operations;
    operations.push_back(operation1);
    operations.push_back(operation2);
    operations.push_back(operation3);
    operations.push_back(operation4);
    operations.push_back(operation5);
    ds = std::make_shared<MapNode>(ds, operations);
    ds = std::make_shared<BatchNode>(ds, 2, false);
    compare_dataset(ds);
  }
  
  TEST_F(MindDataTestDeserialize, TestDeserializeVOC) {
    MS_LOG(INFO) << "Doing MindDataTestDeserialize-VOC.";
    std::string dataset_dir = "./data/dataset/testVOC2012";
    std::vector<int64_t> indices = {0, 1};
    std::shared_ptr<SamplerObj> sampler = std::make_shared<SubsetRandomSamplerObj>(indices, 3);
    std::string task = "Detection";
    std::string usage = "train";
    std::map<std::string, int32_t> class_indexing = {};
    std::shared_ptr<DatasetCache> cache = nullptr;
    std::shared_ptr<DatasetNode> ds =
      std::make_shared<VOCNode>(dataset_dir, task, usage, class_indexing, true, sampler, cache);
    std::vector<float> brightness = {0.5, 0.5};
    std::vector<float> contrast = {1.0, 1.0};
    std::vector<float> hue = {0.0, 0.0};
    std::vector<float> saturation = {1.0, 1.0};
    std::shared_ptr<TensorOperation> operation =
      std::make_shared<vision::RandomColorAdjustOperation>(brightness, contrast, saturation, hue);
    std::vector<std::shared_ptr<TensorOperation>> ops = {operation};
    ds = std::make_shared<MapNode>(ds, ops);
    ds = std::make_shared<SkipNode>(ds, 2);
    compare_dataset(ds);
  }
  
  TEST_F(MindDataTestDeserialize, TestDeserializeCLUE) {
    MS_LOG(INFO) << "Doing MindDataTestDeserialize-CLUE.";
    std::string train_file = "./data/dataset/testCLUE/afqmc/train.json";
    std::string task = "AFQMC";
    std::string usage = "train";
    std::vector<std::string> files = {train_file};
    std::shared_ptr<DatasetCache> cache = nullptr;
    std::shared_ptr<DatasetNode> ds = std::make_shared<CLUENode>(files, task, usage, 1, ShuffleMode::kFalse, 1, 0, cache);
    ds = std::make_shared<RepeatNode>(ds, 1);
    std::shared_ptr<TensorOperation> operation1 = std::make_shared<vision::DecodeOperation>(true);
    std::vector<std::shared_ptr<TensorOperation>> ops = {operation1};
    ds = std::make_shared<MapNode>(ds, ops);
    compare_dataset(ds);
  }
  
  TEST_F(MindDataTestDeserialize, TestDeserializeCoco) {
    MS_LOG(INFO) << "Doing MindDataTestDeserialize-Coco.";
    std::string folder_path = "./data/dataset/testCOCO/train";
    std::string annotation_file = "./data/dataset/testCOCO/annotations/train.json";
    std::string task = "Detection";
    std::vector<int64_t> indices = {0, 1};
    std::shared_ptr<SamplerObj> sampler = std::make_shared<SubsetRandomSamplerObj>(indices, 3);
    std::shared_ptr<DatasetCache> cache = nullptr;
    std::shared_ptr<DatasetNode> ds =
      std::make_shared<CocoNode>(folder_path, annotation_file, task, true, sampler, cache, false);
    std::vector<uint8_t> fill_value = {150, 150, 150};
    std::vector<float> degrees = {0.0, 0.0};
    std::vector<float> scale = {0.5, 0.5};
    std::vector<float> ratio = {0.5, 0.5};
    std::vector<int32_t> size = {224, 224};
    std::vector<int32_t> padding = {20, 20, 20, 20};
    InterpolationMode interpolation = InterpolationMode::kLinear;
    std::shared_ptr<TensorOperation> operation1 =
      std::make_shared<vision::RandomCropDecodeResizeOperation>(size, scale, ratio, interpolation, 2);
    std::shared_ptr<TensorOperation> operation2 =
      std::make_shared<vision::RandomCropWithBBoxOperation>(size, padding, true, fill_value, BorderType::kConstant);
    std::shared_ptr<TensorOperation> operation3 = std::make_shared<vision::RandomHorizontalFlipOperation>(0.1);
    std::shared_ptr<TensorOperation> operation4 = std::make_shared<vision::RandomHorizontalFlipWithBBoxOperation>(0.1);
    std::vector<std::shared_ptr<TensorOperation>> operations;
    operations.push_back(operation1);
    operations.push_back(operation2);
    operations.push_back(operation3);
    operations.push_back(operation4);
    ds = std::make_shared<MapNode>(ds, operations);
    compare_dataset(ds);
  }
  
  TEST_F(MindDataTestDeserialize, TestDeserializeTFRecord) {
    MS_LOG(INFO) << "Doing MindDataTestDeserialize-TFRecord.";
    int num_samples = 12;
    int32_t num_shards = 1;
    int32_t shard_id = 0;
    bool shard_equal_rows = false;
    std::shared_ptr<DatasetCache> cache = nullptr;
    std::vector<std::string> columns_list = {};
    std::vector<std::string> dataset_files = {"./data/dataset/testTFTestAllTypes/test.data"};
  
    std::shared_ptr<SchemaObj> schema = Schema();
    ASSERT_OK(schema->add_column("col1", mindspore::DataType::kNumberTypeInt32, {4}));
    ASSERT_OK(schema->add_column("col2", mindspore::DataType::kNumberTypeInt64, {4}));
  
    std::shared_ptr<DatasetNode> ds =
      std::make_shared<TFRecordNode>(dataset_files, schema, columns_list, num_samples, ShuffleMode::kFiles, num_shards,
                                     shard_id, shard_equal_rows, cache);
    ds = std::make_shared<ShuffleNode>(ds, 10000, true);
    std::vector<std::string> input_columns = {"col_sint16", "col_sint32", "col_sint64", "col_float",
                                              "col_1d",     "col_2d",     "col_3d",     "col_binary"};
    std::vector<std::string> output_columns = {"column_sint16", "column_sint32", "column_sint64", "column_float",
                                               "column_1d",     "column_2d",     "column_3d",     "column_binary"};
    std::shared_ptr<TensorOperation> operation = std::make_shared<vision::InvertOperation>();
    std::vector<std::shared_ptr<TensorOperation>> ops = {operation};
    ds = std::make_shared<MapNode>(ds, ops, input_columns, output_columns);
    std::string train_file = "./data/dataset/testCLUE/afqmc/train.json";
    std::string task1 = "AFQMC";
    std::string usage = "train";
    std::vector<std::string> files = {train_file};
    std::shared_ptr<DatasetNode> ds_child1 =
      std::make_shared<CLUENode>(files, task1, usage, 0, ShuffleMode::kFalse, 1, 0, cache);
    std::vector<std::string> dataset_files2 = {"./data/dataset/testTextFileDataset/1.txt"};
    std::shared_ptr<DatasetNode> ds_child2 =
      std::make_shared<TextFileNode>(dataset_files2, 2, ShuffleMode::kFiles, 1, 0, cache);
    std::vector<std::shared_ptr<DatasetNode>> datasets = {ds, ds_child1, ds_child2};
    ds = std::make_shared<ZipNode>(datasets);
    compare_dataset(ds);
  }
  
  TEST_F(MindDataTestDeserialize, TestDeserializeTextfile) {
    MS_LOG(INFO) << "Doing MindDataTestDeserialize-Textfile.";
    std::vector<std::string> dataset_files = {"./data/dataset/testTextFileDataset/1.txt"};
    std::shared_ptr<DatasetCache> cache = nullptr;
    std::shared_ptr<DatasetNode> ds = std::make_shared<TextFileNode>(dataset_files, 2, ShuffleMode::kFiles, 1, 0, cache);
    std::shared_ptr<TensorOperation> operation = std::make_shared<vision::InvertOperation>();
    std::vector<std::shared_ptr<TensorOperation>> ops = {operation};
    ds = std::make_shared<MapNode>(ds, ops);
    ds = std::make_shared<BatchNode>(ds, 10, true);
    compare_dataset(ds);
  }
  
  TEST_F(MindDataTestDeserialize, TestDeserializeInvalidJson) {
    std::shared_ptr<DatasetNode> ds;
    // check the invalid json path would return error
    ASSERT_ERROR(Serdes::Deserialize("invalid_dataset.json", &ds));
    // check the invalid json object would return error
    ASSERT_ERROR(Serdes::Deserialize("./data/dataset/testDataset1/datasetTestInvalidJson.json", &ds));
    EXPECT_EQ(ds, nullptr);
  }
  
  TEST_F(MindDataTestDeserialize, TestDeserializeFill) {
    MS_LOG(INFO) << "Doing MindDataTestDeserialize-Fill.";
    std::vector<std::string> dataset_files = {"./data/dataset/testTextFileDataset/1.txt"};
    std::shared_ptr<DatasetCache> cache = nullptr;
    std::shared_ptr<DatasetNode> ds = std::make_shared<TextFileNode>(dataset_files, 2, ShuffleMode::kFiles, 1, 0, cache);
    std::shared_ptr<Tensor> fill_value;
    ASSERT_OK(Tensor::CreateScalar(true, &fill_value));
    std::shared_ptr<TensorOperation> operation1 = std::make_shared<transforms::FillOperation>(fill_value);
    std::shared_ptr<TensorOperation> operation2 = std::make_shared<text::ToNumberOperation>("int32_t");
    std::vector<std::shared_ptr<TensorOperation>> ops = {operation1, operation2};
    ds = std::make_shared<MapNode>(ds, ops);
    ds = std::make_shared<TransferNode>(ds, "queue", "type", 1, true, 10, true);
    compare_dataset(ds);
  }
  
  TEST_F(MindDataTestDeserialize, TestDeserializeTensor) {
    MS_LOG(INFO) << "Doing MindDataTestDeserialize-Tensor.";
    std::shared_ptr<Tensor> test_tensor;
    std::vector<float> input = {1.1, 0.2, 0.3, 0.4, 0.5, 0.6, 1.2, 0.7, 0.8, 0.9, 1.0, 2.0, 1.3, 3.0, 4.0};
    ASSERT_OK(Tensor::CreateFromVector(input, TensorShape{3, 5}, &test_tensor));
    nlohmann::json json_obj;
    ASSERT_OK(test_tensor->to_json(&json_obj));
    std::shared_ptr<Tensor> test_tensor1;
    ASSERT_OK(Tensor::from_json(json_obj, &test_tensor1));
    nlohmann::json json_obj1;
    ASSERT_OK(test_tensor1->to_json(&json_obj1));
    std::stringstream json_ss;
    json_ss << json_obj;
    std::stringstream json_ss1;
    json_ss1 << json_obj1;
    EXPECT_EQ(json_ss.str(), json_ss1.str());
  }
  
  // Helper function to get the session id from SESSION_ID env variable
  Status GetSessionFromEnv(session_id_type *session_id);
  
  TEST_F(MindDataTestDeserialize, DISABLED_TestDeserializeCache) {
    MS_LOG(INFO) << "Doing MindDataTestDeserialize-Cache.";
    std::string data_dir = "./data/dataset/testCache";
    std::string usage = "all";
    session_id_type env_session;
    ASSERT_TRUE(GetSessionFromEnv(&env_session));
    std::shared_ptr<DatasetCache> some_cache = CreateDatasetCache(env_session, 0, false, "127.0.0.1", 50052, 1, 1);
  
    std::shared_ptr<SamplerObj> sampler = std::make_shared<SequentialSamplerObj>(0, 10);
    std::shared_ptr<DatasetNode> ds = std::make_shared<Cifar10Node>(data_dir, usage, sampler, some_cache);
    compare_dataset(ds);
  }
  
  TEST_F(MindDataTestDeserialize, TestDeserializeConcatAlbumFlickr) {
    MS_LOG(INFO) << "Doing MindDataTestDeserialize-ConcatAlbumFlickr.";
    std::string dataset_dir = "./data/dataset/testAlbum";
    std::vector<std::string> column_names = {"col1", "col2", "col3"};
    bool decode = false;
    std::shared_ptr<SamplerObj> sampler = std::make_shared<SequentialSamplerObj>(0, 10);
    std::string data_schema = "./data/dataset/testAlbum/datasetSchema.json";
    std::shared_ptr<DatasetNode> ds =
      std::make_shared<AlbumNode>(dataset_dir, data_schema, column_names, decode, sampler, nullptr);
    std::shared_ptr<TensorOperation> operation = std::make_shared<vision::AdjustGammaOperation>(0.5, 0.5);
    std::vector<std::shared_ptr<TensorOperation>> ops = {operation};
    ds = std::make_shared<MapNode>(ds, ops);
    std::string dataset_path = "./data/dataset/testFlickrData/flickr30k/flickr30k-images";
    std::string annotation_file = "./data/dataset/testFlickrData/flickr30k/test1.token";
    std::shared_ptr<DatasetNode> ds_child1 =
      std::make_shared<FlickrNode>(dataset_path, annotation_file, decode, sampler, nullptr);
    std::vector<std::shared_ptr<DatasetNode>> datasets = {ds, ds_child1};
    std::pair<int, int> pair = std::make_pair(1, 1);
    std::vector<std::pair<int, int>> children_flag_and_nums = {pair};
    std::vector<std::pair<int, int>> children_start_end_index = {pair};
    ds = std::make_shared<ConcatNode>(datasets, sampler, children_flag_and_nums, children_start_end_index);
    compare_dataset(ds);
  }
  
  TEST_F(MindDataTestDeserialize, TestDeserializePyFunc) {
    MS_LOG(INFO) << "Doing MindDataTestDeserialize-PyFunc.";
    if (Py_IsInitialized() != 0) {
      std::shared_ptr<DatasetNode> ds1;
      ASSERT_OK(Serdes::Deserialize("./data/dataset/tf_file_dataset/pyvision_dataset_pipeline.json", &ds1));
      EXPECT_NE(ds1, nullptr);
    }
  }
  