code: # Copyright 2021 Huawei Technologies Co., Ltd
  #
  # Licensed under the Apache License, Version 2.0 (the "License");
  # you may not use this file except in compliance with the License.
  # You may obtain a copy of the License at
  #
  # http://www.apache.org/licenses/LICENSE-2.0
  #
  # Unless required by applicable law or agreed to in writing, software
  # distributed under the License is distributed on an "AS IS" BASIS,
  # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  # See the License for the specific language governing permissions and
  # limitations under the License.
  # ============================================================================
  '''dataloader'''
  import os
  import glob
  import numpy as np
  import PIL.Image as Image
  import mindspore.dataset as ds
  import mindspore.dataset.vision.c_transforms as CV
  
  class DataLoader_Imagenet_val:
      '''DataLoader_Imagenet_val'''
      def __init__(self, data_dir, patch=256, noise_style="gauss25", batch_size=4):
          super(DataLoader_Imagenet_val, self).__init__()
          self.data_dir = data_dir
          self.patch = patch
          self.train_fns = glob.glob(os.path.join(self.data_dir, "*"))
          self.train_fns.sort()
          print('fetch {} samples for training'.format(len(self.train_fns)))
          self.noise_generator = AugmentNoise(noise_style)
          self.batch_size = batch_size
          self.test = 1
      def __getitem__(self, index):
          # fetch image
          fn = self.train_fns[index]
          im = Image.open(fn)
          im = np.array(im, dtype=np.float32)
          # random crop
          H = im.shape[0]
          W = im.shape[1]
          if H - self.patch > 0:
              xx = np.random.randint(0, H - self.patch)
              im = im[xx:xx + self.patch, :, :]
          if W - self.patch > 0:
              yy = np.random.randint(0, W - self.patch)
              im = im[:, yy:yy + self.patch, :]
          im /= 255.0 #clean image
          noisy = self.noise_generator.add_noise(im)
  
          return im, noisy
  
      def __len__(self):
          return len(self.train_fns)
  
  class AugmentNoise():
      '''AugmentNoise'''
      def __init__(self, style):
          if style.startswith('gauss'):
              self.params = [
                  float(p) / 255.0 for p in style.replace('gauss', '').split('_')
              ]
              if len(self.params) == 1:
                  self.style = "gauss_fix"
              elif len(self.params) == 2:
                  self.style = "gauss_range"
          elif style.startswith('poisson'):
              self.params = [
                  float(p) for p in style.replace('poisson', '').split('_')
              ]
              if len(self.params) == 1:
                  self.style = "poisson_fix"
              elif len(self.params) == 2:
                  self.style = "poisson_range"
  
      def add_noise(self, x):
          '''add_noise'''
          shape = x.shape
          if self.style == "gauss_fix":
              std = self.params[0]
              return np.array(x + np.random.normal(size=shape) * std,
                              dtype=np.float32)
          if self.style == "gauss_range":
              min_std, max_std = self.params
              std = np.random.uniform(low=min_std, high=max_std, size=(1, 1, 1))
              return np.array(x + np.random.normal(size=shape) * std,
                              dtype=np.float32)
          if self.style == "poisson_fix":
              lam = self.params[0]
              return np.array(np.random.poisson(lam * x) / lam, dtype=np.float32)
          assert self.style == "poisson_range"
          min_lam, max_lam = self.params
          lam = np.random.uniform(low=min_lam, high=max_lam, size=(1, 1, 1))
          return np.array(np.random.poisson(lam * x) / lam, dtype=np.float32)
  
  
  def create_Dataset(data_dir, patch, noise_style, batch_size, device_num, rank, shuffle):
  
      dataset = DataLoader_Imagenet_val(data_dir, patch, noise_style, batch_size)
      hwc_to_chw = CV.HWC2CHW()
      data_set = ds.GeneratorDataset(dataset, column_names=["image", "noisy"], \
                 num_parallel_workers=8, shuffle=shuffle, num_shards=device_num, shard_id=rank)
      data_set = data_set.map(input_columns=["image"], operations=hwc_to_chw, num_parallel_workers=8)
      data_set = data_set.map(input_columns=["noisy"], operations=hwc_to_chw, num_parallel_workers=8)
      data_set = data_set.batch(batch_size, drop_remainder=True)
      return data_set, data_set.get_dataset_size()
   # Copyright 2020 Huawei Technologies Co., Ltd
  #
  # Licensed under the Apache License, Version 2.0 (the "License");
  # you may not use this file except in compliance with the License.
  # You may obtain a copy of the License at
  #
  # http://www.apache.org/licenses/LICENSE-2.0
  #
  # Unless required by applicable law or agreed to in writing, software
  # distributed under the License is distributed on an "AS IS" BASIS,
  # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  # See the License for the specific language governing permissions and
  # limitations under the License.
  # ============================================================================
  """Dataset loading, creation and processing"""
  import logging
  import math
  import os
  import time
  import timeit
  import pickle
  import numpy as np
  import pandas as pd
  
  from mindspore.dataset import GeneratorDataset, Sampler
  
  import src.constants as rconst
  import src.movielens as movielens
  import src.stat_utils as stat_utils
  
  DATASET_TO_NUM_USERS_AND_ITEMS = {
      "ml-1m": (6040, 3706),
      "ml-20m": (138493, 26744)
  }
  
  _EXPECTED_CACHE_KEYS = (
      rconst.TRAIN_USER_KEY, rconst.TRAIN_ITEM_KEY, rconst.EVAL_USER_KEY,
      rconst.EVAL_ITEM_KEY, rconst.USER_MAP, rconst.ITEM_MAP)
  
  
  def load_data(data_dir, dataset):
      """
      Load data in .csv format and output structured data.
  
      This function reads in the raw CSV of positive items, and performs three
      preprocessing transformations:
  
      1)  Filter out all users who have not rated at least a certain number
        of items. (Typically 20 items)
  
      2)  Zero index the users and items such that the largest user_id is
        `num_users - 1` and the largest item_id is `num_items - 1`
  
      3)  Sort the dataframe by user_id, with timestamp as a secondary sort key.
        This allows the dataframe to be sliced by user in-place, and for the last
        item to be selected simply by calling the `-1` index of a user's slice.
  
      While all of these transformations are performed by Pandas (and are therefore
      single-threaded), they only take ~2 minutes, and the overhead to apply a
      MapReduce pattern to parallel process the dataset adds significant complexity
      for no computational gain. For a larger dataset parallelizing this
      preprocessing could yield speedups. (Also, this preprocessing step is only
      performed once for an entire run.
      """
      logging.info("Beginning loading data...")
  
      raw_rating_path = os.path.join(data_dir, dataset, movielens.RATINGS_FILE)
      cache_path = os.path.join(data_dir, dataset, rconst.RAW_CACHE_FILE)
  
      valid_cache = os.path.exists(cache_path)
      if valid_cache:
          with open(cache_path, 'rb') as f:
              cached_data = pickle.load(f)
  
          for key in _EXPECTED_CACHE_KEYS:
              if key not in cached_data:
                  valid_cache = False
  
          if not valid_cache:
              logging.info("Removing stale raw data cache file.")
              os.remove(cache_path)
  
      if valid_cache:
          data = cached_data
      else:
          # process data and save to .csv
          with open(raw_rating_path) as f:
              df = pd.read_csv(f)
  
          # Get the info of users who have more than 20 ratings on items
          grouped = df.groupby(movielens.USER_COLUMN)
          df = grouped.filter(lambda x: len(x) >= rconst.MIN_NUM_RATINGS)
  
          original_users = df[movielens.USER_COLUMN].unique()
          original_items = df[movielens.ITEM_COLUMN].unique()
  
          # Map the ids of user and item to 0 based index for following processing
          logging.info("Generating user_map and item_map...")
          user_map = {user: index for index, user in enumerate(original_users)}
          item_map = {item: index for index, item in enumerate(original_items)}
  
          df[movielens.USER_COLUMN] = df[movielens.USER_COLUMN].apply(
              lambda user: user_map[user])
          df[movielens.ITEM_COLUMN] = df[movielens.ITEM_COLUMN].apply(
              lambda item: item_map[item])
  
          num_users = len(original_users)
          num_items = len(original_items)
  
          assert num_users <= np.iinfo(rconst.USER_DTYPE).max
          assert num_items <= np.iinfo(rconst.ITEM_DTYPE).max
          assert df[movielens.USER_COLUMN].max() == num_users - 1
          assert df[movielens.ITEM_COLUMN].max() == num_items - 1
  
          # This sort is used to shard the dataframe by user, and later to select
          # the last item for a user to be used in validation.
          logging.info("Sorting by user, timestamp...")
  
          # This sort is equivalent to
          #   df.sort_values([movielens.USER_COLUMN, movielens.TIMESTAMP_COLUMN],
          #   inplace=True)
          # except that the order of items with the same user and timestamp are
          # sometimes different. For some reason, this sort results in a better
          # hit-rate during evaluation, matching the performance of the MLPerf
          # reference implementation.
          df.sort_values(by=movielens.TIMESTAMP_COLUMN, inplace=True)
          df.sort_values([movielens.USER_COLUMN, movielens.TIMESTAMP_COLUMN],
                         inplace=True, kind="mergesort")
  
          # The dataframe does not reconstruct indices in the sort or filter steps.
          df = df.reset_index()
  
          grouped = df.groupby(movielens.USER_COLUMN, group_keys=False)
          eval_df, train_df = grouped.tail(1), grouped.apply(lambda x: x.iloc[:-1])
  
          data = {
              rconst.TRAIN_USER_KEY:
                  train_df[movielens.USER_COLUMN].values.astype(rconst.USER_DTYPE),
              rconst.TRAIN_ITEM_KEY:
                  train_df[movielens.ITEM_COLUMN].values.astype(rconst.ITEM_DTYPE),
              rconst.EVAL_USER_KEY:
                  eval_df[movielens.USER_COLUMN].values.astype(rconst.USER_DTYPE),
              rconst.EVAL_ITEM_KEY:
                  eval_df[movielens.ITEM_COLUMN].values.astype(rconst.ITEM_DTYPE),
              rconst.USER_MAP: user_map,
              rconst.ITEM_MAP: item_map,
              "create_time": time.time(),
          }
  
          logging.info("Writing raw data cache.")
          with open(cache_path, "wb") as f:
              pickle.dump(data, f, protocol=pickle.HIGHEST_PROTOCOL)
  
      num_users, num_items = DATASET_TO_NUM_USERS_AND_ITEMS[dataset]
      if num_users != len(data[rconst.USER_MAP]):
          raise ValueError("Expected to find {} users, but found {}".format(
              num_users, len(data[rconst.USER_MAP])))
      if num_items != len(data[rconst.ITEM_MAP]):
          raise ValueError("Expected to find {} items, but found {}".format(
              num_items, len(data[rconst.ITEM_MAP])))
  
      return data, num_users, num_items
  
  
  def construct_lookup_variables(train_pos_users, train_pos_items, num_users):
      """Lookup variables"""
      index_bounds = None
      sorted_train_pos_items = None
  
      def index_segment(user):
          lower, upper = index_bounds[user:user + 2]
          items = sorted_train_pos_items[lower:upper]
  
          negatives_since_last_positive = np.concatenate(
              [items[0][np.newaxis], items[1:] - items[:-1] - 1])
  
          return np.cumsum(negatives_since_last_positive)
  
      start_time = timeit.default_timer()
      inner_bounds = np.argwhere(train_pos_users[1:] -
                                 train_pos_users[:-1])[:, 0] + 1
      (upper_bound,) = train_pos_users.shape
      index_bounds = np.array([0] + inner_bounds.tolist() + [upper_bound])
  
      # Later logic will assume that the users are in sequential ascending order.
      assert np.array_equal(train_pos_users[index_bounds[:-1]], np.arange(num_users))
  
      sorted_train_pos_items = train_pos_items.copy()
  
      for i in range(num_users):
          lower, upper = index_bounds[i:i + 2]
          sorted_train_pos_items[lower:upper].sort()
  
      total_negatives = np.concatenate([
          index_segment(i) for i in range(num_users)])
  
      logging.info("Negative total vector built. Time: {:.1f} seconds".format(
          timeit.default_timer() - start_time))
  
      return total_negatives, index_bounds, sorted_train_pos_items
  
  
  class NCFDataset:
      """
      A dataset for NCF network.
      """
  
      def __init__(self,
                   pos_users,
                   pos_items,
                   num_users,
                   num_items,
                   batch_size,
                   total_negatives,
                   index_bounds,
                   sorted_train_pos_items,
                   num_neg,
                   is_training=True):
          self._pos_users = pos_users
          self._pos_items = pos_items
          self._num_users = num_users
          self._num_items = num_items
  
          self._batch_size = batch_size
  
          self._total_negatives = total_negatives
          self._index_bounds = index_bounds
          self._sorted_train_pos_items = sorted_train_pos_items
  
          self._is_training = is_training
  
          if self._is_training:
              self._train_pos_count = self._pos_users.shape[0]
          else:
              self._eval_users_per_batch = int(
                  batch_size // (1 + rconst.NUM_EVAL_NEGATIVES))
  
          _pos_count = pos_users.shape[0]
          _num_samples = (1 + num_neg) * _pos_count
          self.dataset_len = math.ceil(_num_samples / batch_size)
  
      def lookup_negative_items(self, negative_users):
          """Lookup negative items"""
          output = np.zeros(shape=negative_users.shape, dtype=rconst.ITEM_DTYPE) - 1
  
          left_index = self._index_bounds[negative_users]
          right_index = self._index_bounds[negative_users + 1] - 1
  
          num_positives = right_index - left_index + 1
          num_negatives = self._num_items - num_positives
          neg_item_choice = stat_utils.very_slightly_biased_randint(num_negatives)
  
          # Shortcuts:
          # For points where the negative is greater than or equal to the tally before
          # the last positive point there is no need to bisect. Instead the item id
          # corresponding to the negative item choice is simply:
          #   last_postive_index + 1 + (neg_choice - last_negative_tally)
          # Similarly, if the selection is less than the tally at the first positive
          # then the item_id is simply the selection.
          #
          # Because MovieLens organizes popular movies into low integers (which is
          # preserved through the preprocessing), the first shortcut is very
          # efficient, allowing ~60% of samples to bypass the bisection. For the same
          # reason, the second shortcut is rarely triggered (<0.02%) and is therefore
          # not worth implementing.
          use_shortcut = neg_item_choice >= self._total_negatives[right_index]
          output[use_shortcut] = (
              self._sorted_train_pos_items[right_index] + 1 +
              (neg_item_choice - self._total_negatives[right_index])
              )[use_shortcut]
  
          if np.all(use_shortcut):
              # The bisection code is ill-posed when there are no elements.
              return output
  
          not_use_shortcut = np.logical_not(use_shortcut)
          left_index = left_index[not_use_shortcut]
          right_index = right_index[not_use_shortcut]
          neg_item_choice = neg_item_choice[not_use_shortcut]
  
          num_loops = np.max(
              np.ceil(np.log2(num_positives[not_use_shortcut])).astype(np.int32))
  
          for _ in range(num_loops):
              mid_index = (left_index + right_index) // 2
              right_criteria = self._total_negatives[mid_index] > neg_item_choice
              left_criteria = np.logical_not(right_criteria)
  
              right_index[right_criteria] = mid_index[right_criteria]
              left_index[left_criteria] = mid_index[left_criteria]
  
          # Expected state after bisection pass:
          #   The right index is the smallest index whose tally is greater than the
          #   negative item choice index.
  
          assert np.all((right_index - left_index) <= 1)
  
          output[not_use_shortcut] = (
              self._sorted_train_pos_items[right_index] - (self._total_negatives[right_index] - neg_item_choice)
              )
  
          assert np.all(output >= 0)
  
          return output
  
      def _get_train_item(self, index):
          """Get train item"""
          (mask_start_index,) = index.shape
          index_mod = np.mod(index, self._train_pos_count)
  
          # get batch of users
          users = self._pos_users[index_mod]
  
          # get batch of items
          negative_indices = np.greater_equal(index, self._train_pos_count)
          negative_users = users[negative_indices]
          negative_items = self.lookup_negative_items(negative_users=negative_users)
          items = self._pos_items[index_mod]
          items[negative_indices] = negative_items
  
          # get batch of labels
          labels = np.logical_not(negative_indices)
  
          # pad last partial batch
          pad_length = self._batch_size - index.shape[0]
          if pad_length:
              user_pad = np.arange(pad_length, dtype=users.dtype) % self._num_users
              item_pad = np.arange(pad_length, dtype=items.dtype) % self._num_items
              label_pad = np.zeros(shape=(pad_length,), dtype=labels.dtype)
              users = np.concatenate([users, user_pad])
              items = np.concatenate([items, item_pad])
              labels = np.concatenate([labels, label_pad])
  
          users = np.reshape(users, (self._batch_size, 1))  # (_batch_size, 1), int32
          items = np.reshape(items, (self._batch_size, 1))  # (_batch_size, 1), int32
          mask_start_index = np.array(mask_start_index, dtype=np.int32)  # (_batch_size, 1), int32
          valid_pt_mask = np.expand_dims(
              np.less(np.arange(self._batch_size), mask_start_index), -1).astype(np.float32)  # (_batch_size, 1), bool
          labels = np.reshape(labels, (self._batch_size, 1)).astype(np.int32)  # (_batch_size, 1), bool
  
          return users, items, labels, valid_pt_mask
  
      @staticmethod
      def _assemble_eval_batch(users, positive_items, negative_items,
                               users_per_batch):
          """Construct duplicate_mask and structure data accordingly.
  
          The positive items should be last so that they lose ties. However, they
          should not be masked out if the true eval positive happens to be
          selected as a negative. So instead, the positive is placed in the first
          position, and then switched with the last element after the duplicate
          mask has been computed.
  
          Args:
            users: An array of users in a batch. (should be identical along axis 1)
            positive_items: An array (batch_size x 1) of positive item indices.
            negative_items: An array of negative item indices.
            users_per_batch: How many users should be in the batch. This is passed
              as an argument so that ncf_test.py can use this method.
  
          Returns:
            User, item, and duplicate_mask arrays.
          """
          items = np.concatenate([positive_items, negative_items], axis=1)
  
          # We pad the users and items here so that the duplicate mask calculation
          # will include padding. The metric function relies on all padded elements
          # except the positive being marked as duplicate to mask out padded points.
          if users.shape[0] < users_per_batch:
              pad_rows = users_per_batch - users.shape[0]
              padding = np.zeros(shape=(pad_rows, users.shape[1]), dtype=np.int32)
              users = np.concatenate([users, padding.astype(users.dtype)], axis=0)
              items = np.concatenate([items, padding.astype(items.dtype)], axis=0)
  
          duplicate_mask = stat_utils.mask_duplicates(items, axis=1).astype(np.float32)
  
          items[:, (0, -1)] = items[:, (-1, 0)]
          duplicate_mask[:, (0, -1)] = duplicate_mask[:, (-1, 0)]
  
          assert users.shape == items.shape == duplicate_mask.shape
          return users, items, duplicate_mask
  
      def _get_eval_item(self, index):
          """Get eval item"""
          low_index, high_index = index
          users = np.repeat(self._pos_users[low_index:high_index, np.newaxis],
                            1 + rconst.NUM_EVAL_NEGATIVES, axis=1)
          positive_items = self._pos_items[low_index:high_index, np.newaxis]
          negative_items = (self.lookup_negative_items(negative_users=users[:, :-1])
                            .reshape(-1, rconst.NUM_EVAL_NEGATIVES))
  
          users, items, duplicate_mask = self._assemble_eval_batch(
              users, positive_items, negative_items, self._eval_users_per_batch)
  
          users = np.reshape(users.flatten(), (self._batch_size, 1))  # (self._batch_size, 1), int32
          items = np.reshape(items.flatten(), (self._batch_size, 1))  # (self._batch_size, 1), int32
          duplicate_mask = np.reshape(duplicate_mask.flatten(), (self._batch_size, 1))  # (self._batch_size, 1), bool
  
          return users, items, duplicate_mask
  
      def __getitem__(self, index):
          """
          Get a batch of samples.
          """
          if self._is_training:
              return self._get_train_item(index)
  
          return self._get_eval_item(index)
  
      def __len__(self):
          """
          Return length of the dataset, i.e., the number of batches for an epoch
          """
          return self.dataset_len
  
  
  class RandomSampler(Sampler):
      """
      A random sampler for dataset.
      """
  
      def __init__(self, pos_count, num_train_negatives, batch_size):
          self.pos_count = pos_count
          self._num_samples = (1 + num_train_negatives) * self.pos_count
          self._batch_size = batch_size
          self._num_batches = math.ceil(self._num_samples / self._batch_size)
          super().__init__(self._num_batches)
  
      def __iter__(self):
          """
          Return indices of all batches within an epoch.
          """
          indices = stat_utils.permutation((self._num_samples, stat_utils.random_int32()))
  
          batch_indices = [indices[x * self._batch_size:(x + 1) * self._batch_size] for x in range(self._num_batches)]
  
          # padding last batch indices if necessary
          if len(batch_indices) > 2 and len(batch_indices[-2]) != len(batch_indices[-1]):
              pad_nums = len(batch_indices[-2]) - len(batch_indices[-1])
              pad_indices = np.random.randint(0, self._num_samples, pad_nums)
              batch_indices[-1] = np.hstack((batch_indices[-1], pad_indices))
  
          return iter(batch_indices)
  
  
  class DistributedSamplerOfTrain:
      """
      A distributed sampler for dataset.
      """
  
      def __init__(self, pos_count, num_train_negatives, batch_size, rank_id, rank_size):
          """
          Distributed sampler of training dataset.
          """
          self._num_samples = (1 + num_train_negatives) * pos_count
          self._rank_id = rank_id
          self._rank_size = rank_size
          self._batch_size = batch_size
  
          self._batchs_per_rank = int(math.ceil(self._num_samples / self._batch_size / rank_size))
          self._samples_per_rank = int(math.ceil(self._batchs_per_rank * self._batch_size))
          self._total_num_samples = self._samples_per_rank * self._rank_size
  
      def __iter__(self):
          """
          Returns the data after each sampling.
          """
          indices = stat_utils.permutation((self._num_samples, stat_utils.random_int32()))
          indices = indices.tolist()
          indices.extend(indices[:self._total_num_samples - len(indices)])
          indices = indices[self._rank_id:self._total_num_samples:self._rank_size]
          batch_indices = [indices[x * self._batch_size:(x + 1) * self._batch_size] for x in range(self._batchs_per_rank)]
  
          return iter(np.array(batch_indices))
  
      def __len__(self):
          """
          Returns the length after each sampling.
          """
          return self._batchs_per_rank
  
  
  class SequenceSampler(Sampler):
      """
      A sequence sampler for dataset.
      """
  
      def __init__(self, eval_batch_size, num_users):
          self._eval_users_per_batch = int(
              eval_batch_size // (1 + rconst.NUM_EVAL_NEGATIVES))
          self._eval_elements_in_epoch = num_users * (1 + rconst.NUM_EVAL_NEGATIVES)
          self._eval_batches_per_epoch = self.count_batches(
              self._eval_elements_in_epoch, eval_batch_size)
          super().__init__(self._eval_batches_per_epoch)
  
      def __iter__(self):
          indices = [(x * self._eval_users_per_batch, (x + 1) * self._eval_users_per_batch)
                     for x in range(self._eval_batches_per_epoch)]
  
          # padding last batch indices if necessary
          if len(indices) > 2 and len(indices[-2]) != len(indices[-1]):
              pad_nums = len(indices[-2]) - len(indices[-1])
              pad_indices = np.random.randint(0, self._eval_elements_in_epoch, pad_nums)
              indices[-1] = np.hstack((indices[-1], pad_indices))
  
          return iter(indices)
  
      @staticmethod
      def count_batches(example_count, batch_size, batches_per_step=1):
          """Determine the number of batches, rounding up to fill all devices."""
          x = (example_count + batch_size - 1) // batch_size
          return (x + batches_per_step - 1) // batches_per_step * batches_per_step
  
  
  class DistributedSamplerOfEval:
      """
      A distributed sampler for eval dataset.
      """
  
      def __init__(self, eval_batch_size, num_users, rank_id, rank_size):
          self._eval_users_per_batch = int(
              eval_batch_size // (1 + rconst.NUM_EVAL_NEGATIVES))
          self._eval_elements_in_epoch = num_users * (1 + rconst.NUM_EVAL_NEGATIVES)
          self._eval_batches_per_epoch = self.count_batches(
              self._eval_elements_in_epoch, eval_batch_size)
  
          self._rank_id = rank_id
          self._rank_size = rank_size
          self._eval_batch_size = eval_batch_size
  
          self._batchs_per_rank = int(math.ceil(self._eval_batches_per_epoch / rank_size))
  
      def __iter__(self):
          indices = [(x * self._eval_users_per_batch, (x + self._rank_id + 1) * self._eval_users_per_batch)
                     for x in range(self._batchs_per_rank)]
  
          return iter(np.array(indices))
  
      @staticmethod
      def count_batches(example_count, batch_size, batches_per_step=1):
          """Determine the number of batches, rounding up to fill all devices."""
          x = (example_count + batch_size - 1) // batch_size
          return (x + batches_per_step - 1) // batches_per_step * batches_per_step
  
      def __len__(self):
          return self._batchs_per_rank
  
  
  def parse_eval_batch_size(eval_batch_size):
      """
      Parse eval batch size.
      """
      if eval_batch_size % (1 + rconst.NUM_EVAL_NEGATIVES):
          raise ValueError("Eval batch size {} is not divisible by {}".format(
              eval_batch_size, 1 + rconst.NUM_EVAL_NEGATIVES))
      return eval_batch_size
  
  
  def create_dataset(test_train=True, data_dir='./dataset/', dataset='ml-1m', train_epochs=14, batch_size=256,
                     eval_batch_size=160000, num_neg=4, rank_id=None, rank_size=None):
      """
      Create NCF dataset.
      """
      data, num_users, num_items = load_data(data_dir, dataset)
  
      train_pos_users = data[rconst.TRAIN_USER_KEY]
      train_pos_items = data[rconst.TRAIN_ITEM_KEY]
      eval_pos_users = data[rconst.EVAL_USER_KEY]
      eval_pos_items = data[rconst.EVAL_ITEM_KEY]
  
      total_negatives, index_bounds, sorted_train_pos_items = \
          construct_lookup_variables(train_pos_users, train_pos_items, num_users)
  
      if test_train:
          print(train_pos_users, train_pos_items, num_users, num_items, batch_size, total_negatives, index_bounds,
                sorted_train_pos_items)
          dataset = NCFDataset(train_pos_users, train_pos_items, num_users, num_items, batch_size, total_negatives,
                               index_bounds, sorted_train_pos_items, num_neg)
          sampler = RandomSampler(train_pos_users.shape[0], num_neg, batch_size)
          if rank_id is not None and rank_size is not None:
              sampler = DistributedSamplerOfTrain(train_pos_users.shape[0], num_neg, batch_size, rank_id, rank_size)
  
          ds = GeneratorDataset(dataset,
                                column_names=[movielens.USER_COLUMN,
                                              movielens.ITEM_COLUMN,
                                              "labels",
                                              rconst.VALID_POINT_MASK],
                                sampler=sampler)
  
      else:
          eval_batch_size = parse_eval_batch_size(eval_batch_size=eval_batch_size)
          dataset = NCFDataset(eval_pos_users, eval_pos_items, num_users, num_items,
                               eval_batch_size, total_negatives, index_bounds,
                               sorted_train_pos_items, num_neg, is_training=False)
          sampler = SequenceSampler(eval_batch_size, num_users)
  
          ds = GeneratorDataset(dataset,
                                column_names=[movielens.USER_COLUMN,
                                              movielens.ITEM_COLUMN,
                                              rconst.DUPLICATE_MASK],
                                sampler=sampler)
  
      repeat_count = train_epochs if test_train else train_epochs + 1
      ds = ds.repeat(repeat_count)
  
      return ds, num_users, num_items
   # Copyright 2020 Huawei Technologies Co., Ltd
  #
  # Licensed under the Apache License, Version 2.0 (the "License");
  # you may not use this file except in compliance with the License.
  # You may obtain a copy of the License at
  #
  # http://www.apache.org/licenses/LICENSE-2.0
  #
  # Unless required by applicable law or agreed to in writing, software
  # distributed under the License is distributed on an "AS IS" BASIS,
  # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  # See the License for the specific language governing permissions and
  # limitations under the License.
  # ============================================================================
  
  """Bbox utils"""
  
  import math
  import itertools as it
  import numpy as np
  from src.model_utils.config import config
  from .anchor_generator import GridAnchorGenerator
  
  
  class GeneratDefaultBoxes():
      """
      Generate Default boxes for SSD, follows the order of (W, H, archor_sizes).
      `self.default_boxes` has a shape of [archor_sizes, H, W, 4], the last dimension is [y, x, h, w].
      `self.default_boxes_tlbr` has a shape as `self.default_boxes`, the last dimension is [y1, x1, y2, x2].
      """
      def __init__(self):
          fk = config.img_shape[0] / np.array(config.steps)
          scale_rate = (config.max_scale - config.min_scale) / (len(config.num_default) - 1)
          scales = [config.min_scale + scale_rate * i for i in range(len(config.num_default))] + [1.0]
          self.default_boxes = []
          for idex, feature_size in enumerate(config.feature_size):
              sk1 = scales[idex]
              sk2 = scales[idex + 1]
              sk3 = math.sqrt(sk1 * sk2)
              if idex == 0 and not config.aspect_ratios[idex]:
                  w, h = sk1 * math.sqrt(2), sk1 / math.sqrt(2)
                  all_sizes = [(0.1, 0.1), (w, h), (h, w)]
              else:
                  all_sizes = [(sk1, sk1)]
                  for aspect_ratio in config.aspect_ratios[idex]:
                      w, h = sk1 * math.sqrt(aspect_ratio), sk1 / math.sqrt(aspect_ratio)
                      all_sizes.append((w, h))
                      all_sizes.append((h, w))
                  all_sizes.append((sk3, sk3))
  
              assert len(all_sizes) == config.num_default[idex]
  
              for i, j in it.product(range(feature_size), repeat=2):
                  for w, h in all_sizes:
                      cx, cy = (j + 0.5) / fk[idex], (i + 0.5) / fk[idex]
                      self.default_boxes.append([cy, cx, h, w])
  
          def to_tlbr(cy, cx, h, w):
              return cy - h / 2, cx - w / 2, cy + h / 2, cx + w / 2
  
          # For IoU calculation
          self.default_boxes_tlbr = np.array(tuple(to_tlbr(*i) for i in self.default_boxes), dtype='float32')
          self.default_boxes = np.array(self.default_boxes, dtype='float32')
  
  if hasattr(config, 'use_anchor_generator') and config.use_anchor_generator:
      generator = GridAnchorGenerator(config.img_shape, 4, 2, [1.0, 2.0, 0.5])
      default_boxes, default_boxes_tlbr = generator.generate_multi_levels(config.steps)
  else:
      default_boxes_tlbr = GeneratDefaultBoxes().default_boxes_tlbr
      default_boxes = GeneratDefaultBoxes().default_boxes
  y1, x1, y2, x2 = np.split(default_boxes_tlbr[:, :4], 4, axis=-1)
  vol_anchors = (x2 - x1) * (y2 - y1)
  matching_threshold = config.match_threshold
  
  
  def ssd_bboxes_encode(boxes):
      """
      Labels anchors with ground truth inputs.
  
      Args:
          boxex: ground truth with shape [N, 5], for each row, it stores [y, x, h, w, cls].
  
      Returns:
          gt_loc: location ground truth with shape [num_anchors, 4].
          gt_label: class ground truth with shape [num_anchors, 1].
          num_matched_boxes: number of positives in an image.
      """
  
      def jaccard_with_anchors(bbox):
          """Compute jaccard score a box and the anchors."""
          # Intersection bbox and volume.
          ymin = np.maximum(y1, bbox[0])
          xmin = np.maximum(x1, bbox[1])
          ymax = np.minimum(y2, bbox[2])
          xmax = np.minimum(x2, bbox[3])
          w = np.maximum(xmax - xmin, 0.)
          h = np.maximum(ymax - ymin, 0.)
  
          # Volumes.
          inter_vol = h * w
          union_vol = vol_anchors + (bbox[2] - bbox[0]) * (bbox[3] - bbox[1]) - inter_vol
          jaccard = inter_vol / union_vol
          return np.squeeze(jaccard)
  
      pre_scores = np.zeros((config.num_ssd_boxes), dtype=np.float32)
      t_boxes = np.zeros((config.num_ssd_boxes, 4), dtype=np.float32)
      t_label = np.zeros((config.num_ssd_boxes), dtype=np.int64)
      for bbox in boxes:
          label = int(bbox[4])
          scores = jaccard_with_anchors(bbox)
          idx = np.argmax(scores)
          scores[idx] = 2.0
          mask = (scores > matching_threshold)
          mask = mask & (scores > pre_scores)
          pre_scores = np.maximum(pre_scores, scores * mask)
          t_label = mask * label + (1 - mask) * t_label
          for i in range(4):
              t_boxes[:, i] = mask * bbox[i] + (1 - mask) * t_boxes[:, i]
  
      index = np.nonzero(t_label)
  
      # Transform to tlbr.
      bboxes = np.zeros((config.num_ssd_boxes, 4), dtype=np.float32)
      bboxes[:, [0, 1]] = (t_boxes[:, [0, 1]] + t_boxes[:, [2, 3]]) / 2
      bboxes[:, [2, 3]] = t_boxes[:, [2, 3]] - t_boxes[:, [0, 1]]
  
      # Encode features.
      bboxes_t = bboxes[index]
      default_boxes_t = default_boxes[index]
      bboxes_t[:, :2] = (bboxes_t[:, :2] - default_boxes_t[:, :2]) / (default_boxes_t[:, 2:] * config.prior_scaling[0])
      tmp = np.maximum(bboxes_t[:, 2:4] / default_boxes_t[:, 2:4], 0.000001)
      bboxes_t[:, 2:4] = np.log(tmp) / config.prior_scaling[1]
      bboxes[index] = bboxes_t
  
      num_match = np.array([len(np.nonzero(t_label)[0])], dtype=np.int32)
      return bboxes, t_label.astype(np.int32), num_match
  
  
  def ssd_bboxes_decode(boxes):
      """Decode predict boxes to [y, x, h, w]"""
      boxes_t = boxes.copy()
      default_boxes_t = default_boxes.copy()
      boxes_t[:, :2] = boxes_t[:, :2] * config.prior_scaling[0] * default_boxes_t[:, 2:] + default_boxes_t[:, :2]
      boxes_t[:, 2:4] = np.exp(boxes_t[:, 2:4] * config.prior_scaling[1]) * default_boxes_t[:, 2:4]
  
      bboxes = np.zeros((len(boxes_t), 4), dtype=np.float32)
  
      bboxes[:, [0, 1]] = boxes_t[:, [0, 1]] - boxes_t[:, [2, 3]] / 2
      bboxes[:, [2, 3]] = boxes_t[:, [0, 1]] + boxes_t[:, [2, 3]] / 2
  
      return np.clip(bboxes, 0, 1)
  
  
  def intersect(box_a, box_b):
      """Compute the intersect of two sets of boxes."""
      max_yx = np.minimum(box_a[:, 2:4], box_b[2:4])
      min_yx = np.maximum(box_a[:, :2], box_b[:2])
      inter = np.clip((max_yx - min_yx), a_min=0, a_max=np.inf)
      return inter[:, 0] * inter[:, 1]
  
  
  def jaccard_numpy(box_a, box_b):
      """Compute the jaccard overlap of two sets of boxes."""
      inter = intersect(box_a, box_b)
      area_a = ((box_a[:, 2] - box_a[:, 0]) *
                (box_a[:, 3] - box_a[:, 1]))
      area_b = ((box_b[2] - box_b[0]) *
                (box_b[3] - box_b[1]))
      union = area_a + area_b - inter
      return inter / union
   # Copyright 2021 Huawei Technologies Co., Ltd
  #
  # Licensed under the Apache License, Version 2.0 (the "License");
  # you may not use this file except in compliance with the License.
  # You may obtain a copy of the License at
  #
  # http://www.apache.org/licenses/LICENSE-2.0
  #
  # Unless required by applicable law or agreed to in writing, software
  # distributed under the License is distributed on an "AS IS" BASIS,
  # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  # See the License for the specific language governing permissions and
  # limitations under the License.
  # ============================================================================
  """
      postprocess
  """
  import os
  import numpy as np
  from PIL import Image
  from src.utils.config import get_args
  from mindspore import Tensor
  
  def save_image(img, img_path):
      """Save a numpy image to the disk
  
      Parameters:
          img (numpy array / Tensor): image to save.
          image_path (str): the path of the image.
      """
      if isinstance(img, Tensor):
          img = img.asnumpy()
      elif not isinstance(img, np.ndarray):
          raise ValueError("img should be Tensor or numpy array, but get {}".format(type(img)))
      img = decode_image(img)
  
      img_pil = Image.fromarray(img)
      img_pil.save(img_path + ".jpg")
  
  def decode_image(img):
      """Decode a [1, C, H, W] Tensor to image numpy array."""
      mean = 0.5 * 255
      std = 0.5 * 255
  
      return (img * std + mean).astype(np.uint8).transpose((1, 2, 0))
  
  if __name__ == '__main__':
      args = get_args()
  
      result_dir = "./result_Files"
      object_imageSize = 256
      rst_path = result_dir
  
      for i in range(len(os.listdir(rst_path))):
          file_name = os.path.join(rst_path, "Pix2Pix_data_bs" + str(args.batch_size) + '_' + str(i) + '_0.bin')
          output = np.fromfile(file_name, np.float32).reshape(3, object_imageSize, object_imageSize)
          print(output.shape)
          save_image(output, './310_infer_img' + str(i + 1))
          print("=======image", i + 1, "saved success=======")
      print("Generate images success!")
   # Copyright 2021 Huawei Technologies Co., Ltd
  #
  # Licensed under the Apache License, Version 2.0 (the "License");
  # you may not use this file except in compliance with the License.
  # You may obtain a copy of the License at
  #
  # http://www.apache.org/licenses/LICENSE-2.0
  #
  # Unless required by applicable law or agreed to in writing, software
  # distributed under the License is distributed on an "AS IS" BASIS,
  # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  # See the License for the specific language governing permissions and
  # limitations under the License.
  # ============================================================================
  """FSNS dataset"""
  
  import cv2
  import numpy as np
  from PIL import Image
  
  import mindspore.dataset as de
  import mindspore.dataset.vision.c_transforms as C
  import mindspore.dataset.vision.py_transforms as P
  import mindspore.dataset.transforms.c_transforms as ops
  import mindspore.common.dtype as mstype
  
  from src.model_utils.config import config
  
  
  class AugmentationOps():
      def __init__(self, min_area_ratio=0.8, aspect_ratio_range=(0.8, 1.2), brightness=32./255.,
                   contrast=0.5, saturation=0.5, hue=0.2, img_tile_shape=(150, 150)):
          self.min_area_ratio = min_area_ratio
          self.aspect_ratio_range = aspect_ratio_range
          self.img_tile_shape = img_tile_shape
          self.random_image_distortion_ops = P.RandomColorAdjust(brightness=brightness,
                                                                 contrast=contrast,
                                                                 saturation=saturation,
                                                                 hue=hue)
  
      def __call__(self, img):
          img_h = self.img_tile_shape[0]
          img_w = self.img_tile_shape[1]
          img_new = np.zeros([128, 512, 3], dtype=np.float32)
  
          for i in range(4):
              img_tile = img[:, (i*150):((i+1)*150), :]
              # Random crop cut from the street sign image, resized to the same size.
              # Assures that the crop covers at least 0.8 area of the input image.
              # Aspect ratio of cropped image is within [0.8,1.2] range.
              h = img_h + 1
              w = img_w + 1
  
              while (w >= img_w or h >= img_h):
                  aspect_ratio = np.random.uniform(self.aspect_ratio_range[0],
                                                   self.aspect_ratio_range[1])
                  h_low = np.ceil(np.sqrt(self.min_area_ratio * img_h * img_w / aspect_ratio))
                  h_high = np.floor(np.sqrt(img_h * img_w / aspect_ratio))
                  h = np.random.randint(h_low, h_high)
                  w = int(h * aspect_ratio)
  
              y = np.random.randint(img_w - w)
              x = np.random.randint(img_h - h)
              img_tile = img_tile[x:(x+h), y:(y+w), :]
              # Randomly chooses one of the 4 interpolation resize methods.
              interpolation = np.random.choice([cv2.INTER_LINEAR,
                                                cv2.INTER_CUBIC,
                                                cv2.INTER_AREA,
                                                cv2.INTER_NEAREST])
              img_tile = cv2.resize(img_tile, (128, 128), interpolation=interpolation)
              # Random color distortion ops.
              img_tile_pil = Image.fromarray(img_tile)
              img_tile_pil = self.random_image_distortion_ops(img_tile_pil)
              img_tile = np.array(img_tile_pil)
              img_new[:, (i*128):((i+1)*128), :] = img_tile
  
          img_new = 2 * (img_new / 255.) - 1
          return img_new
  
  
  class ImageResizeWithRescale():
      def __init__(self, standard_img_height, standard_img_width, channel_size=3):
          self.standard_img_height = standard_img_height
          self.standard_img_width = standard_img_width
          self.channel_size = channel_size
  
      def __call__(self, img):
          img = cv2.resize(img, (self.standard_img_width, self.standard_img_height))
          img = 2 * (img / 255.) - 1
          return img
  
  
  def random_teacher_force(images, source_ids, target_ids):
      teacher_force = np.random.random() < config.teacher_force_ratio
      teacher_force_array = np.array([teacher_force], dtype=bool)
      return images, source_ids, target_ids, teacher_force_array
  
  
  def create_ocr_train_dataset(mindrecord_file, batch_size=32, rank_size=1, rank_id=0,
                               is_training=True, num_parallel_workers=4, use_multiprocessing=True):
      ds = de.MindDataset(mindrecord_file,
                          columns_list=["image", "decoder_input", "decoder_target"],
                          num_shards=rank_size,
                          shard_id=rank_id,
                          num_parallel_workers=num_parallel_workers,
                          shuffle=is_training)
      aug_ops = AugmentationOps()
      transforms = [C.Decode(),
                    aug_ops,
                    C.HWC2CHW()]
      ds = ds.map(operations=transforms, input_columns=["image"], python_multiprocessing=use_multiprocessing,
                  num_parallel_workers=num_parallel_workers)
      ds = ds.map(operations=ops.PadEnd([config.max_length], 0), input_columns=["decoder_target"])
      ds = ds.map(operations=random_teacher_force, input_columns=["image", "decoder_input", "decoder_target"],
                  output_columns=["image", "decoder_input", "decoder_target", "teacher_force"],
                  column_order=["image", "decoder_input", "decoder_target", "teacher_force"])
      type_cast_op_bool = ops.TypeCast(mstype.bool_)
      ds = ds.map(operations=type_cast_op_bool, input_columns="teacher_force")
      print("Train dataset size= %s" % (int(ds.get_dataset_size())))
      ds = ds.batch(batch_size, drop_remainder=True)
      return ds
  
  
  def create_ocr_val_dataset(mindrecord_file, batch_size=32, rank_size=1, rank_id=0,
                             num_parallel_workers=4, use_multiprocessing=True):
      ds = de.MindDataset(mindrecord_file,
                          columns_list=["image", "annotation", "decoder_input", "decoder_target"],
                          num_shards=rank_size,
                          shard_id=rank_id,
                          num_parallel_workers=num_parallel_workers,
                          shuffle=False)
      resize_rescale_op = ImageResizeWithRescale(standard_img_height=128, standard_img_width=512)
      transforms = [C.Decode(),
                    resize_rescale_op,
                    C.HWC2CHW()]
      ds = ds.map(operations=transforms, input_columns=["image"], python_multiprocessing=use_multiprocessing,
                  num_parallel_workers=num_parallel_workers)
      ds = ds.map(operations=ops.PadEnd([config.max_length], 0), input_columns=["decoder_target"],
                  python_multiprocessing=use_multiprocessing, num_parallel_workers=8)
      ds = ds.map(operations=ops.PadEnd([config.max_length], 0), input_columns=["decoder_input"],
                  python_multiprocessing=use_multiprocessing, num_parallel_workers=8)
      ds = ds.batch(batch_size, drop_remainder=True)
      print("Val dataset size= %s" % (str(int(ds.get_dataset_size())*batch_size)))
      return ds
   # Copyright 2021 Huawei Technologies Co., Ltd
  #
  # Licensed under the Apache License, Version 2.0 (the "License");
  # you may not use this file except in compliance with the License.
  # You may obtain a copy of the License at
  #
  # http://www.apache.org/licenses/LICENSE-2.0
  #
  # Unless required by applicable law or agreed to in writing, software
  # distributed under the License is distributed on an "AS IS" BASIS,
  # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  # See the License for the specific language governing permissions and
  # limitations under the License.
  # ============================================================================
  """Monitor loss and time"""
  import time
  import numpy as np
  from mindspore import Tensor
  from mindspore.train.callback import Callback
  class Monitor(Callback):
      """
      Monitor loss and time.
  
      Args:
          lr_init (numpy array): train lr
  
      Returns:
          None
  
      Examples:
          >>> Monitor(100,lr_init=Tensor([0.05]*100).asnumpy())
      """
  
      def __init__(self, lr_init=None):
          super(Monitor, self).__init__()
          self.lr_init = lr_init
          self.lr_init_len = len(lr_init)
  
      def epoch_begin(self, run_context):
          self.losses = []
          self.epoch_time = time.time()
  
      def epoch_end(self, run_context):
          cb_params = run_context.original_args()
  
          epoch_mseconds = (time.time() - self.epoch_time) * 1000
          per_step_mseconds = epoch_mseconds / cb_params.batch_num
          print("epoch time: {:5.3f}, per step time: {:5.3f}, avg loss: {:5.3f}".format(epoch_mseconds,
                                                                                        per_step_mseconds,
                                                                                        np.mean(self.losses)))
  
      def step_begin(self, run_context):
          self.step_time = time.time()
  
      def step_end(self, run_context):
          """step end"""
          cb_params = run_context.original_args()
          step_mseconds = (time.time() - self.step_time) * 1000
          step_loss = cb_params.net_outputs
  
          if isinstance(step_loss, (tuple, list)) and isinstance(step_loss[0], Tensor):
              step_loss = step_loss[0]
          if isinstance(step_loss, Tensor):
              step_loss = np.mean(step_loss.asnumpy())
  
          self.losses.append(step_loss)
          cur_step_in_epoch = (cb_params.cur_step_num - 1) % cb_params.batch_num
  
          print("epoch: [{:3d}/{:3d}], step:[{:5d}/{:5d}], loss:[{:5.3f}/{:5.3f}], time:[{:5.3f}], lr:[{:5.3f}]".format(
              cb_params.cur_epoch_num -
              1, cb_params.epoch_num, cur_step_in_epoch, cb_params.batch_num, step_loss,
              np.mean(self.losses), step_mseconds, self.lr_init[cb_params.cur_step_num - 1]))
   # Copyright 2021 Huawei Technologies Co., Ltd
  #
  # Licensed under the Apache License, Version 2.0 (the "License");
  # you may not use this file except in compliance with the License.
  # You may obtain a copy of the License at
  #
  # http://www.apache.org/licenses/LICENSE-2.0
  #
  # Unless required by applicable law or agreed to in writing, software
  # distributed under the License is distributed on an "AS IS" BASIS,
  # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
  # See the License for the specific language governing permissions and
  # limitations under the License.
  # ============================================================================
  
  import os
  import numpy as np
  from mindspore import dtype as mstype
  from mindspore import Model, context, Tensor
  from mindspore.train.serialization import load_checkpoint, load_param_into_net
  from src.dataset import create_dataset
  from src.unet3d_model import UNet3d, UNet3d_
  from src.utils import create_sliding_window, CalculateDice
  from src.model_utils.config import config
  from src.model_utils.moxing_adapter import moxing_wrapper
  
  device_id = int(os.getenv('DEVICE_ID'))
  context.set_context(mode=context.GRAPH_MODE, device_target=config.device_target, save_graphs=False, device_id=device_id)
  
  @moxing_wrapper()
  def test_net(data_path, ckpt_path):
      data_dir = data_path + "/image/"
      seg_dir = data_path + "/seg/"
      eval_dataset = create_dataset(data_path=data_dir, seg_path=seg_dir, is_training=False)
      eval_data_size = eval_dataset.get_dataset_size()
      print("train dataset length is:", eval_data_size)
  
      if config.device_target == 'Ascend':
          network = UNet3d()
      else:
          network = UNet3d_()
      network.set_train(False)
      param_dict = load_checkpoint(ckpt_path)
      load_param_into_net(network, param_dict)
      model = Model(network)
      index = 0
      total_dice = 0
      for batch in eval_dataset.create_dict_iterator(num_epochs=1, output_numpy=True):
          image = batch["image"]
          seg = batch["seg"]
          print("current image shape is {}".format(image.shape), flush=True)
          sliding_window_list, slice_list = create_sliding_window(image, config.roi_size, config.overlap)
          image_size = (config.batch_size, config.num_classes) + image.shape[2:]
          output_image = np.zeros(image_size, np.float32)
          count_map = np.zeros(image_size, np.float32)
          importance_map = np.ones(config.roi_size, np.float32)
          for window, slice_ in zip(sliding_window_list, slice_list):
              window_image = Tensor(window, mstype.float32)
              pred_probs = model.predict(window_image)
              output_image[slice_] += pred_probs.asnumpy()
              count_map[slice_] += importance_map
          output_image = output_image / count_map
          dice, _ = CalculateDice(output_image, seg)
          print("The {} batch dice is {}".format(index, dice), flush=True)
          total_dice += dice
          index = index + 1
      avg_dice = total_dice / eval_data_size
      print("**********************End Eval***************************************")
      print("eval average dice is {}".format(avg_dice))
  
  if __name__ == '__main__':
      test_net(data_path=config.data_path,
               ckpt_path=config.checkpoint_file_path)
  
apis: numpy.random.uniform numpy.random.normal numpy.random.poisson numpy.array numpy.random.randint
apis: numpy.log2 numpy.cumsum numpy.argwhere numpy.zeros pandas.read_csv numpy.reshape numpy.repeat numpy.mod numpy.arange numpy.logical_not numpy.greater_equal numpy.all numpy.hstack numpy.iinfo numpy.array numpy.concatenate numpy.random.randint
apis: numpy.array numpy.zeros numpy.squeeze numpy.argmax numpy.exp numpy.clip numpy.log numpy.maximum numpy.minimum numpy.nonzero numpy.split
apis: numpy.fromfile
apis: numpy.random.uniform numpy.sqrt numpy.zeros numpy.random.choice numpy.random.random numpy.array numpy.random.randint
apis: numpy.mean
apis: numpy.ones numpy.zeros