code: """
   Copyright (C) 2021-2022 Intel Corporation
  
   Licensed under the Apache License, Version 2.0 (the "License");
   you may not use this file except in compliance with the License.
   You may obtain a copy of the License at
  
        http://www.apache.org/licenses/LICENSE-2.0
  
   Unless required by applicable law or agreed to in writing, software
   distributed under the License is distributed on an "AS IS" BASIS,
   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
   See the License for the specific language governing permissions and
   limitations under the License.
  """
  
  import cv2
  import numpy as np
  import logging as log
  from pathlib import Path
  from scipy.special import softmax
  from openvino.runtime import PartialShape
  
  class SegmentorMstcn:
      def __init__(self, ie, device, i3d_path, mstcn_path):
          self.ActionTerms = [
              "background",
              "noise_action",
              "remove_support_sleeve",
              "remove_pointer_sleeve",
              "adjust_rider",
              "adjust_nut",
              "adjust_balancing",
              "open_box",
              "close_box",
              "choose_weight",
              "put_left",
              "put_right",
              "take_left",
              "take_right",
              "install_support_sleeve",
              "install_pointer_sleeve",
          ]
  
          self.EmbedBufferTop = np.zeros((1024, 0))
          self.EmbedBufferFront = np.zeros((1024, 0))
          self.ImgSizeHeight = 224
          self.ImgSizeWidth = 224
          self.EmbedBatchSize = 1
          self.SegBatchSize = 24
          self.EmbedWindowLength = 16
          self.EmbedWindowStride = 1
          self.EmbedWindowAtrous = 3
          self.TemporalLogits = np.zeros((0, len(self.ActionTerms)))
  
          net = ie.read_model(i3d_path)
          net.reshape({net.inputs[0]: PartialShape(
              [self.EmbedBatchSize, self.EmbedWindowLength, self.ImgSizeHeight, self.ImgSizeWidth, 3])})
          nodes = net.get_ops()
          net.add_outputs(nodes[13].output(0))
          self.i3d = ie.compile_model(model=net, device_name=device)
  
          self.mstcn_net = ie.read_model(mstcn_path)
          self.mstcn = ie.compile_model(model=self.mstcn_net, device_name=device)
          self.mstcn_input_keys = self.mstcn.inputs
          self.mstcn_output_key = self.mstcn.outputs
          self.mstcn_net.reshape({'input': PartialShape([1, 2048, 1])})
          self.reshape_mstcn = ie.compile_model(model=self.mstcn_net, device_name=device)
          file_path = Path(__file__).parent / 'init_his.npz'
          init_his_feature = np.load(file_path)
          self.his_fea = {f'fhis_in_{i}': init_his_feature[f'arr_{i}'] for i in range(4)}
  
      def inference(self, buffer_top, buffer_front, frame_index):
          """
          Args:
              buffer_top: buffers of the input image arrays for the top view
              buffer_front: buffers of the input image arrays for the front view
              frame_index: frame index of the latest frame
          Returns: the temporal prediction results for each frame (including the historical predictions)，
                   length of predictions == frame_index()
          """
          ### run encoder ###
          self.EmbedBufferTop = self.feature_embedding(
              img_buffer=buffer_top,
              embedding_buffer=self.EmbedBufferTop,
              frame_index=frame_index)
          self.EmbedBufferFront = self.feature_embedding(
              img_buffer=buffer_front,
              embedding_buffer=self.EmbedBufferFront,
              frame_index=frame_index)
  
          ### run mstcn++ only batch size 1###
          if min(self.EmbedBufferTop.shape[-1], self.EmbedBufferFront.shape[-1]) > 0:
              self.action_segmentation()
  
          # ### get label ###
          valid_index = self.TemporalLogits.shape[0]
          if valid_index == 0:
              return []
          else:
              frame_predictions = [self.ActionTerms[i] for i in np.argmax(self.TemporalLogits, axis=1)]
              frame_predictions = ["background" for i in range(self.EmbedWindowLength - 1)] + frame_predictions
  
          return frame_predictions[-1]
  
      def feature_embedding(self, img_buffer, embedding_buffer, frame_index):
          # minimal temporal length for processor
          min_t = (self.EmbedWindowLength - 1) * self.EmbedWindowAtrous
  
          infer_request = self.i3d.create_infer_request()
          if frame_index > min_t:
              num_embedding = embedding_buffer.shape[-1]
              img_buffer = list(img_buffer)
              curr_t = self.EmbedWindowStride * num_embedding + (self.EmbedWindowLength - 1) * self.EmbedWindowAtrous
              while curr_t < frame_index:
                  # absolute index in temporal shaft
                  start_index = self.EmbedWindowStride * num_embedding
  
                  if frame_index > len(img_buffer):
                      # absolute index in buffer shaft
                      start_index = start_index - (frame_index - len(img_buffer))
  
                  input_data = [
                      [cv2.resize(img_buffer[start_index + i * self.EmbedWindowAtrous],
                      (self.ImgSizeHeight, self.ImgSizeWidth)) for i in range(self.EmbedWindowLength)]
                      for j in range(self.EmbedBatchSize)]
                  input_data = np.asarray(input_data).transpose((0, 4, 1, 2, 3))
                  input_data = input_data * 127.5 + 127.5
  
                  input_dict = {self.i3d.inputs[0]: input_data}
                  out_logits = infer_request.infer(input_dict)[self.i3d.outputs[1]]
                  out_logits = out_logits.squeeze((0, 3, 4))
  
                  # ndarray: C x num_embedding
                  embedding_buffer = np.concatenate((embedding_buffer, out_logits), axis=1)
  
                  curr_t += self.EmbedWindowStride
          return embedding_buffer
  
      def action_segmentation(self):
          # read buffer
          embed_buffer_top = self.EmbedBufferTop
          embed_buffer_front = self.EmbedBufferFront
          batch_size = self.SegBatchSize
          start_index = self.TemporalLogits.shape[0]
          end_index = min(embed_buffer_top.shape[-1], embed_buffer_front.shape[-1])
          num_batch = (end_index - start_index) // batch_size
  
          infer_request = self.reshape_mstcn.create_infer_request()
          if num_batch < 0:
              log.debug("Waiting for the next frame ...")
          elif num_batch == 0:
              log.debug(f"start_index: {start_index} end_index: {end_index}")
              unit1 = embed_buffer_top[:, start_index:end_index]
              unit2 = embed_buffer_front[:, start_index:end_index]
              feature_unit = np.concatenate([unit1[:, ], unit2[:, ]], axis=0)
              input_mstcn = np.expand_dims(feature_unit, 0)
  
              feed_dict = {}
              if len(self.his_fea) != 0:
                  for key in self.mstcn_input_keys:
                      if 'fhis_in_' in str(key.names):
                          string = list(key.names)[0]
                          feed_dict[string] = self.his_fea[string]
              feed_dict['input'] = input_mstcn
              if input_mstcn.shape == (1, 2048, 1):
                  out = infer_request.infer(feed_dict)
  
              predictions = out[list(out.keys())[-1]]
              for key in self.mstcn_output_key:
                  if 'fhis_in_' in str(key.names):
                      string = list(key.names)[0]
                      self.his_fea[string] = out[string]
  
              """
                  predictions --> 4x1x64x24
                  his_fea --> [12*[1x64x2048], 11*[1x64x2048], 11*[1x64x2048], 11*[1x64x2048]]
              """
              temporal_logits = predictions[:, :, :len(self.ActionTerms), :]  # 4x1x16xN
              temporal_logits = softmax(temporal_logits[-1], 1)  # 1x16xN
              temporal_logits = temporal_logits.transpose((0, 2, 1)).squeeze(axis=0)
              self.TemporalLogits = np.concatenate([self.TemporalLogits, temporal_logits], axis=0)
          else:
              for batch_idx in range(num_batch):
                  unit1 = embed_buffer_top[:,
                          start_index + batch_idx * batch_size:start_index + batch_idx * batch_size + batch_size]
                  unit2 = embed_buffer_front[:,
                          start_index + batch_idx * batch_size:start_index + batch_idx * batch_size + batch_size]
                  feature_unit = np.concatenate([unit1[:, ], unit2[:, ]], axis=0)
  
                  feed_dict = {}
                  if len(self.his_fea) != 0:
                      for key in self.mstcn_input_keys:
                          if 'fhis_in_' in str(key.names):
                              string = list(key.names)[0]
                              feed_dict[key] = self.his_fea[string]
                  feed_dict['input'] = feature_unit
                  out = infer_request.infer(feed_dict)
                  predictions = out[list(out.keys())[-1]]
                  for key in self.mstcn_output_key:
                      if 'fhis_in_' in str(key.names):
                          string = list(key.names)[0]
                          self.his_fea[string] = out[string]
  
                  temporal_logits = predictions[:, :, :len(self.ActionTerms), :]  # 4x1x16xN
                  temporal_logits = softmax(temporal_logits[-1], 1)  # 1x16xN
                  temporal_logits = temporal_logits.transpose((0, 2, 1)).squeeze(axis=0)
                  self.TemporalLogits = np.concatenate([self.TemporalLogits, temporal_logits], axis=0)
   #!/usr/bin/env python3
  """
   Copyright (c) 2018-2021 Intel Corporation
  
   Licensed under the Apache License, Version 2.0 (the "License");
   you may not use this file except in compliance with the License.
   You may obtain a copy of the License at
  
        http://www.apache.org/licenses/LICENSE-2.0
  
   Unless required by applicable law or agreed to in writing, software
   distributed under the License is distributed on an "AS IS" BASIS,
   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
   See the License for the specific language governing permissions and
   limitations under the License.
  """
  
  from openvino.runtime import Core, get_version
  import cv2 as cv
  import numpy as np
  import logging as log
  from time import perf_counter
  import sys
  from argparse import ArgumentParser, SUPPRESS
  from pathlib import Path
  
  sys.path.append(str(Path(__file__).resolve().parents[2] / 'common/python'))
  
  import monitors
  from images_capture import open_images_capture
  from openvino.model_zoo.model_api.performance_metrics import PerformanceMetrics
  
  log.basicConfig(format='[ %(levelname)s ] %(message)s', level=log.DEBUG, stream=sys.stdout)
  
  
  def build_arg():
      parser = ArgumentParser(add_help=False)
      in_args = parser.add_argument_group('Options')
      in_args.add_argument('-h', '--help', action='help', default=SUPPRESS, help='Help with the script.')
      in_args.add_argument("-m", "--model", help="Required. Path to .xml file with pre-trained model.",
                           required=True, type=Path)
      in_args.add_argument("-d", "--device",
                           help="Optional. Specify target device for infer: CPU, GPU, HDDL or MYRIAD. "
                                "Default: CPU",
                           default="CPU", type=str)
      in_args.add_argument('-i', "--input", required=True,
                           help='Required. An input to process. The input must be a single image, '
                                'a folder of images, video file or camera id.')
      in_args.add_argument('--loop', default=False, action='store_true',
                           help='Optional. Enable reading the input in a loop.')
      in_args.add_argument('-o', '--output', required=False,
                           help='Optional. Name of the output file(s) to save.')
      in_args.add_argument('-limit', '--output_limit', required=False, default=1000, type=int,
                           help='Optional. Number of frames to store in output. '
                                'If 0 is set, all frames are stored.')
      in_args.add_argument("--no_show", help="Optional. Don't show output.",
                           action='store_true', default=False)
      in_args.add_argument("-u", "--utilization_monitors", default="", type=str,
                           help="Optional. List of monitors to show initially.")
      return parser
  
  def main(args):
      cap = open_images_capture(args.input, args.loop)
  
      log.info('OpenVINO Inference Engine')
      log.info('\tbuild: {}'.format(get_version()))
      core = Core()
  
      log.info('Reading model {}'.format(args.model))
      model = core.read_model(args.model)
  
      input_tensor_name = 'data_l'
      input_shape = model.input(input_tensor_name).shape
      assert input_shape[1] == 1, "Expected model input shape with 1 channel"
  
      inputs = {}
      for input in model.inputs:
          inputs[input.get_any_name()] = np.zeros(input.shape)
  
      assert len(model.outputs) == 1, "Expected number of outputs is equal 1"
  
      compiled_model = core.compile_model(model, device_name=args.device)
      output_tensor = compiled_model.outputs[0]
      infer_request = compiled_model.create_infer_request()
      log.info('The model {} is loaded to {}'.format(args.model, args.device))
  
      _, _, h_in, w_in = input_shape
  
      frames_processed = 0
      imshow_size = (640, 480)
      graph_size = (imshow_size[0] // 2, imshow_size[1] // 4)
      presenter = monitors.Presenter(args.utilization_monitors, imshow_size[1] * 2 - graph_size[1], graph_size)
      metrics = PerformanceMetrics()
  
      video_writer = cv.VideoWriter()
      if args.output and not video_writer.open(args.output, cv.VideoWriter_fourcc(*'MJPG'),
                                               cap.fps(), (imshow_size[0] * 2, imshow_size[1] * 2)):
          raise RuntimeError("Can't open video writer")
  
      start_time = perf_counter()
      original_frame = cap.read()
      if original_frame is None:
          raise RuntimeError("Can't read an image from the input")
  
      while original_frame is not None:
          (h_orig, w_orig) = original_frame.shape[:2]
  
          if original_frame.shape[2] > 1:
              frame = cv.cvtColor(cv.cvtColor(original_frame, cv.COLOR_BGR2GRAY), cv.COLOR_GRAY2RGB)
          else:
              frame = cv.cvtColor(original_frame, cv.COLOR_GRAY2RGB)
  
          img_rgb = frame.astype(np.float32) / 255
          img_lab = cv.cvtColor(img_rgb, cv.COLOR_RGB2Lab)
          img_l_rs = cv.resize(img_lab.copy(), (w_in, h_in))[:, :, 0]
  
          inputs[input_tensor_name] = np.expand_dims(img_l_rs, axis=[0, 1])
  
          res = infer_request.infer(inputs)[output_tensor]
  
          update_res = np.squeeze(res)
  
          out = update_res.transpose((1, 2, 0))
          out = cv.resize(out, (w_orig, h_orig))
          img_lab_out = np.concatenate((img_lab[:, :, 0][:, :, np.newaxis], out), axis=2)
          img_bgr_out = np.clip(cv.cvtColor(img_lab_out, cv.COLOR_Lab2BGR), 0, 1)
  
          original_image = cv.resize(original_frame, imshow_size)
          grayscale_image = cv.resize(frame, imshow_size)
          colorize_image = (cv.resize(img_bgr_out, imshow_size) * 255).astype(np.uint8)
          lab_image = cv.resize(img_lab_out, imshow_size).astype(np.uint8)
  
          original_image = cv.putText(original_image, 'Original', (25, 50),
                                      cv.FONT_HERSHEY_SIMPLEX, 1, (0, 0, 255), 2, cv.LINE_AA)
          grayscale_image = cv.putText(grayscale_image, 'Grayscale', (25, 50),
                                       cv.FONT_HERSHEY_SIMPLEX, 1, (0, 0, 255), 2, cv.LINE_AA)
          colorize_image = cv.putText(colorize_image, 'Colorize', (25, 50),
                                      cv.FONT_HERSHEY_SIMPLEX, 1, (0, 0, 255), 2, cv.LINE_AA)
          lab_image = cv.putText(lab_image, 'LAB interpretation', (25, 50),
                                 cv.FONT_HERSHEY_SIMPLEX, 1, (0, 0, 255), 2, cv.LINE_AA)
  
          ir_image = [cv.hconcat([original_image, grayscale_image]),
                      cv.hconcat([lab_image, colorize_image])]
          final_image = cv.vconcat(ir_image)
  
          metrics.update(start_time, final_image)
  
          frames_processed += 1
          if video_writer.isOpened() and (args.output_limit <= 0 or frames_processed <= args.output_limit):
              video_writer.write(final_image)
  
          presenter.drawGraphs(final_image)
          if not args.no_show:
              cv.imshow('Colorization Demo', final_image)
              key = cv.waitKey(1)
              if key in {ord("q"), ord("Q"), 27}:
                  break
              presenter.handleKey(key)
          start_time = perf_counter()
          original_frame = cap.read()
  
      metrics.log_total()
      for rep in presenter.reportMeans():
          log.info(rep)
  
  if __name__ == "__main__":
      args = build_arg().parse_args()
      sys.exit(main(args) or 0)
  
apis: numpy.load numpy.zeros scipy.special.softmax numpy.asarray numpy.argmax numpy.expand_dims numpy.concatenate
apis: numpy.squeeze numpy.concatenate numpy.expand_dims numpy.zeros