text: /*
   *  Copyright (c) 2012 The WebRTC project authors. All Rights Reserved.
   *
   *  Use of this source code is governed by a BSD-style license
   *  that can be found in the LICENSE file in the root of the source
   *  tree. An additional intellectual property rights grant can be found
   *  in the file PATENTS.  All contributing project authors may
   *  be found in the AUTHORS file in the root of the source tree.
   */
  
  #ifndef MODULES_VIDEO_CODING_CODECS_TEST_VIDEOPROCESSOR_H_
  #define MODULES_VIDEO_CODING_CODECS_TEST_VIDEOPROCESSOR_H_
  
  #include <stddef.h>
  #include <stdint.h>
  
  #include <map>
  #include <memory>
  #include <utility>
  #include <vector>
  
  #include "absl/types/optional.h"
  #include "api/task_queue/queued_task.h"
  #include "api/task_queue/task_queue_base.h"
  #include "api/test/videocodec_test_fixture.h"
  #include "api/video/encoded_image.h"
  #include "api/video/i420_buffer.h"
  #include "api/video/video_bitrate_allocation.h"
  #include "api/video/video_bitrate_allocator.h"
  #include "api/video/video_frame.h"
  #include "api/video_codecs/video_decoder.h"
  #include "api/video_codecs/video_encoder.h"
  #include "modules/include/module_common_types.h"
  #include "modules/video_coding/codecs/test/videocodec_test_stats_impl.h"
  #include "modules/video_coding/include/video_codec_interface.h"
  #include "modules/video_coding/utility/ivf_file_writer.h"
  #include "rtc_base/buffer.h"
  #include "rtc_base/checks.h"
  #include "rtc_base/constructor_magic.h"
  #include "rtc_base/synchronization/sequence_checker.h"
  #include "rtc_base/thread_annotations.h"
  #include "rtc_base/thread_checker.h"
  #include "test/testsupport/frame_reader.h"
  #include "test/testsupport/frame_writer.h"
  
  namespace webrtc {
  namespace test {
  
  // Handles encoding/decoding of video using the VideoEncoder/VideoDecoder
  // interfaces. This is done in a sequential manner in order to be able to
  // measure times properly.
  // The class processes a frame at the time for the configured input file.
  // It maintains state of where in the source input file the processing is at.
  class VideoProcessor {
   public:
    using VideoDecoderList = std::vector<std::unique_ptr<VideoDecoder>>;
    using LayerKey = std::pair<int /* spatial_idx */, int /* temporal_idx */>;
    using IvfFileWriterMap = std::map<LayerKey, std::unique_ptr<IvfFileWriter>>;
    // TODO(brandtr): Consider changing FrameWriterList to be a FrameWriterMap,
    // to be able to save different TLs separately.
    using FrameWriterList = std::vector<std::unique_ptr<FrameWriter>>;
    using FrameStatistics = VideoCodecTestStats::FrameStatistics;
  
    VideoProcessor(webrtc::VideoEncoder* encoder,
                   VideoDecoderList* decoders,
                   FrameReader* input_frame_reader,
                   const VideoCodecTestFixture::Config& config,
                   VideoCodecTestStatsImpl* stats,
                   IvfFileWriterMap* encoded_frame_writers,
                   FrameWriterList* decoded_frame_writers);
    ~VideoProcessor();
  
    // Reads a frame and sends it to the encoder. When the encode callback
    // is received, the encoded frame is buffered. After encoding is finished
    // buffered frame is sent to decoder. Quality evaluation is done in
    // the decode callback.
    void ProcessFrame();
  
    // Updates the encoder with target rates. Must be called at least once.
    void SetRates(size_t bitrate_kbps, double framerate_fps);
  
    // Signals processor to finalize frame processing and handle possible tail
    // drops. If not called expelicitly, this will be called in dtor. It is
    // unexpected to get ProcessFrame() or SetRates() calls after Finalize().
    void Finalize();
  
   private:
    class VideoProcessorEncodeCompleteCallback
        : public webrtc::EncodedImageCallback {
     public:
      explicit VideoProcessorEncodeCompleteCallback(
          VideoProcessor* video_processor)
          : video_processor_(video_processor),
            task_queue_(TaskQueueBase::Current()) {
        RTC_DCHECK(video_processor_);
        RTC_DCHECK(task_queue_);
      }
  
      Result OnEncodedImage(
          const webrtc::EncodedImage& encoded_image,
          const webrtc::CodecSpecificInfo* codec_specific_info) override {
        RTC_CHECK(codec_specific_info);
  
        // Post the callback to the right task queue, if needed.
        if (!task_queue_->IsCurrent()) {
          task_queue_->PostTask(std::make_unique<EncodeCallbackTask>(
              video_processor_, encoded_image, codec_specific_info));
          return Result(Result::OK, 0);
        }
  
        video_processor_->FrameEncoded(encoded_image, *codec_specific_info);
        return Result(Result::OK, 0);
      }
  
     private:
      class EncodeCallbackTask : public QueuedTask {
       public:
        EncodeCallbackTask(VideoProcessor* video_processor,
                           const webrtc::EncodedImage& encoded_image,
                           const webrtc::CodecSpecificInfo* codec_specific_info)
            : video_processor_(video_processor),
              encoded_image_(encoded_image),
              codec_specific_info_(*codec_specific_info) {
          encoded_image_.Retain();
        }
  
        bool Run() override {
          video_processor_->FrameEncoded(encoded_image_, codec_specific_info_);
          return true;
        }
  
       private:
        VideoProcessor* const video_processor_;
        webrtc::EncodedImage encoded_image_;
        const webrtc::CodecSpecificInfo codec_specific_info_;
      };
  
      VideoProcessor* const video_processor_;
      TaskQueueBase* const task_queue_;
    };
  
    class VideoProcessorDecodeCompleteCallback
        : public webrtc::DecodedImageCallback {
     public:
      explicit VideoProcessorDecodeCompleteCallback(
          VideoProcessor* video_processor,
          size_t simulcast_svc_idx)
          : video_processor_(video_processor),
            simulcast_svc_idx_(simulcast_svc_idx),
            task_queue_(TaskQueueBase::Current()) {
        RTC_DCHECK(video_processor_);
        RTC_DCHECK(task_queue_);
      }
  
      int32_t Decoded(webrtc::VideoFrame& image) override;
  
      int32_t Decoded(webrtc::VideoFrame& image,
                      int64_t decode_time_ms) override {
        return Decoded(image);
      }
  
      void Decoded(webrtc::VideoFrame& image,
                   absl::optional<int32_t> decode_time_ms,
                   absl::optional<uint8_t> qp) override {
        Decoded(image);
      }
  
     private:
      VideoProcessor* const video_processor_;
      const size_t simulcast_svc_idx_;
      TaskQueueBase* const task_queue_;
    };
  
    // Invoked by the callback adapter when a frame has completed encoding.
    void FrameEncoded(const webrtc::EncodedImage& encoded_image,
                      const webrtc::CodecSpecificInfo& codec_specific);
  
    // Invoked by the callback adapter when a frame has completed decoding.
    void FrameDecoded(const webrtc::VideoFrame& image, size_t simulcast_svc_idx);
  
    void DecodeFrame(const EncodedImage& encoded_image, size_t simulcast_svc_idx);
  
    // In order to supply the SVC decoders with super frames containing all
    // lower layer frames, we merge and store the layer frames in this method.
    const webrtc::EncodedImage* BuildAndStoreSuperframe(
        const EncodedImage& encoded_image,
        const VideoCodecType codec,
        size_t frame_number,
        size_t simulcast_svc_idx,
        bool inter_layer_predicted) RTC_RUN_ON(sequence_checker_);
  
    void CalcFrameQuality(const I420BufferInterface& decoded_frame,
                          FrameStatistics* frame_stat);
  
    void WriteDecodedFrame(const I420BufferInterface& decoded_frame,
                           FrameWriter& frame_writer);
  
    void HandleTailDrops();
  
    // Test config.
    const VideoCodecTestFixture::Config config_;
    const size_t num_simulcast_or_spatial_layers_;
    const bool analyze_frame_quality_;
  
    // Frame statistics.
    VideoCodecTestStatsImpl* const stats_;
  
    // Codecs.
    webrtc::VideoEncoder* const encoder_;
    VideoDecoderList* const decoders_;
    const std::unique_ptr<VideoBitrateAllocator> bitrate_allocator_;
    VideoBitrateAllocation bitrate_allocation_ RTC_GUARDED_BY(sequence_checker_);
    double framerate_fps_ RTC_GUARDED_BY(sequence_checker_);
  
    // Adapters for the codec callbacks.
    VideoProcessorEncodeCompleteCallback encode_callback_;
    // Assign separate callback object to each decoder. This allows us to identify
    // decoded layer in frame decode callback.
    // simulcast_svc_idx -> decode callback.
    std::vector<std::unique_ptr<VideoProcessorDecodeCompleteCallback>>
        decode_callback_;
  
    // Each call to ProcessFrame() will read one frame from |input_frame_reader_|.
    FrameReader* const input_frame_reader_;
  
    // Input frames are used as reference for frame quality evaluations.
    // Async codecs might queue frames. To handle that we keep input frame
    // and release it after corresponding coded frame is decoded and quality
    // measurement is done.
    // frame_number -> frame.
    std::map<size_t, VideoFrame> input_frames_ RTC_GUARDED_BY(sequence_checker_);
  
    // Encoder delivers coded frame layer-by-layer. We store coded frames and
    // then, after all layers are encoded, decode them. Such separation of
    // frame processing on superframe level simplifies encoding/decoding time
    // measurement.
    // simulcast_svc_idx -> merged SVC encoded frame.
    std::vector<EncodedImage> merged_encoded_frames_
        RTC_GUARDED_BY(sequence_checker_);
  
    // These (optional) file writers are used to persistently store the encoded
    // and decoded bitstreams. Each frame writer is enabled by being non-null.
    IvfFileWriterMap* const encoded_frame_writers_;
    FrameWriterList* const decoded_frame_writers_;
  
    // Metadata for inputed/encoded/decoded frames. Used for frame identification,
    // frame drop detection, etc. We assume that encoded/decoded frames are
    // ordered within each simulcast/spatial layer, but we do not make any
    // assumptions of frame ordering between layers.
    size_t last_inputed_frame_num_ RTC_GUARDED_BY(sequence_checker_);
    size_t last_inputed_timestamp_ RTC_GUARDED_BY(sequence_checker_);
    // simulcast_svc_idx -> encode status.
    std::vector<bool> first_encoded_frame_ RTC_GUARDED_BY(sequence_checker_);
    // simulcast_svc_idx -> frame_number.
    std::vector<size_t> last_encoded_frame_num_ RTC_GUARDED_BY(sequence_checker_);
    // simulcast_svc_idx -> decode status.
    std::vector<bool> first_decoded_frame_ RTC_GUARDED_BY(sequence_checker_);
    // simulcast_svc_idx -> frame_number.
    std::vector<size_t> last_decoded_frame_num_ RTC_GUARDED_BY(sequence_checker_);
    // simulcast_svc_idx -> buffer.
    std::vector<rtc::scoped_refptr<I420Buffer>> last_decoded_frame_buffer_
        RTC_GUARDED_BY(sequence_checker_);
  
    // Time spent in frame encode callback. It is accumulated for layers and
    // reset when frame encode starts. When next layer is encoded post-encode time
    // is substracted from measured encode time. Thus we get pure encode time.
    int64_t post_encode_time_ns_ RTC_GUARDED_BY(sequence_checker_);
  
    // Indicates whether Finalize() was called or not.
    bool is_finalized_ RTC_GUARDED_BY(sequence_checker_);
  
    // This class must be operated on a TaskQueue.
    SequenceChecker sequence_checker_;
  
    RTC_DISALLOW_COPY_AND_ASSIGN(VideoProcessor);
  };
  
  }  // namespace test
  }  // namespace webrtc
  
  #endif  // MODULES_VIDEO_CODING_CODECS_TEST_VIDEOPROCESSOR_H_
  