text: // Copyright 2015 The Chromium Authors. All rights reserved.
  // Use of this source code is governed by a BSD-style license that can be
  // found in the LICENSE file.
  
  #include "remoting/protocol/webrtc_video_renderer_adapter.h"
  
  #include <memory>
  #include <string>
  #include <utility>
  
  #include "base/bind.h"
  #include "base/callback.h"
  #include "base/location.h"
  #include "base/memory/ptr_util.h"
  #include "base/single_thread_task_runner.h"
  #include "base/task_runner_util.h"
  #include "base/threading/thread_task_runner_handle.h"
  #include "base/threading/worker_pool.h"
  #include "remoting/protocol/client_video_stats_dispatcher.h"
  #include "remoting/protocol/frame_consumer.h"
  #include "remoting/protocol/frame_stats.h"
  #include "remoting/protocol/video_renderer.h"
  #include "remoting/protocol/webrtc_transport.h"
  #include "third_party/libyuv/include/libyuv/convert_from.h"
  #include "third_party/webrtc/modules/desktop_capture/desktop_frame.h"
  
  namespace remoting {
  namespace protocol {
  
  namespace {
  
  // Maximum number of ClientFrameStats instances to keep.
  const int kMaxQueuedStats = 200;
  
  std::unique_ptr<webrtc::DesktopFrame> ConvertYuvToRgb(
      scoped_refptr<webrtc::VideoFrameBuffer> yuv_frame,
      std::unique_ptr<webrtc::DesktopFrame> rgb_frame,
      FrameConsumer::PixelFormat pixel_format) {
    DCHECK(rgb_frame->size().equals(
        webrtc::DesktopSize(yuv_frame->width(), yuv_frame->height())));
    auto yuv_to_rgb_function = (pixel_format == FrameConsumer::FORMAT_BGRA)
                                   ? &libyuv::I420ToARGB
                                   : &libyuv::I420ToABGR;
    yuv_to_rgb_function(yuv_frame->DataY(), yuv_frame->StrideY(),
                        yuv_frame->DataU(), yuv_frame->StrideU(),
                        yuv_frame->DataV(), yuv_frame->StrideV(),
                        rgb_frame->data(), rgb_frame->stride(),
                        yuv_frame->width(), yuv_frame->height());
  
    rgb_frame->mutable_updated_region()->AddRect(
        webrtc::DesktopRect::MakeSize(rgb_frame->size()));
    return rgb_frame;
  }
  
  }  // namespace
  
  WebrtcVideoRendererAdapter::WebrtcVideoRendererAdapter(
      const std::string& label,
      VideoRenderer* video_renderer)
      : label_(label),
        video_renderer_(video_renderer),
        task_runner_(base::ThreadTaskRunnerHandle::Get()),
        weak_factory_(this) {}
  
  WebrtcVideoRendererAdapter::~WebrtcVideoRendererAdapter() {
    DCHECK(task_runner_->BelongsToCurrentThread());
  
    webrtc::VideoTrackVector video_tracks = media_stream_->GetVideoTracks();
    DCHECK(!video_tracks.empty());
    video_tracks[0]->RemoveSink(this);
  }
  
  void WebrtcVideoRendererAdapter::SetMediaStream(
      scoped_refptr<webrtc::MediaStreamInterface> media_stream) {
    DCHECK_EQ(media_stream->label(), label());
  
    media_stream_ = std::move(media_stream);
  
    webrtc::VideoTrackVector video_tracks = media_stream_->GetVideoTracks();
  
    // Caller must verify that the media stream contains video tracks.
    DCHECK(!video_tracks.empty());
  
    if (video_tracks.size() > 1U) {
      LOG(WARNING) << "Received media stream with multiple video tracks.";
    }
  
    video_tracks[0]->AddOrUpdateSink(this, rtc::VideoSinkWants());
  }
  
  void WebrtcVideoRendererAdapter::SetVideoStatsChannel(
      std::unique_ptr<MessagePipe> message_pipe) {
    // Expect that the host also creates video_stats data channel.
    video_stats_dispatcher_.reset(new ClientVideoStatsDispatcher(label_, this));
    video_stats_dispatcher_->Init(std::move(message_pipe), this);
  }
  
  void WebrtcVideoRendererAdapter::OnFrame(const webrtc::VideoFrame& frame) {
    if (frame.timestamp_us() >= rtc::TimeMicros()) {
      // The host sets playout delay to 0, so all incoming frames are expected to
      // be rendered as so as they are received.
      LOG(WARNING) << "Received frame with playout delay greater than 0.";
    }
  
    task_runner_->PostTask(
        FROM_HERE,
        base::Bind(&WebrtcVideoRendererAdapter::HandleFrameOnMainThread,
                   weak_factory_.GetWeakPtr(), frame.transport_frame_id(),
                   base::TimeTicks::Now(),
                   scoped_refptr<webrtc::VideoFrameBuffer>(
                       frame.video_frame_buffer().get())));
  }
  
  void WebrtcVideoRendererAdapter::OnVideoFrameStats(
      uint32_t frame_id,
      const HostFrameStats& host_stats) {
    DCHECK(task_runner_->BelongsToCurrentThread());
  
    // Drop all ClientFrameStats for frames before |frame_id|. Stats messages are
    // expected to be received in the same order as the corresponding video
    // frames, so we are not going to receive HostFrameStats for the frames before
    // |frame_id|. This may happen only if for some reason the host doesn't
    // generate stats message for all video frames.
    while (!client_stats_queue_.empty() &&
           client_stats_queue_.front().first != frame_id) {
      client_stats_queue_.pop_front();
    }
  
    // If there are no ClientFrameStats in the queue then queue HostFrameStats
    // to be processed in FrameRendered().
    if (client_stats_queue_.empty()) {
      if (host_stats_queue_.size() > kMaxQueuedStats) {
        LOG(ERROR) << "video_stats channel is out of sync with the video stream. "
                      "Performance stats will not be reported.";
        video_stats_dispatcher_.reset();
        return;
      }
      host_stats_queue_.push_back(std::make_pair(frame_id, host_stats));
      return;
    }
  
    // The correspond frame has been received and now we have both HostFrameStats
    // and ClientFrameStats. Report the stats to FrameStatsConsumer.
    DCHECK_EQ(client_stats_queue_.front().first, frame_id);
    FrameStats frame_stats;
    frame_stats.client_stats = client_stats_queue_.front().second;
    client_stats_queue_.pop_front();
    frame_stats.host_stats = host_stats;
    FrameStatsConsumer* frame_stats_consumer =
        video_renderer_->GetFrameStatsConsumer();
    if (frame_stats_consumer)
      frame_stats_consumer->OnVideoFrameStats(frame_stats);
  }
  
  void WebrtcVideoRendererAdapter::OnChannelInitialized(
      ChannelDispatcherBase* channel_dispatcher) {}
  
  void WebrtcVideoRendererAdapter::OnChannelClosed(
      ChannelDispatcherBase* channel_dispatcher) {
    LOG(WARNING) << "video_stats channel was closed by the host.";
  }
  
  void WebrtcVideoRendererAdapter::HandleFrameOnMainThread(
      uint32_t frame_id,
      base::TimeTicks time_received,
      scoped_refptr<webrtc::VideoFrameBuffer> frame) {
    DCHECK(task_runner_->BelongsToCurrentThread());
  
    std::unique_ptr<ClientFrameStats> stats(new ClientFrameStats());
    // TODO(sergeyu): |time_received| is not reported correctly here because the
    // frame is already decoded at this point.
    stats->time_received = time_received;
  
    std::unique_ptr<webrtc::DesktopFrame> rgb_frame =
        video_renderer_->GetFrameConsumer()->AllocateFrame(
            webrtc::DesktopSize(frame->width(), frame->height()));
  
    base::PostTaskAndReplyWithResult(
        base::WorkerPool::GetTaskRunner(false).get(), FROM_HERE,
        base::Bind(&ConvertYuvToRgb, base::Passed(&frame),
                   base::Passed(&rgb_frame),
                   video_renderer_->GetFrameConsumer()->GetPixelFormat()),
        base::Bind(&WebrtcVideoRendererAdapter::DrawFrame,
                   weak_factory_.GetWeakPtr(), frame_id, base::Passed(&stats)));
  }
  
  void WebrtcVideoRendererAdapter::DrawFrame(
      uint32_t frame_id,
      std::unique_ptr<ClientFrameStats> stats,
      std::unique_ptr<webrtc::DesktopFrame> frame) {
    DCHECK(task_runner_->BelongsToCurrentThread());
    stats->time_decoded = base::TimeTicks::Now();
    video_renderer_->GetFrameConsumer()->DrawFrame(
        std::move(frame),
        base::Bind(&WebrtcVideoRendererAdapter::FrameRendered,
                   weak_factory_.GetWeakPtr(), frame_id, base::Passed(&stats)));
  }
  
  void WebrtcVideoRendererAdapter::FrameRendered(
      uint32_t frame_id,
      std::unique_ptr<ClientFrameStats> client_stats) {
    DCHECK(task_runner_->BelongsToCurrentThread());
  
    if (!video_stats_dispatcher_ || !video_stats_dispatcher_->is_connected())
      return;
  
    client_stats->time_rendered = base::TimeTicks::Now();
  
    // Drop all HostFrameStats for frames before |frame_id|. Stats messages are
    // expected to be received in the same order as the corresponding video
    // frames. This may happen only if the host generates HostFrameStats without
    // the corresponding frame.
    while (!host_stats_queue_.empty() &&
           host_stats_queue_.front().first != frame_id) {
      LOG(WARNING) << "Host sent VideoStats message for a frame that was never "
                      "received.";
      host_stats_queue_.pop_front();
    }
  
    // If HostFrameStats hasn't been received for |frame_id| then queue
    // ClientFrameStats to be processed in OnVideoFrameStats().
    if (host_stats_queue_.empty()) {
      if (client_stats_queue_.size() > kMaxQueuedStats) {
        LOG(ERROR) << "video_stats channel is out of sync with the video "
                      "stream. Performance stats will not be reported.";
        video_stats_dispatcher_.reset();
        return;
      }
      client_stats_queue_.push_back(std::make_pair(frame_id, *client_stats));
      return;
    }
  
    // The correspond HostFrameStats has been received already and now we have
    // both HostFrameStats and ClientFrameStats. Report the stats to
    // FrameStatsConsumer.
    DCHECK_EQ(host_stats_queue_.front().first, frame_id);
    FrameStats frame_stats;
    frame_stats.host_stats = host_stats_queue_.front().second;
    frame_stats.client_stats = *client_stats;
    host_stats_queue_.pop_front();
    FrameStatsConsumer* frame_stats_consumer =
        video_renderer_->GetFrameStatsConsumer();
    if (frame_stats_consumer)
      frame_stats_consumer->OnVideoFrameStats(frame_stats);
  }
  
  }  // namespace protocol
  }  // namespace remoting
  