text: // Copyright 2013 The Chromium Authors. All rights reserved.
  // Use of this source code is governed by a BSD-style license that can be
  // found in the LICENSE file.
  
  #ifndef THIRD_PARTY_BLINK_RENDERER_MODULES_MEDIASTREAM_MEDIA_STREAM_AUDIO_PROCESSOR_H_
  #define THIRD_PARTY_BLINK_RENDERER_MODULES_MEDIASTREAM_MEDIA_STREAM_AUDIO_PROCESSOR_H_
  
  #include <memory>
  
  #include "base/atomicops.h"
  #include "base/files/file.h"
  #include "base/macros.h"
  #include "base/memory/scoped_refptr.h"
  #include "base/optional.h"
  #include "base/threading/thread_checker.h"
  #include "base/time/time.h"
  #include "media/webrtc/audio_delay_stats_reporter.h"
  #include "third_party/blink/renderer/modules/modules_export.h"
  #include "third_party/blink/renderer/platform/mediastream/aec_dump_agent_impl.h"
  #include "third_party/blink/renderer/platform/mediastream/media_stream_audio_processor_options.h"
  #include "third_party/blink/renderer/platform/webrtc/webrtc_source.h"
  #include "third_party/webrtc/api/media_stream_interface.h"
  #include "third_party/webrtc/modules/audio_processing/include/audio_processing.h"
  #include "third_party/webrtc/rtc_base/task_queue.h"
  
  namespace media {
  class AudioBus;
  class AudioParameters;
  }  // namespace media
  
  namespace webrtc {
  class TypingDetection;
  }
  
  namespace blink {
  
  class AecDumpAgentImpl;
  class MediaStreamAudioBus;
  class MediaStreamAudioFifo;
  
  using webrtc::AudioProcessorInterface;
  
  // This class owns an object of webrtc::AudioProcessing which contains signal
  // processing components like AGC, AEC and NS. It enables the components based
  // on the getUserMedia constraints, processes the data and outputs it in a unit
  // of 10 ms data chunk.
  class MODULES_EXPORT MediaStreamAudioProcessor
      : public WebRtcPlayoutDataSource::Sink,
        public AudioProcessorInterface,
        public AecDumpAgentImpl::Delegate {
   public:
    // |playout_data_source| is used to register this class as a sink to the
    // WebRtc playout data for processing AEC. If clients do not enable AEC,
    // |playout_data_source| won't be used.
    //
    // Threading note: The constructor assumes it is being run on the main render
    // thread.
    MediaStreamAudioProcessor(const AudioProcessingProperties& properties,
                              WebRtcPlayoutDataSource* playout_data_source);
  
    // Called when the format of the capture data has changed.
    // Called on the main render thread. The caller is responsible for stopping
    // the capture thread before calling this method.
    // After this method, the capture thread will be changed to a new capture
    // thread.
    void OnCaptureFormatChanged(const media::AudioParameters& source_params);
  
    // Pushes capture data in |audio_source| to the internal FIFO. Each call to
    // this method should be followed by calls to ProcessAndConsumeData() while
    // it returns false, to pull out all available data.
    // Called on the capture audio thread.
    void PushCaptureData(const media::AudioBus& audio_source,
                         base::TimeDelta capture_delay);
  
    // Processes a block of 10 ms data from the internal FIFO, returning true if
    // |processed_data| contains the result. Returns false and does not modify the
    // outputs if the internal FIFO has insufficient data. The caller does NOT own
    // the object pointed to by |*processed_data|.
    // |capture_delay| is an adjustment on the |capture_delay| value provided in
    // the last call to PushCaptureData().
    // |new_volume| receives the new microphone volume from the AGC.
    // The new microphone volume range is [0, 255], and the value will be 0 if
    // the microphone volume should not be adjusted.
    // Called on the capture audio thread.
    bool ProcessAndConsumeData(int volume,
                               bool key_pressed,
                               media::AudioBus** processed_data,
                               base::TimeDelta* capture_delay,
                               int* new_volume);
  
    // Stops the audio processor, no more AEC dump or render data after calling
    // this method.
    void Stop();
  
    // The audio formats of the capture input to and output from the processor.
    // Must only be called on the main render or audio capture threads.
    const media::AudioParameters& InputFormat() const;
    const media::AudioParameters& OutputFormat() const;
  
    // Accessor to check if the audio processing is enabled or not.
    bool has_audio_processing() const { return !!audio_processing_; }
  
    // AecDumpAgentImpl::Delegate implementation.
    // Called on the main render thread.
    void OnStartDump(base::File dump_file) override;
    void OnStopDump() override;
  
    // Returns true if MediaStreamAudioProcessor would modify the audio signal,
    // based on |properties|. If the audio signal would not be modified, there is
    // no need to instantiate a MediaStreamAudioProcessor and feed audio through
    // it. Doing so would waste a non-trivial amount of memory and CPU resources.
    static bool WouldModifyAudio(const AudioProcessingProperties& properties);
  
   protected:
    ~MediaStreamAudioProcessor() override;
  
   private:
    friend class MediaStreamAudioProcessorTest;
  
    FRIEND_TEST_ALL_PREFIXES(MediaStreamAudioProcessorTest,
                             TestAgcEnableDefaultAgc1);
    FRIEND_TEST_ALL_PREFIXES(MediaStreamAudioProcessorTest,
                             TestAgcEnableHybridAgc);
    FRIEND_TEST_ALL_PREFIXES(MediaStreamAudioProcessorTest,
                             TestAgcEnableHybridAgcSimdNotAllowed);
  
    // WebRtcPlayoutDataSource::Sink implementation.
    void OnPlayoutData(media::AudioBus* audio_bus,
                       int sample_rate,
                       int audio_delay_milliseconds) override;
    void OnPlayoutDataSourceChanged() override;
    void OnRenderThreadChanged() override;
  
    base::Optional<webrtc::AudioProcessing::Config>
    GetAudioProcessingModuleConfig() const {
      if (audio_processing_) {
        return audio_processing_->GetConfig();
      }
      return base::nullopt;
    }
  
    // This method is called on the libjingle thread.
    AudioProcessorStatistics GetStats(bool has_remote_tracks) override;
  
    // Helper to initialize the WebRtc AudioProcessing.
    void InitializeAudioProcessingModule(
        const AudioProcessingProperties& properties);
  
    // Helper to initialize the capture converter.
    void InitializeCaptureFifo(const media::AudioParameters& input_format);
  
    // Called by ProcessAndConsumeData().
    // Returns the new microphone volume in the range of |0, 255].
    // When the volume does not need to be updated, it returns 0.
    int ProcessData(const float* const* process_ptrs,
                    int process_frames,
                    base::TimeDelta capture_delay,
                    int volume,
                    bool key_pressed,
                    float* const* output_ptrs);
  
    // Update AEC stats. Called on the main render thread.
    void UpdateAecStats();
  
    // Cached value for the render delay latency. This member is accessed by
    // both the capture audio thread and the render audio thread.
    base::subtle::Atomic32 render_delay_ms_;
  
    // For reporting audio delay stats.
    media::AudioDelayStatsReporter audio_delay_stats_reporter_;
  
    // Low-priority task queue for doing AEC dump recordings. It has to
    // out-live audio_processing_ and be created/destroyed from the same
    // thread.
    std::unique_ptr<rtc::TaskQueue> worker_queue_;
  
    // Module to handle processing and format conversion.
    std::unique_ptr<webrtc::AudioProcessing> audio_processing_;
  
    // FIFO to provide 10 ms capture chunks.
    std::unique_ptr<MediaStreamAudioFifo> capture_fifo_;
    // Receives processing output.
    std::unique_ptr<MediaStreamAudioBus> output_bus_;
  
    // Indicates whether the audio processor playout signal has ever had
    // asymmetric left and right channel content.
    bool assume_upmixed_mono_playout_ = true;
  
    // These are mutated on the main render thread in OnCaptureFormatChanged().
    // The caller guarantees this does not run concurrently with accesses on the
    // capture audio thread.
    media::AudioParameters input_format_;
    media::AudioParameters output_format_;
  
    // Raw pointer to the WebRtcPlayoutDataSource, which is valid for the
    // lifetime of RenderThread.
    //
    // TODO(crbug.com/704136): Replace with Member at some point.
    WebRtcPlayoutDataSource* playout_data_source_;
  
    // Task runner for the main render thread.
    const scoped_refptr<base::SingleThreadTaskRunner> main_thread_runner_;
  
    // Used to DCHECK that some methods are called on the capture audio thread.
    THREAD_CHECKER(capture_thread_checker_);
    // Used to DCHECK that some methods are called on the render audio thread.
    THREAD_CHECKER(render_thread_checker_);
  
    // Flag to enable stereo channel mirroring.
    bool audio_mirroring_;
  
    // Typing detector. |typing_detected_| is used to show the result of typing
    // detection. It can be accessed by the capture audio thread and by the
    // libjingle thread which calls GetStats().
    std::unique_ptr<webrtc::TypingDetection> typing_detector_;
    base::subtle::Atomic32 typing_detected_;
  
    // Communication with browser for AEC dump.
    std::unique_ptr<AecDumpAgentImpl> aec_dump_agent_impl_;
  
    // Flag to avoid executing Stop() more than once.
    bool stopped_;
  
    // Counters to avoid excessively logging errors in OnPlayoutData.
    size_t unsupported_buffer_size_log_count_ = 0;
    size_t apm_playout_error_code_log_count_ = 0;
    size_t large_delay_log_count_ = 0;
  
    // Flag indicating whether capture multi channel processing should be active.
    const bool use_capture_multi_channel_processing_;
  
    DISALLOW_COPY_AND_ASSIGN(MediaStreamAudioProcessor);
  };
  
  }  // namespace blink
  
  #endif  // THIRD_PARTY_BLINK_RENDERER_MODULES_MEDIASTREAM_MEDIA_STREAM_AUDIO_PROCESSOR_H_
  