title: how to stream h.264 video with mp3 audio using libavcodec?
body: <p>I read h.264 frames from webcamera and capture audio from microphone. I need to stream live video to ffserver. During debug I read video from ffserver using ffmpeg with following command:</p>
  
  <pre><code>ffmpeg -i http://127.0.0.1:12345/robot.avi -vcodec copy -acodec copy out.avi
  </code></pre>
  
  <p>My video in output file is slightly accelerated. If I add a audio stream it is accelerated several times. Sometimes there is no audio in the output file.</p>
  
  <p>Here is my code for encoding audio:</p>
  
  <pre><code>#include "v_audio_encoder.h"
  
  extern "C" {
  #include &lt;libavcodec/avcodec.h&gt;
  }
  #include &lt;cassert&gt;
  
  struct VAudioEncoder::Private
  {
      AVCodec *m_codec;
      AVCodecContext *m_context;
  
      std::vector&lt;uint8_t&gt; m_outBuffer;
  };
  
  VAudioEncoder::VAudioEncoder( int sampleRate, int bitRate )
  {
      d = new Private( );
      d-&gt;m_codec = avcodec_find_encoder( CODEC_ID_MP3 );
      assert( d-&gt;m_codec );
      d-&gt;m_context = avcodec_alloc_context3( d-&gt;m_codec );
  
      // put sample parameters
      d-&gt;m_context-&gt;channels = 2;
      d-&gt;m_context-&gt;bit_rate = bitRate;
      d-&gt;m_context-&gt;sample_rate = sampleRate;
      d-&gt;m_context-&gt;sample_fmt = AV_SAMPLE_FMT_S16;
      strcpy( d-&gt;m_context-&gt;codec_name, "libmp3lame" );
  
      // open it
      int res = avcodec_open2( d-&gt;m_context, d-&gt;m_codec, 0 );
      assert( res &gt;= 0 );
  
      d-&gt;m_outBuffer.resize( d-&gt;m_context-&gt;frame_size );
  }
  
  VAudioEncoder::~VAudioEncoder( )
  {
      avcodec_close( d-&gt;m_context );
      av_free( d-&gt;m_context );
      delete d;
  }
  
  void VAudioEncoder::encode( const std::vector&lt;uint32_t&gt;&amp; samples, std::vector&lt;uint8_t&gt;&amp; outbuf )
  {
      assert( (int)samples.size( ) == d-&gt;m_context-&gt;frame_size );
  
      int outSize = avcodec_encode_audio( d-&gt;m_context, d-&gt;m_outBuffer.data( ),
                                          d-&gt;m_outBuffer.size( ), reinterpret_cast&lt;const short*&gt;( samples.data( ) ) );
      if( outSize ) {
          outbuf.resize( outSize );
          memcpy( outbuf.data( ), d-&gt;m_outBuffer.data( ), outSize );
      }
      else
          outbuf.clear( );
  }
  
  int VAudioEncoder::getFrameSize( ) const
  {
      return d-&gt;m_context-&gt;frame_size;
  }
  </code></pre>
  
  <p>Here is my code for streaming video:</p>
  
  <pre><code>#include "v_out_video_stream.h"
  
  extern "C" {
  #include &lt;libavformat/avformat.h&gt;
  #include &lt;libavutil/opt.h&gt;
  #include &lt;libavutil/avstring.h&gt;
  #include &lt;libavformat/avio.h&gt;
  }
  
  #include &lt;stdexcept&gt;
  #include &lt;cassert&gt;
  
  struct VStatticRegistrar
  {
      VStatticRegistrar( )
      {
          av_register_all( );
          avformat_network_init( );
      }
  };
  
  VStatticRegistrar __registrar;
  
  struct VOutVideoStream::Private
  {
      AVFormatContext * m_context;
      int m_videoStreamIndex;
      int m_audioStreamIndex;
  
      int m_videoBitrate;
      int m_width;
      int m_height;
      int m_fps;
      int m_bitrate;
  
      bool m_waitKeyFrame;
  };
  
  VOutVideoStream::VOutVideoStream( int width, int height, int fps, int bitrate )
  {
      d = new Private( );
      d-&gt;m_width = width;
      d-&gt;m_height = height;
      d-&gt;m_fps = fps;
      d-&gt;m_context = 0;
      d-&gt;m_videoStreamIndex = -1;
      d-&gt;m_audioStreamIndex = -1;
      d-&gt;m_bitrate = bitrate;
      d-&gt;m_waitKeyFrame = true;
  }
  
  bool VOutVideoStream::connectToServer( const std::string&amp; uri )
  {
      assert( ! d-&gt;m_context );
  
      // initalize the AV context
      d-&gt;m_context = avformat_alloc_context();
      if( !d-&gt;m_context )
          return false;
      // get the output format
      d-&gt;m_context-&gt;oformat = av_guess_format( "ffm", NULL, NULL );
      if( ! d-&gt;m_context-&gt;oformat )
          return false;
  
      strcpy( d-&gt;m_context-&gt;filename, uri.c_str( ) );
  
      // add an H.264 stream
      AVStream *stream = avformat_new_stream( d-&gt;m_context, NULL );
      if ( ! stream )
          return false;
      // initalize codec
      AVCodecContext* codec = stream-&gt;codec;
      if( d-&gt;m_context-&gt;oformat-&gt;flags &amp; AVFMT_GLOBALHEADER )
          codec-&gt;flags |= CODEC_FLAG_GLOBAL_HEADER;
      codec-&gt;codec_id = CODEC_ID_H264;
      codec-&gt;codec_type = AVMEDIA_TYPE_VIDEO;
      strcpy( codec-&gt;codec_name, "libx264" );
  //    codec-&gt;codec_tag = ( unsigned('4') &lt;&lt; 24 ) + (unsigned('6') &lt;&lt; 16 ) + ( unsigned('2') &lt;&lt; 8 ) + 'H';
      codec-&gt;width = d-&gt;m_width;
      codec-&gt;height = d-&gt;m_height;
      codec-&gt;time_base.den = d-&gt;m_fps;
      codec-&gt;time_base.num = 1;
      codec-&gt;bit_rate = d-&gt;m_bitrate;
      d-&gt;m_videoStreamIndex = stream-&gt;index;
  
      // add an MP3 stream
      stream = avformat_new_stream( d-&gt;m_context, NULL );
      if ( ! stream )
          return false;
      // initalize codec
      codec = stream-&gt;codec;
      if( d-&gt;m_context-&gt;oformat-&gt;flags &amp; AVFMT_GLOBALHEADER )
          codec-&gt;flags |= CODEC_FLAG_GLOBAL_HEADER;
      codec-&gt;codec_id = CODEC_ID_MP3;
      codec-&gt;codec_type = AVMEDIA_TYPE_AUDIO;
      strcpy( codec-&gt;codec_name, "libmp3lame" );
      codec-&gt;sample_fmt = AV_SAMPLE_FMT_S16;
      codec-&gt;channels = 2;
      codec-&gt;bit_rate = 64000;
      codec-&gt;sample_rate = 44100;
      d-&gt;m_audioStreamIndex = stream-&gt;index;
  
      // try to open the stream
      if( avio_open( &amp;d-&gt;m_context-&gt;pb, d-&gt;m_context-&gt;filename, AVIO_FLAG_WRITE ) &lt; 0 )
           return false;
  
      // write the header
      return avformat_write_header( d-&gt;m_context, NULL ) == 0;
  }
  
  void VOutVideoStream::disconnect( )
  {
      assert( d-&gt;m_context );
  
      avio_close( d-&gt;m_context-&gt;pb );
      avformat_free_context( d-&gt;m_context );
      d-&gt;m_context = 0;
  }
  
  VOutVideoStream::~VOutVideoStream( )
  {
      if( d-&gt;m_context )
          disconnect( );
      delete d;
  }
  
  int VOutVideoStream::getVopType( const std::vector&lt;uint8_t&gt;&amp; image )
  {
      if( image.size( ) &lt; 6 )
          return -1;
      unsigned char *b = (unsigned char*)image.data( );
  
      // Verify NAL marker
      if( b[ 0 ] || b[ 1 ] || 0x01 != b[ 2 ] ) {
          ++b;
          if ( b[ 0 ] || b[ 1 ] || 0x01 != b[ 2 ] )
              return -1;
      }
  
      b += 3;
  
      // Verify VOP id
      if( 0xb6 == *b ) {
          ++b;
          return ( *b &amp; 0xc0 ) &gt;&gt; 6;
      }
  
      switch( *b ) {
      case 0x65: return 0;
      case 0x61: return 1;
      case 0x01: return 2;
      }
  
      return -1;
  }
  
  bool VOutVideoStream::sendVideoFrame( std::vector&lt;uint8_t&gt;&amp; image )
  {
      // Init packet
      AVPacket pkt;
      av_init_packet( &amp;pkt );
      pkt.flags |= ( 0 &gt;= getVopType( image ) ) ? AV_PKT_FLAG_KEY : 0;
  
      // Wait for key frame
      if ( d-&gt;m_waitKeyFrame ) {
          if( pkt.flags &amp; AV_PKT_FLAG_KEY )
              d-&gt;m_waitKeyFrame = false;
          else
              return true;
      }
  
      pkt.stream_index = d-&gt;m_videoStreamIndex;
      pkt.data = image.data( );
      pkt.size = image.size( );
      pkt.pts = pkt.dts = AV_NOPTS_VALUE;
  
      return av_write_frame( d-&gt;m_context, &amp;pkt ) &gt;= 0;
  }
  
  bool VOutVideoStream::sendAudioFrame( std::vector&lt;uint8_t&gt;&amp; audio )
  {
      // Init packet
      AVPacket pkt;
      av_init_packet( &amp;pkt );
      pkt.stream_index = d-&gt;m_audioStreamIndex;
      pkt.data = audio.data( );
      pkt.size = audio.size( );
      pkt.pts = pkt.dts = AV_NOPTS_VALUE;
  
      return av_write_frame( d-&gt;m_context, &amp;pkt ) &gt;= 0;
  }
  </code></pre>
  
  <p>Here is how I use it:</p>
  
  <pre><code>BOOST_AUTO_TEST_CASE(testSendingVideo)
  {
      const int framesToGrab = 90000;
  
      VOutVideoStream stream( VIDEO_WIDTH, VIDEO_HEIGHT, FPS, VIDEO_BITRATE );
      if( stream.connectToServer( URI ) ) {
          VAudioEncoder audioEncoder( AUDIO_SAMPLE_RATE, AUDIO_BIT_RATE );
          VAudioCapture microphone( MICROPHONE_NAME, AUDIO_SAMPLE_RATE, audioEncoder.getFrameSize( ) );
  
          VLogitecCamera camera( VIDEO_WIDTH, VIDEO_HEIGHT );
          BOOST_REQUIRE( camera.open( CAMERA_PORT ) );
          BOOST_REQUIRE( camera.startCapturing( ) );
  
          std::vector&lt;uint8_t&gt; image, encodedAudio;
          std::vector&lt;uint32_t&gt; voice;
          boost::system_time startTime;
          int delta;
          for( int i = 0; i &lt; framesToGrab; ++i ) {
              startTime = boost::posix_time::microsec_clock::universal_time( );
  
              BOOST_REQUIRE( camera.read( image ) );
              BOOST_REQUIRE( microphone.read( voice ) );
              audioEncoder.encode( voice, encodedAudio );
  
              BOOST_REQUIRE( stream.sendVideoFrame( image ) );
              BOOST_REQUIRE( stream.sendAudioFrame( encodedAudio ) );
  
              delta = ( boost::posix_time::microsec_clock::universal_time( ) - startTime ).total_milliseconds( );
              if( delta &lt; 1000 / FPS )
                  boost::thread::sleep( startTime + boost::posix_time::milliseconds( 1000 / FPS - delta ) );
          }
  
          BOOST_REQUIRE( camera.stopCapturing( ) );
          BOOST_REQUIRE( camera.close( ) );
      }
      else
          std::cout &lt;&lt; "failed to connect to server" &lt;&lt; std::endl;
  }
  </code></pre>
  
  <p>I think my problem is in PTS and DTS. Can anyone help me?</p>