title: Decode audio and video and process both streams -- ffmpeg, sdl, opencv
body: <p>My goal is to proceed on audio and video of mpeg-2 file independently, and to keep synchronicity on both flows. Duration of video is about 1 or 2 minutes maximum.</p>
  
  <ol>
  <li><p>First, following this <a href="https://stackoverflow.com/questions/8187745/audio-output-with-video-processing-with-opencv">post</a> "opencv for reading videos (and do the process),ffmpeg for audio , and SDL used to play both" sounds perfect. I have done some modification on the code considering recent ffmpeg naming changes. Compilation with cmake on 64-bits machine is fine. I get an error "Unsupported codec [3]" when opening codec.
  The code is following.</p></li>
  <li><p>Second, I looking for code dealing with synchronicity on both flows.</p></li>
  </ol>
  
  <hr>
  
  <pre><code>#include "opencv/highgui.h"
  #include "opencv/cv.h"
  
  #ifndef INT64_C
  #define INT64_C(c) (c ## LL)
  #define UINT64_C(c) (c ## ULL)
  #endif
  
  extern "C"{
  #include &lt;SDL/SDL.h&gt;
  #include &lt;SDL/SDL_thread.h&gt;
  #include &lt;libavcodec/avcodec.h&gt;
  #include &lt;libavformat/avformat.h&gt;
  }
  
  #include &lt;iostream&gt;
  #include &lt;stdio.h&gt;
  #include &lt;malloc.h&gt;
  
  using namespace cv;
  
  #define SDL_AUDIO_BUFFER_SIZE 1024
  
  typedef struct PacketQueue
  {
     AVPacketList *first_pkt, *last_pkt;
     int nb_packets;
     int size;
     SDL_mutex *mutex;
     SDL_cond *cond;
  } PacketQueue;
  PacketQueue audioq;
  
  int audioStream = -1;
  int videoStream = -1;
  int quit = 0;
  
  SDL_Surface* screen = NULL;
  SDL_Surface* surface = NULL;
  
  AVFormatContext* pFormatCtx = NULL;
  AVCodecContext* aCodecCtx = NULL;
  AVCodecContext* pCodecCtx = NULL;
  
  void show_frame(IplImage* img){
     if (!screen){
        screen = SDL_SetVideoMode(img-&gt;width, img-&gt;height, 0, 0);
        if (!screen){
           fprintf(stderr, "SDL: could not set video mode - exiting\n");
           exit(1);
        }
     }
     // Assuming IplImage packed as BGR 24bits
     SDL_Surface* surface = SDL_CreateRGBSurfaceFrom((void*)img-&gt;imageData,
                                                     img-&gt;width,
                                                     img-&gt;height,
                                                     img-&gt;depth * img-&gt;nChannels,
                                                     img-&gt;widthStep,
                                                     0xff0000, 0x00ff00, 0x0000ff, 0
                                                    );
  
     SDL_BlitSurface(surface, 0, screen, 0);
     SDL_Flip(screen);
  }
  
  void packet_queue_init(PacketQueue *q){
     memset(q, 0, sizeof(PacketQueue));
     q-&gt;mutex = SDL_CreateMutex();
     q-&gt;cond = SDL_CreateCond();
  }
  
  int packet_queue_put(PacketQueue *q, AVPacket *pkt){
     AVPacketList *pkt1;
     if (av_dup_packet(pkt) &lt; 0){
        return -1;
     }
  
     pkt1 = (AVPacketList*) av_malloc(sizeof(AVPacketList));
     //pkt1 = (AVPacketList*) malloc(sizeof(AVPacketList));
     if (!pkt1) return -1;
     pkt1-&gt;pkt = *pkt;
     pkt1-&gt;next = NULL;
  
     SDL_LockMutex(q-&gt;mutex);
  
     if (!q-&gt;last_pkt)
        q-&gt;first_pkt = pkt1;
     else
        q-&gt;last_pkt-&gt;next = pkt1;
  
     q-&gt;last_pkt = pkt1;
     q-&gt;nb_packets++;
     q-&gt;size += pkt1-&gt;pkt.size;
     SDL_CondSignal(q-&gt;cond);
  
     SDL_UnlockMutex(q-&gt;mutex);
     return 0;
  }
  
  static int packet_queue_get(PacketQueue *q, AVPacket *pkt, int block){
     AVPacketList *pkt1;
     int ret;
  
     SDL_LockMutex(q-&gt;mutex);
     for (;;){
        if( quit){
           ret = -1;
           break;
        }
  
        pkt1 = q-&gt;first_pkt;
        if (pkt1){
           q-&gt;first_pkt = pkt1-&gt;next;
           if (!q-&gt;first_pkt)
              q-&gt;last_pkt = NULL;
  
           q-&gt;nb_packets--;
           q-&gt;size -= pkt1-&gt;pkt.size;
           *pkt = pkt1-&gt;pkt;
           av_free(pkt1);
           //free(pkt1);
           ret = 1;
           break;
        }
  
        else if (!block){
           ret = 0;
           break;
        }
        else{
           SDL_CondWait(q-&gt;cond, q-&gt;mutex);
        }
     }
  
     SDL_UnlockMutex(q-&gt;mutex);
     return ret;
  }
  
  int audio_decode_frame(AVCodecContext *aCodecCtx, uint8_t *audio_buf, int buf_size){
     static AVPacket pkt;
     static uint8_t *audio_pkt_data = NULL;
     static int audio_pkt_size = 0;
  
     int len1, data_size;
  
     for (;;){
        while (audio_pkt_size &gt; 0){
           data_size = buf_size;
           len1 = avcodec_decode_audio3(aCodecCtx, (int16_t*)audio_buf, &amp;data_size, &amp;pkt);
           if (len1 &lt; 0){
              // if error, skip frame
              audio_pkt_size = 0;
              break;
           }
           audio_pkt_data += len1;
           audio_pkt_size -= len1;
           if (data_size &lt;= 0){
              // No data yet, get more frames
              continue;
           }
           // We have data, return it and come back for more later
           return data_size;
       }
  
       if (pkt.data)
          av_free_packet(&amp;pkt);
       if (quit) return -1;
       if (packet_queue_get(&amp;audioq, &amp;pkt, 1) &lt; 0) return -1;
       audio_pkt_data = pkt.data;
       audio_pkt_size = pkt.size;
    }
  }
  
  void audio_callback(void *userdata, Uint8 *stream, int len){
    AVCodecContext *aCodecCtx = (AVCodecContext *)userdata;
    int len1, audio_size;
  
    static uint8_t audio_buf[(AVCODEC_MAX_AUDIO_FRAME_SIZE * 3) / 2];
    static unsigned int audio_buf_size = 0;
    static unsigned int audio_buf_index = 0;
  
    while (len &gt; 0){
       if (audio_buf_index &gt;= audio_buf_size){
          // We have already sent all our data; get more
          audio_size = audio_decode_frame(aCodecCtx, audio_buf, sizeof(audio_buf));
          if(audio_size &lt; 0){
             // If error, output silence
             audio_buf_size = 1024; // arbitrary?
             memset(audio_buf, 0, audio_buf_size);
          }
          else{
             audio_buf_size = audio_size;
          }
          audio_buf_index = 0;
      }
  
      len1 = audio_buf_size - audio_buf_index;
      if (len1 &gt; len)
         len1 = len;
      memcpy(stream, (uint8_t *)audio_buf + audio_buf_index, len1);
      len -= len1;
      stream += len1;
      audio_buf_index += len1;
    }
  }
  
       void setup_ffmpeg(char* filename)
       {
          if (avformat_open_input(&amp;pFormatCtx, filename, NULL, NULL) != 0){
             fprintf(stderr, "FFmpeg failed to open file %s!\n", filename);
             exit(-1);
          }
  
          if (av_find_stream_info(pFormatCtx) &lt; 0){
             fprintf(stderr, "FFmpeg failed to retrieve stream info!\n");
             exit(-1);
          }
  
          // Dump information about file onto standard error
          av_dump_format(pFormatCtx, 0, filename, 0);
  
          // Find the first video stream
          int i = 0;
          for (i; i &lt; pFormatCtx-&gt;nb_streams; i++){
             if (pFormatCtx-&gt;streams[i]-&gt;codec-&gt;codec_type == AVMEDIA_TYPE_VIDEO &amp;&amp; videoStream &lt; 0){
                videoStream = i;
             }
  
             if (pFormatCtx-&gt;streams[i]-&gt;codec-&gt;codec_type == AVMEDIA_TYPE_VIDEO &amp;&amp; audioStream &lt; 0){
                audioStream = i;
             }
          }
  
          if (videoStream == -1){
             fprintf(stderr, "No video stream found in %s!\n", filename);
             exit(-1);
          }
  
          if (audioStream == -1){
             fprintf(stderr, "No audio stream found in %s!\n", filename);
             exit(-1);
          }
  
          // Get a pointer to the codec context for the audio stream
          aCodecCtx = pFormatCtx-&gt;streams[audioStream]-&gt;codec;
  
          // Set audio settings from codec info
          SDL_AudioSpec wanted_spec;
          wanted_spec.freq = aCodecCtx-&gt;sample_rate;
          wanted_spec.format = AUDIO_S16SYS;
          wanted_spec.channels = aCodecCtx-&gt;channels;
          wanted_spec.silence = 0;
          wanted_spec.samples = SDL_AUDIO_BUFFER_SIZE;
          wanted_spec.callback = audio_callback;
          wanted_spec.userdata = aCodecCtx;
  
          SDL_AudioSpec spec;
          if (SDL_OpenAudio(&amp;wanted_spec, &amp;spec) &lt; 0){
             fprintf(stderr, "SDL_OpenAudio: %s\n", SDL_GetError());
             exit(-1);
          }
  
          AVCodec* aCodec = avcodec_find_decoder(aCodecCtx-&gt;codec_id);
          if (!aCodec){
             fprintf(stderr, "Unsupported codec [1]!\n");
             exit(-1);
          }
          avcodec_open(aCodecCtx, aCodec);
  
          // audio_st = pFormatCtx-&gt;streams[index]
          packet_queue_init(&amp;audioq);
          SDL_PauseAudio(0);
  
          // Get a pointer to the codec context for the video stream
          pCodecCtx = pFormatCtx-&gt;streams[videoStream]-&gt;codec;
  
          // Find the decoder for the video stream
          AVCodec* pCodec = avcodec_find_decoder(pCodecCtx-&gt;codec_id);
          if (pCodec == NULL){
             fprintf(stderr, "Unsupported codec [2]!\n");
             exit(-1); // Codec not found
          }
  
          // Open codec
          if (avcodec_open(pCodecCtx, pCodec) &lt; 0){
             fprintf(stderr, "Unsupported codec [3]!\n");
             exit(-1); // Could not open codec
          }
       }
  
  
       int main(int argc, char* argv[])
       {
          if (argc &lt; 2){
              std::cout &lt;&lt; "Usage: " &lt;&lt; argv[0] &lt;&lt; " &lt;video&gt;" &lt;&lt; std::endl;
              return -1;
          }
  
          av_register_all();
  
          // Init SDL
          if (SDL_Init(SDL_INIT_VIDEO | SDL_INIT_AUDIO | SDL_INIT_TIMER))
          {
             fprintf(stderr, "Could not initialize SDL - %s\n", SDL_GetError());
             return -1;
          }
  
          // Init ffmpeg and setup some SDL stuff related to Audio
          setup_ffmpeg(argv[1]);
  
          VideoCapture cap(argv[1]);
          if (!cap.isOpened()){
             std::cout &lt;&lt; "Failed to load file!" &lt;&lt; std::endl;
             return -1;
          }
  
          AVPacket packet;
          while (av_read_frame(pFormatCtx, &amp;packet) &gt;= 0)
          {
             if (packet.stream_index == videoStream)
             {
                // Actually this is were SYNC between audio/video would happen.
                // Right now I assume that every VIDEO packet contains an entire video frame, and that's not true. A video frame can be made by multiple packets!
                // But for the time being, assume 1 video frame == 1 video packet,
                // so instead of reading the frame through ffmpeg, I read it through OpenCV.
  
                Mat frame;
                cap &gt;&gt; frame; // get a new frame from camera
  
                // do some processing on the frame, either as a Mat or as IplImage.
                // For educational purposes, applying a lame grayscale conversion
                IplImage ipl_frame = frame;
                for (int i = 0; i &lt; ipl_frame.width * ipl_frame.height * ipl_frame.nChannels; i += ipl_frame.nChannels)
                {
                   ipl_frame.imageData[i] = (ipl_frame.imageData[i] + ipl_frame.imageData[i+1] + ipl_frame.imageData[i+2])/3;   //B
                   ipl_frame.imageData[i+1] = (ipl_frame.imageData[i] + ipl_frame.imageData[i+1] + ipl_frame.imageData[i+2])/3; //G
                   ipl_frame.imageData[i+2] = (ipl_frame.imageData[i] + ipl_frame.imageData[i+1] + ipl_frame.imageData[i+2])/3; //R
                }
  
                // Display it on SDL window
                show_frame(&amp;ipl_frame);
  
                av_free_packet(&amp;packet);
             }
             else if (packet.stream_index == audioStream)
             {
                packet_queue_put(&amp;audioq, &amp;packet);
             }
             else
             {
                av_free_packet(&amp;packet);
             }
  
             SDL_Event event;
             SDL_PollEvent(&amp;event);
             switch (event.type)
             {
             case SDL_QUIT:
                SDL_FreeSurface(surface);
                SDL_Quit();
                break;
  
             default:
                break;
             }
          }
  
          // the camera will be deinitialized automatically in VideoCapture destructor
  
          // Close the codec
          avcodec_close(pCodecCtx);
  
          // Close the video file
          av_close_input_file(pFormatCtx);
  
          return 0;
       }
  </code></pre>