title: How to mux raw PCM with h.264 to MP4 with ffmpeg API?
body: <p>I'm doing some stuff with real-time multimedia transporting with FFmpeg API. Due to the requests of transporting audio/video data separately, so I need to mix them into an mp4 file at the receiver side.</p>
  
  <p>Now I can mux PCM and H.264 into mp4 file, but when playing, only images come out, the audio can't.
  PCM raw data attribute: 8000 sample rate, mono channel, 16 bit.</p>
  
  <p>Can anybody give some advice to me? Thanks a lot.</p>
  
  <p>I've tried to add WAV header to in_pcm_file, and make sure the pcm file can be played by Windows Media Player. But still not working in MP4 file.</p>
  
  <pre><code>int muxerData(char *in_h264_file,
                char *in_pcm_file,
                char *out_mp4_file,
                char *angle)
  {
      AVFormatContext *ifmt_ctx_v = NULL;
      AVFormatContext *ifmt_ctx_a = NULL;
      AVFormatContext *ofmt_ctx = NULL;
      AVFormatContext *ifmt_ctx = NULL;
  
      AVOutputFormat *ofmt = NULL;
  
      AVPacket pkt = {0};
      AVCodec *dec = NULL;
  
      AVStream *in_stream = NULL;
      AVStream *out_stream = NULL;
  
      int ret = 0;
      unsigned int i = 0;
      int videoindex_v = -1;
      int videoindex_out = -1;
      int audioindex_a = -1;
      int audioindex_out = -1;
      int frame_index = 0;
      int64_t cur_pts_v = 0;
      int64_t cur_pts_a = 0;
  
      int stream_index = 0;
  
      int compare_tag = -1;
  
      char log_buf[1024] = {0};
  
      avcodec_register_all();
      av_register_all();
  
      //Input
      if ((ret = avformat_open_input(&amp;ifmt_ctx_a, in_pcm_file, NULL, NULL)) &lt; 0)
      {
          av_strerror(ret, log_buf, 1024);
          printf("Couldn't open file %s: %d(%s)", in_pcm_file, ret, log_buf);
          goto end;
      }
  
      if ((ret = avformat_find_stream_info(ifmt_ctx_a, 0)) &lt; 0)
      {
          printf("Failed to retrieve input stream information");
          //if (acc_length&gt;0)
          //  goto end;
      }
  
      if ((ret = avformat_open_input(&amp;ifmt_ctx_v, in_h264_file, NULL, NULL)) &lt; 0)
      {
          printf("Could not open input file:%d\n", ret);
          goto end;
      }
  
      if ((ret = avformat_find_stream_info(ifmt_ctx_v, 0)) &lt; 0)
      {
          printf("Failed to retrieve input stream information");
          goto end;
      }
  
      av_dump_format(ifmt_ctx_v, 0, in_h264_file, 0);
      av_dump_format(ifmt_ctx_a, 0, in_pcm_file, 0);
  
      //Output
      avformat_alloc_output_context2(&amp;ofmt_ctx, NULL, NULL, out_mp4_file);
      if (!ofmt_ctx)
      {
          printf("Could not create output context\n");
          ret = AVERROR_UNKNOWN;
          goto end;
      }
  
      ofmt = ofmt_ctx-&gt;oformat;
  
      for (i = 0; i &lt; ifmt_ctx_v-&gt;nb_streams; i++)
      {
          //Create output AVStream according to input AVStream
          if (ifmt_ctx_v-&gt;streams[i]-&gt;codec-&gt;codec_type == AVMEDIA_TYPE_VIDEO)
          {
              in_stream = ifmt_ctx_v-&gt;streams[i];
              out_stream = avformat_new_stream(ofmt_ctx, in_stream-&gt;codec-&gt;codec);
              videoindex_v = i;
  
              if (!out_stream)
              {
                  printf("Failed allocating output stream\n");
                  ret = AVERROR_UNKNOWN;
                  goto end;
              }
  
              videoindex_out = out_stream-&gt;index;
  
              //Copy the settings of AVCodecContext
              ret = av_dict_set(&amp;out_stream-&gt;metadata, "rotate", angle, 0);
  
              if (avcodec_copy_context(out_stream-&gt;codec, in_stream-&gt;codec) &lt; 0)
              {
                  printf("Failed to copy context from input to output stream codec context\n");
                  goto end;
              }
  
              out_stream-&gt;codec-&gt;codec_tag = 0;
              if (ofmt_ctx-&gt;oformat-&gt;flags &amp; AVFMT_GLOBALHEADER)
              {
                  out_stream-&gt;codec-&gt;flags |= CODEC_FLAG_GLOBAL_HEADER;
              }
  
              break;
          }
      }
  
  
      for (i = 0; i &lt; ifmt_ctx_a-&gt;nb_streams; i++)
      {
          printf("===========acc=====from======:%d\n", ifmt_ctx_a-&gt;nb_streams);
  
          //Create output AVStream according to input AVStream
          if (ifmt_ctx_a-&gt;streams[i]-&gt;codec-&gt;codec_type == AVMEDIA_TYPE_AUDIO)
          {
              in_stream = ifmt_ctx_a-&gt;streams[i];
              out_stream = avformat_new_stream(ofmt_ctx, in_stream-&gt;codec-&gt;codec);
              audioindex_a = i;
  
              if (!out_stream)
              {
                  printf("Failed allocating output stream\n");
                  ret = AVERROR_UNKNOWN;
                  goto end;
              }
  
              audioindex_out = out_stream-&gt;index;
  
              //Copy the settings of AVCodecContext
              if (avcodec_copy_context(out_stream-&gt;codec, in_stream-&gt;codec) &lt; 0)
              {
                  printf("Failed to copy context from input to output stream codec context\n");
                  goto end;
              }
  
              if (ofmt_ctx-&gt;oformat-&gt;flags &amp; AVFMT_GLOBALHEADER)
              {
                  out_stream-&gt;codec-&gt;flags |= CODEC_FLAG_GLOBAL_HEADER;
              }
  
              break;
          }
      }
  
  
      printf("==========Output Information==========\n");
      av_dump_format(ofmt_ctx, 0, out_mp4_file, 1);
      printf("======================================\n");
  
      //Open output file
      if (!(ofmt-&gt;flags &amp; AVFMT_NOFILE))
      {
          if (avio_open(&amp;ofmt_ctx-&gt;pb, out_mp4_file, AVIO_FLAG_WRITE) &lt; 0)
          {
              printf("Could not open output file '%s'", out_mp4_file);
              goto end;
          }
      }
  
      //Write file header
      int header_ret = avformat_write_header(ofmt_ctx, NULL);
      if (header_ret &lt; 0)
      {
          av_strerror(header_ret, log_buf, 1024);
          printf("Error occurred when opening output file:%d (%s)\n", header_ret, log_buf);
          goto end;
      }
  
      //FIX
      //AVBitStreamFilterContext* h264bsfc = av_bitstream_filter_init("h264_mp4toannexb");
      //AVBitStreamFilterContext* aacbsfc = av_bitstream_filter_init("aac_adtstoasc");
  
  
      while (1)
      {
          //Get an AVPacket
          //if (acc_length&gt;0)
          //{
          compare_tag = av_compare_ts(cur_pts_v,
                                      ifmt_ctx_v-&gt;streams[videoindex_v]-&gt;time_base,
                                      cur_pts_a,
                                      ifmt_ctx_a-&gt;streams[audioindex_a]-&gt;time_base);
          //}
  
          if (compare_tag &lt;= 0)
          {
              ifmt_ctx = ifmt_ctx_v;
              stream_index = videoindex_out;
  
              if (av_read_frame(ifmt_ctx, &amp;pkt) &gt;= 0)
              {
                  do
                  {
                      in_stream = ifmt_ctx-&gt;streams[pkt.stream_index];
                      out_stream = ofmt_ctx-&gt;streams[stream_index];
  
                      if (pkt.stream_index == videoindex_v)
                      {
                          //FIX No PTS (Example: Raw H.264)
                          //Simple Write PTS
                          if (pkt.pts == AV_NOPTS_VALUE)
                          {
                              //Write PTS
                              AVRational time_base1 = in_stream-&gt;time_base;
                              //Duration between 2 frames (us)
                              int64_t calc_duration = (double)AV_TIME_BASE / av_q2d(in_stream-&gt;r_frame_rate);
                              //Parameters
                              pkt.pts = (double)(frame_index*calc_duration) / (double)(av_q2d(time_base1)*AV_TIME_BASE);
                              pkt.dts = pkt.pts;
                              pkt.duration = (double)calc_duration / (double)(av_q2d(time_base1)*AV_TIME_BASE);
                              frame_index++;
                          }
  
                          cur_pts_v = pkt.pts;
                          break;
                      }
                  } while (av_read_frame(ifmt_ctx, &amp;pkt) &gt;= 0);
              }
              else
              {
                  break;
              }
          }
          else
          {
              ifmt_ctx = ifmt_ctx_a;
              stream_index = audioindex_out;
              if (av_read_frame(ifmt_ctx, &amp;pkt) &gt;= 0)
              {
                  do
                  {
                      in_stream = ifmt_ctx-&gt;streams[pkt.stream_index];
                      out_stream = ofmt_ctx-&gt;streams[stream_index];
  
                      if (pkt.stream_index == audioindex_a)
                      {
                          //FIX No PTS
                          //Simple Write PTS
                          if (pkt.pts == AV_NOPTS_VALUE)
                          {
                              //Write PTS
                              AVRational time_base1 = in_stream-&gt;time_base;
                              //Duration between 2 frames (us)
                              int64_t calc_duration = (double)AV_TIME_BASE / av_q2d(in_stream-&gt;r_frame_rate);
                              //Parameters
                              pkt.pts = (double)(frame_index*calc_duration) / (double)(av_q2d(time_base1)*AV_TIME_BASE);
                              pkt.dts = pkt.pts;
                              pkt.duration = (double)calc_duration / (double)(av_q2d(time_base1)*AV_TIME_BASE);
                              frame_index++;
                          }
                          cur_pts_a = pkt.pts;
  
                          break;
                      }
                  } while (av_read_frame(ifmt_ctx, &amp;pkt) &gt;= 0);
              }
              else
              {
                  break;
              }
          }
  
          //FIX:Bitstream Filter
          //av_bitstream_filter_filter(h264bsfc, in_stream-&gt;codec, NULL, &amp;pkt.data, &amp;pkt.size, pkt.data, pkt.size, 0);
          //av_bitstream_filter_filter(aacbsfc, out_stream-&gt;codec, NULL, &amp;pkt.data, &amp;pkt.size, pkt.data, pkt.size, 0);
  
          //Convert PTS/DTS
          pkt.pts = av_rescale_q_rnd(pkt.pts,
                                     in_stream-&gt;time_base,
                                     out_stream-&gt;time_base,
                                     (AVRounding)(AV_ROUND_NEAR_INF | AV_ROUND_PASS_MINMAX));
  
          pkt.dts = av_rescale_q_rnd(pkt.dts,
                                     in_stream-&gt;time_base,
                                     out_stream-&gt;time_base,
                                     (AVRounding)(AV_ROUND_NEAR_INF | AV_ROUND_PASS_MINMAX));
  
          pkt.duration = av_rescale_q(pkt.duration, in_stream-&gt;time_base, out_stream-&gt;time_base);
          pkt.pos = -1;
          pkt.stream_index = stream_index;
  
          printf("Write 1 Packet. size:%5d\tpts:%lld\n", pkt.size, pkt.pts);
  
          if (av_interleaved_write_frame(ofmt_ctx, &amp;pkt) &lt; 0)
          {
              printf("Error muxing packet\n");
              break;
          }
  
          av_free_packet(&amp;pkt);
      }
  
      //Write file trailer
      av_write_trailer(ofmt_ctx);
  
      //av_bitstream_filter_close(h264bsfc);
      //av_bitstream_filter_close(aacbsfc);
  
  
  end:
  
      avformat_close_input(&amp;ifmt_ctx_v);
      avformat_close_input(&amp;ifmt_ctx_a);
  
      /* close output */
      if (ofmt_ctx &amp;&amp; !(ofmt-&gt;flags &amp; AVFMT_NOFILE))
      {
          avio_close(ofmt_ctx-&gt;pb);
      }
  
      avformat_free_context(ofmt_ctx);
  
      if ((ret &lt; 0) &amp;&amp; (ret != AVERROR_EOF))
      {
          printf("Error occurred.\n");
          return -1;
      }
  
      printf("======muxer mp4 success =====!\n");
      return 0;
  }
  </code></pre>