MP4中提取H.264码流



1.获取数据
ffmpeg读取mp4中的H264数据，并不能直接得到NALU，文件中也没有储存0x00000001的分隔符。下面这张图为packet.data中的数据


从图中可以发现，packet中的数据起始处没有分隔符(0x00000001), 也不是0x65、0x67、0x68、0x41等字节，所以可以肯定这不是标准的nalu。

其实，前4个字0x000032ce表示的是nalu的长度，从第5个字节开始才是nalu的数据。所以直接将前4个字节替换为0x00000001即可得到标准的nalu数据。


2.获取pps及sps

pps及sps不能从packet获得，而是保存在AVCodecContext的extradata数据域中。如下：




如何从extradata中解析出sps及pps呢？ffmpeg中提供了一个流过滤器"h264_mp4toannexb"完成这项工作，关键代码如下

[cpp] view plaincopyprint?

1. //h264_mp4toannexb_bsf.c
2. static int h264_mp4toannexb_filter(AVBitStreamFilterContext *bsfc,
3. AVCodecContext *avctx, const char *args,
4. uint8_t **poutbuf, int *poutbuf_size,
5. const uint8_t *buf, int buf_size,
6. int keyframe) {
7. H264BSFContext *ctx = bsfc->priv_data;
8. uint8_t unit_type;
9. int32_t nal_size;
10. uint32_t cumul_size = 0;
11. const uint8_t *buf_end = buf + buf_size;
13. if (!avctx->extradata || avctx->extradata_size < 6) {
14. *poutbuf = (uint8_t*) buf;
15. *poutbuf_size = buf_size;
16. return 0;
17. }
18. //
19. //从extradata中分析出SPS、PPS
20. //
22. if (!ctx->extradata_parsed) {
23. uint16_t unit_size;
24. uint64_t total_size = 0;
25. uint8_t *out = NULL, unit_nb, sps_done = 0, sps_seen = 0, pps_seen = 0;
26. const uint8_t *extradata = avctx->extradata+4;//跳过前4个字节
27. static const uint8_t nalu_header[4] = {0, 0, 0, 1};
29. ctx->length_size = (*extradata++ & 0x3) + 1; //用于指示表示编码数据长度所需字节数
30. if (ctx->length_size == 3)
31. return AVERROR(EINVAL);
33. unit_nb = *extradata++ & 0x1f;
34. if (!unit_nb) {
35. goto pps;
36. } else {
37. sps_seen = 1;
38. }
39. while (unit_nb--) {
40. void *tmp;
41. unit_size = AV_RB16(extradata);
42. total_size += unit_size+4;
43. if (total_size > INT_MAX - FF_INPUT_BUFFER_PADDING_SIZE ||
44. extradata+2+unit_size > avctx->extradata+avctx->extradata_size) {
45. av_free(out);
46. return AVERROR(EINVAL);
47. }
48. tmp = av_realloc(out, total_size + FF_INPUT_BUFFER_PADDING_SIZE);
49. if (!tmp) {
50. av_free(out);
51. return AVERROR(ENOMEM);
52. }
53. out = tmp;
54. memcpy(out+total_size-unit_size-4, nalu_header, 4);
55. memcpy(out+total_size-unit_size, extradata+2, unit_size);
56. extradata += 2+unit_size;
57. pps:
58. if (!unit_nb && !sps_done++) {
59. unit_nb = *extradata++;
60. if (unit_nb)
61. pps_seen = 1;
62. }
63. }
64. if(out)
65. memset(out + total_size, 0, FF_INPUT_BUFFER_PADDING_SIZE);
66. if (!sps_seen)
67. av_log(avctx, AV_LOG_WARNING, "Warning: SPS NALU missing or invalid. The resulting stream may not play.\n");
68. if (!pps_seen)
69. av_log(avctx, AV_LOG_WARNING, "Warning: PPS NALU missing or invalid. The resulting stream may not play.\n");
70. av_free(avctx->extradata);
71. avctx->extradata = out;
72. avctx->extradata_size = total_size;
73. ctx->first_idr = 1;
74. ctx->extradata_parsed = 1;
75. }
76. *poutbuf_size = 0;
77. *poutbuf = NULL;
78. do {
79. if (buf + ctx->length_size > buf_end)
80. goto fail; //buf为NULL时，以下代码将不再执行
81. //
82. //用于保存数据长度的字节数，是在分析原extradata计算出来的
83. //
84. if (ctx->length_size == 1) {
85. nal_size = buf[0];
86. } else if (ctx->length_size == 2) {
87. nal_size = AV_RB16(buf);
88. } else
89. nal_size = AV_RB32(buf);
90. buf += ctx->length_size;
91. unit_type = *buf & 0x1f;
92. if (buf + nal_size > buf_end || nal_size < 0)
93. goto fail;
95. if (ctx->first_idr && unit_type == 5) {
96. //
97. //copy IDR 帧时，需要将sps及pps一同拷贝
99. //
100. if (alloc_and_copy(poutbuf, poutbuf_size,
101. avctx->extradata, avctx->extradata_size,
102. buf, nal_size) < 0)
103. goto fail;
104. ctx->first_idr = 0;
105. } else {
106. //
107. //非IDR帧，没有sps及pps
108. if (alloc_and_copy(poutbuf, poutbuf_size,
109. NULL, 0,
110. buf, nal_size) < 0)
111. goto fail;
112. if (!ctx->first_idr && unit_type == 1)
113. ctx->first_idr = 1;
114. }
115. buf += nal_size;
116. cumul_size += nal_size + ctx->length_size;
117. } while (cumul_size < buf_size);
118. return 1;
119. fail:
120. av_freep(poutbuf);
121. *poutbuf_size = 0;
122. return AVERROR(EINVAL);
123. }

//h264_mp4toannexb_bsf.c static int h264_mp4toannexb_filter(AVBitStreamFilterContext<wbr> *bsfc, AVCodecContext *avctx, const char *args, uint8_t **poutbuf, int *poutbuf_size, const uint8_t *buf, int buf_size, int keyframe) { H264BSFContext *ctx = bsfc->priv_data; uint8_t unit_type; int32_t nal_size; uint32_t cumul_size = 0; const uint8_t *buf_end = buf + buf_size; if (!avctx->extradata || avctx->extradata_size < 6) { *poutbuf = (uint8_t*) buf; *poutbuf_size = buf_size; return 0; } // //从extradata中分析出SPS、PPS // if (!ctx->extradata_parsed) { uint16_t unit_size; uint64_t total_size = 0; uint8_t *out = NULL, unit_nb, sps_done = 0, sps_seen = 0, pps_seen = 0; const uint8_t *extradata = avctx->extradata+4; //跳过前4个字节 static const uint8_t nalu_header[4] = {0, 0, 0, 1}; ctx->length_size = (*extradata++ & 0x3) + 1; //用于指示表示编码数据长度所需字节数 if (ctx->length_size == 3) return AVERROR(EINVAL); unit_nb = *extradata++ & 0x1f; if (!unit_nb) { goto pps; } else { sps_seen = 1; } while (unit_nb--) { void *tmp; unit_size = AV_RB16(extradata); total_size += unit_size+4; if (total_size > INT_MAX - FF_INPUT_BUFFER_PADDING_SIZE || extradata+2+unit_size > avctx->extradata+avctx->extradata_size) { av_free(out); return AVERROR(EINVAL); } tmp = av_realloc(out, total_size + FF_INPUT_BUFFER_PADDING_SIZE); if (!tmp) { av_free(out); return AVERROR(ENOMEM); } out = tmp; memcpy(out+total_size-unit_size-4, nalu_header, 4); memcpy(out+total_size-unit_size, extradata+2, unit_size); extradata += 2+unit_size; pps: if (!unit_nb && !sps_done++) { unit_nb = *extradata++; if (unit_nb) pps_seen = 1; } } if(out) memset(out + total_size, 0, FF_INPUT_BUFFER_PADDING_SIZE); if (!sps_seen) av_log(avctx, AV_LOG_WARNING, "Warning: SPS NALU missing or invalid. The resulting stream may not play.\n"); if (!pps_seen) av_log(avctx, AV_LOG_WARNING, "Warning: PPS NALU missing or invalid. The resulting stream may not play.\n"); av_free(avctx->extradata); avctx->extradata = out; avctx->extradata_size = total_size; ctx->first_idr = 1; ctx->extradata_parsed = 1; } *poutbuf_size = 0; *poutbuf = NULL; do { if (buf + ctx->length_size > buf_end) goto fail; //buf为NULL时，以下代码将不再执行 // //用于保存数据长度的字节数，是在分析原extradata计算出来的 // if (ctx->length_size == 1) { nal_size = buf[0]; } else if (ctx->length_size == 2) { nal_size = AV_RB16(buf); } else nal_size = AV_RB32(buf); buf += ctx->length_size; unit_type = *buf & 0x1f; if (buf + nal_size > buf_end || nal_size < 0) goto fail; if (ctx->first_idr && unit_type == 5) { // //copy IDR 帧时，需要将sps及pps一同拷贝 // if (alloc_and_copy(poutbuf, poutbuf_size, avctx->extradata, avctx->extradata_size, buf, nal_size) < 0) goto fail; ctx->first_idr = 0; } else { // //非IDR帧，没有sps及pps if (alloc_and_copy(poutbuf, poutbuf_size, NULL, 0, buf, nal_size) < 0) goto fail; if (!ctx->first_idr && unit_type == 1) ctx->first_idr = 1; } buf += nal_size; cumul_size += nal_size + ctx->length_size; } while (cumul_size < buf_size); return 1; fail: av_freep(poutbuf); *poutbuf_size = 0; return AVERROR(EINVAL); }</wbr>

一般情况下，extradata中包含一个sps、一个pps 的nalu, 从上面的代码中容易看出extradata的数据格式。分析后的sps及pps依然储存在extradata域中，并添加了起始符。从代码中还可以看出，上面的函数会将sps、pps及packet中的数据，都copy到poutbuf指示的内存中，如果不需要copy到指定内存，直接给buf参数传入空值即可。




3.使用ffmpeg的流过滤器获取sps及pps
流过滤器"h264_mp4toannexb", 在av_register_all()函数中会被注册。用法示例如下：

[cpp] view plaincopyprint?

1. int ParseH264ExtraDataInMp4(int stream_id)
2. {
3. uint8_t *dummy = NULL;
4. int dummy_size;
5. AVBitStreamFilterContext* bsfc = av_bitstream_filter_init("h264_mp4toannexb");
6. if(bsfc == NULL)
7. {
8. return -1;
9. }
10. av_bitstream_filter_filter(
11. bsfc, format_ctx_->streams[stream_id]->codec, NULL, &dummy, &dummy_size, NULL, 0, 0);

int ParseH264ExtraDataInMp4(int stream_id) { uint8_t *dummy = NULL; int dummy_size; AVBitStreamFilterContext<wbr>* bsfc = av_bitstream_filter_init("h264_mp4toannexb"); if(bsfc == NULL) { return -1; } av_bitstream_filter_filter( bsfc, format_ctx_->streams[stream_id]->codec, NULL, &dummy, &dummy_size, NULL, 0, 0);</wbr>

[cpp] view plaincopyprint?

1. av_bitstream_filter_close(bsfc);
2. return 0;
3. }

 av_bitstream_filter_close(bsfc); return 0; }