Android使用Mp4v2用h264流和aac流合成mp4

来源：互联网发布：燕十八mysql优化编辑：程序博客网时间：2024/06/06 10:46

前言

在能够使用原生的情况下，博主是绝对不会尝试陌生的jni编程的。但是，偏偏android原生的MediaMuxer（合成器）使用有限制不说，在合成的时候出现各种问题，网上的可参考资料也少，绝大多数都是使用MediaMuxer用本地的h264数据和acc数据合成mp4,这对于博主来讲是没有多大用的。博主遇到的情况是要用h264流合成Mp4。

进展：

第一阶段 :添加视频

截至目前，博主根据网上资料，下载mp4v2的源代码，在ubuntu下面使用ndk生成libmp4v2.so包。再在AndroidStudio2.3上使用cmake进行jni编程。因为博主所在的公司，苹果端这一款进度是完成的差不多了，IOS大神直接丢给我已经封装好了的mp4record.c，所以在mp4v2这一块还是给我节省了很多时间的。但是,cmake毕竟没有接触过，熟悉这一款，加上网上找资料，快耗费我两个星期的时间才有所功效。到今天，自己的项目终于能使用本地h264裸流转成mp4，在此记录一下。等demo能够把音频也写入了再上传源码吧
使用的难点在于视频帧需要一帧一帧的往方法里面丢，以及对编码器的使用和理解。

第二阶段 :添加aac音频至mp4

由于很多朋友使用的时候不可能只要视频部分，所以我后面还是研究了一下添加音频部分。我这里添加的音频来至于AudioRecord录制的pcm编码成的aac数据，AudioRecord得到原始的音频数据然后利用MediaCodec 将音频数据硬编码成aac格式的音频流。录制音频的过程有很多的数据设置，比如采样率，以及给aac裸流添加adts头字段时候的一些参数设置都是有些关联的，设置的时候特别注意。c部分的代码，我就简单的把音频部分注释的代码打开了，再设置了一些参数

//这里的1024的值可以更改，设置小了会出现音频断断续续的情况 我之前设置的160，合成的音频断断续续recordCtx->m_aTrackId = MP4AddAudioTrack(recordCtx->m_mp4FHandle, 8000, 1024,                                             MP4_MPEG4_AUDIO_TYPE);

int mp4AEncode(MP4V2_CONTEXT *recordCtx, uint8_t *data, int len) {    if (recordCtx->m_vTrackId == MP4_INVALID_TRACK_ID) {        return -1;    }    MP4WriteSample(recordCtx->m_mp4FHandle, recordCtx->m_aTrackId, data, len, MP4_INVALID_DURATION,                   0, 1);//这里的最后一个参数表示音频是否同步，1表示同步    recordCtx->m_vFrameDur += 1024;    return 0;}

问题记录：
- 添加音频的时候没去去掉aac音频的adts头部的7个字节，导致只有部分播放器播放的时候有声音。我在查资料的时候也发现如果你包含了这个头，我测试下来迅雷播放器可以支持，但是百度影音、暴风影音放出来没声音。
- 所以后面在添加音频的时候偏移了7个字节，移除了adts的头部。而且采样率调整为8000，通道数调整为1的时候播放器播放才正常。

/** * 添加音频数据 */JNIEXPORT jint JNICALL Java_com_seuic_jni_Mp4v2Helper_mp4AEncode        (JNIEnv *env, jclass clz, jbyteArray data, jint size) {    unsigned char *buf = (unsigned char *) (*env)->GetByteArrayElements(env, data, JNI_FALSE);    int nalsize = size;    //减去7为了删除adts头部的7个字节    int reseult = mp4AEncode(_mp4Handle, &buf[7], nalsize-7);//    (*env)->ReleaseByteArrayElements(env, data, (jbyte *)buf, 0);    return reseult;}

源码地址

只录制视频部分：http://download.csdn.net/detail/chezi008/9862298
录制视频和音频代码：http://download.csdn.net/detail/chezi008/9881822

TestMp4Activity.java

public class TestMp4Activity extends Activity {    public String TAG = getClass().getName();    Button test;    private SurfaceView mSurface = null;    private SurfaceHolder mSurfaceHolder;    private Thread mDecodeThread;    private MediaCodec mCodec;    private boolean mStopFlag = false;    private DataInputStream mInputStream;    private String FileName = "mtv.h264";    private String audioFileName = "test.aac";    private static final int VIDEO_WIDTH = 1920;    private static final int VIDEO_HEIGHT = 1080;    private int FrameRate = 15;    private Boolean UseSPSandPPS = true;    private String filePath = Environment.getExternalStorageDirectory() + "/" + FileName;    String outFilepath = Environment.getExternalStorageDirectory().getAbsolutePath() + File.separator + "h264.mp4";    private String audioPath = Environment.getExternalStorageDirectory() + "/" + audioFileName;    private LinkedBlockingQueue<byte[]> mLinkedBlockingQueue;    @Override    protected void onCreate(Bundle savedInstanceState) {        super.onCreate(savedInstanceState);        setContentView(R.layout.main_layout);        mLinkedBlockingQueue = new LinkedBlockingQueue<>();        mSurface = (SurfaceView) findViewById(R.id.surfaceview);        test = (Button) findViewById(R.id.test);        test.setOnClickListener(new View.OnClickListener() {            @Override            public void onClick(View v) {                test();            }        });        try {            //获取文件输入流            mInputStream = new DataInputStream(new FileInputStream(new File(filePath)));        } catch (FileNotFoundException e) {            e.printStackTrace();            try {                mInputStream.close();            } catch (IOException e1) {                e1.printStackTrace();            }        }        mSurfaceHolder = mSurface.getHolder();        mSurfaceHolder.addCallback(new SurfaceHolder.Callback() {            @Override            public void surfaceCreated(SurfaceHolder holder) {                initCodec(holder);                iniAudioCodec();            }            @Override            public void surfaceChanged(SurfaceHolder holder, int format, int width, int height) {            }            @Override            public void surfaceDestroyed(SurfaceHolder holder) {                mCodec.stop();                mCodec.release();            }        });    }    private void test() {        startDecodingThread();    }    private class decodeThread implements Runnable {        @Override        public void run() {            try {                decodeLoop();            } catch (Exception e) {                Log.d("haha", "run: " + e.toString());                Mp4v2Helper.closeMp4Encoder();                Log.d("haha", "decodeLoop: end");            }        }        private void decodeLoop() {            //存放目标文件的数据            ByteBuffer[] inputBuffers = mCodec.getInputBuffers();            //解码后的数据，包含每一个buffer的元数据信息，例如偏差，在相关解码器中有效的数据大小            MediaCodec.BufferInfo info = new MediaCodec.BufferInfo();            long startMs = System.currentTimeMillis();            long timeoutUs = 10000;            byte[] marker0 = new byte[]{0, 0, 0, 1};            byte[] dummyFrame = new byte[]{0x00, 0x00, 0x01, 0x20};            byte[] streamBuffer = new byte[1024 * 1024 * 5];            try {                while (true) {                    int length = mInputStream.available();                    if (length > 0) {                        int count = mInputStream.read(streamBuffer);                        mStopFlag = false;                        int bytes_cnt = 0;                        while (mStopFlag == false) {                            bytes_cnt = streamBuffer.length;                            if (bytes_cnt == 0) {                                streamBuffer = dummyFrame;                            }                            int startIndex = 0;                            int remaining = bytes_cnt;                            while (true) {                                if (remaining == 0 || startIndex >= remaining) {                                    break;                                }                                int nextFrameStart = KMPMatch(marker0, streamBuffer, startIndex + 2, remaining);                                if (nextFrameStart == -1) {                                    nextFrameStart = remaining;                                } else {                                }                                int inIndex = mCodec.dequeueInputBuffer(timeoutUs);                                if (inIndex >= 0) {                                    byte[] newData = new byte[nextFrameStart - startIndex];                                    System.arraycopy(streamBuffer, startIndex, newData, 0, newData.length);                                    int i = Mp4v2Helper.mp4VEncode(newData, newData.length);                                    Log.d("haha", "decodeLoop: result" + i);                                    ByteBuffer byteBuffer = inputBuffers[inIndex];                                    byteBuffer.clear();                                    byteBuffer.put(streamBuffer, startIndex, nextFrameStart - startIndex);                                    //在给指定Index的inputbuffer[]填充数据后，调用这个函数把数据传给解码器                                    mCodec.queueInputBuffer(inIndex, 0, nextFrameStart - startIndex, 0, 0);                                    startIndex = nextFrameStart;                                } else {                                    continue;                                }                                int outIndex = mCodec.dequeueOutputBuffer(info, timeoutUs);                                if (outIndex >= 0) {                                    //帧控制是不在这种情况下工作，因为没有PTS H264是可用的                                    while (info.presentationTimeUs / 1000 > System.currentTimeMillis() - startMs) {                                        try {                                            Thread.sleep(100);                                        } catch (InterruptedException e) {                                            e.printStackTrace();                                        }                                    }                                    boolean doRender = (info.size != 0);                                    //对outputbuffer的处理完后，调用这个函数把buffer重新返回给codec类。                                    mCodec.releaseOutputBuffer(outIndex, doRender);                                } else {                                }                            }                            mStopFlag = true;                            Mp4v2Helper.closeMp4Encoder();                            Log.d("haha", "decodeLoop: end");                        }                    }                }            } catch (IOException e) {                e.printStackTrace();            }        }    }    int KMPMatch(byte[] pattern, byte[] bytes, int start, int remain) {        try {            Thread.sleep(30);        } catch (InterruptedException e) {            e.printStackTrace();        }        int[] lsp = computeLspTable(pattern);        int j = 0;  // Number of chars matched in pattern        for (int i = start; i < remain; i++) {            while (j > 0 && bytes[i] != pattern[j]) {                // Fall back in the pattern                j = lsp[j - 1];  // Strictly decreasing            }            if (bytes[i] == pattern[j]) {                // Next char matched, increment position                j++;                if (j == pattern.length)                    return i - (j - 1);            }        }        return -1;  // Not found    }    int[] computeLspTable(byte[] pattern) {        int[] lsp = new int[pattern.length];        lsp[0] = 0;  // Base case        for (int i = 1; i < pattern.length; i++) {            // Start by assuming we're extending the previous LSP            int j = lsp[i - 1];            while (j > 0 && pattern[i] != pattern[j])                j = lsp[j - 1];            if (pattern[i] == pattern[j])                j++;            lsp[i] = j;        }        return lsp;    }    private void initCodec(SurfaceHolder holder) {        try {            //通过多媒体格式名创建一个可用的解码器            mCodec = MediaCodec.createDecoderByType("video/avc");        } catch (IOException e) {            e.printStackTrace();        }        //初始化编码器        final MediaFormat mediaformat = MediaFormat.createVideoFormat("video/avc", VIDEO_WIDTH, VIDEO_HEIGHT);        //获取h264中的pps及sps数据        if (UseSPSandPPS) {            byte[] header_sps = {0, 0, 0, 1, 103, 66, 0, 42, (byte) 149, (byte) 168, 30, 0, (byte) 137, (byte) 249, 102, (byte) 224, 32, 32, 32, 64};            byte[] header_pps = {0, 0, 0, 1, 104, (byte) 206, 60, (byte) 128, 0, 0, 0, 1, 6, (byte) 229, 1, (byte) 151, (byte) 128};            mediaformat.setByteBuffer("csd-0", ByteBuffer.wrap(header_sps));            mediaformat.setByteBuffer("csd-1", ByteBuffer.wrap(header_pps));        }        //设置帧率        mediaformat.setInteger(MediaFormat.KEY_FRAME_RATE, FrameRate);        //https://developer.android.com/reference/android/media/MediaFormat.html#KEY_MAX_INPUT_SIZE        //设置配置参数，参数介绍 ：        // format   如果为解码器，此处表示输入数据的格式；如果为编码器，此处表示输出数据的格式。        //surface   指定一个surface，可用作decode的输出渲染。        //crypto    如果需要给媒体数据加密，此处指定一个crypto类.        //   flags  如果正在配置的对象是用作编码器，此处加上CONFIGURE_FLAG_ENCODE 标签。        mCodec.configure(mediaformat, holder.getSurface(), null, 0);//        startDecodingThread();        int i = Mp4v2Helper.initMp4Encoder(outFilepath, 1080, 720);        Log.d("hahah", i + "init");    }    private void iniAudioCodec() {        final AudioEncoder audioEncoder = new AudioEncoder();        try {            audioEncoder.setSavePath(audioPath);            audioEncoder.setAudioEnncoderListener(new AudioEncoder.AudioEnncoderListener() {                @Override                public void getAudioData(byte[] temp) {                    try {                        mLinkedBlockingQueue.put(temp);                    } catch (InterruptedException e) {                        e.printStackTrace();                    }                    if (mStopFlag) {                        audioEncoder.stop();                    }else{                        try {                            byte[] audioByte = mLinkedBlockingQueue.take();                            Mp4v2Helper.mp4AEncode(temp, temp.length);                        } catch (InterruptedException e) {                            e.printStackTrace();                        }                    }                }            });            audioEncoder.prepare();            audioEncoder.start();        } catch (IOException e) {            e.printStackTrace();        } catch (InterruptedException e) {            e.printStackTrace();        }    }    private void startDecodingThread() {        mCodec.start();        mDecodeThread = new Thread(new decodeThread());        mDecodeThread.start();    }}

AudioEncoder.java

/** * 描述： * 作者：chezi008 on 2017/6/26 10:14 * 邮箱：chezi008@qq.com */public class AudioEncoder implements Runnable {    private String TAG = getClass().getSimpleName();    private String mime = "audio/mp4a-latm";    private AudioRecord mRecorder;    private MediaCodec mEnc;    private int rate=25600;//9600    //录音设置    private int sampleRate=16000;   //采样率，默认44.1k    private int channelCount=2;     //音频采样通道，默认2通道    private int channelConfig= AudioFormat.CHANNEL_IN_STEREO;        //通道设置，默认立体声    private int audioFormat=AudioFormat.ENCODING_PCM_16BIT;     //设置采样数据格式，默认16比特PCM    private FileOutputStream fos;    private byte[] buffer;    private boolean isRecording;    private Thread mThread;    private int bufferSize;    private String mSavePath;    private AudioEnncoderListener audioEnncoderListener;    public AudioEncoder(){    }    public void setAudioEnncoderListener(AudioEnncoderListener audioEnncoderListener) {        this.audioEnncoderListener = audioEnncoderListener;    }    public void setMime(String mime){        this.mime=mime;    }    public void setRate(int rate){        this.rate=rate;    }    public void setSampleRate(int sampleRate){        this.sampleRate=sampleRate;    }    public void setSavePath(String path){        this.mSavePath=path;    }    public void prepare() throws IOException {        fos=new FileOutputStream(mSavePath);        //音频编码相关        MediaFormat format=MediaFormat.createAudioFormat(mime,sampleRate,channelCount);        format.setInteger(MediaFormat.KEY_AAC_PROFILE, MediaCodecInfo.CodecProfileLevel.AACObjectLC);//        format.setInteger(MediaFormat.KEY_CHANNEL_MASK, AudioFormat.CHANNEL_IN_MONO);        format.setInteger(MediaFormat.KEY_BIT_RATE, rate);        mEnc=MediaCodec.createEncoderByType(mime);        mEnc.configure(format,null,null,MediaCodec.CONFIGURE_FLAG_ENCODE);        //音频录制相关        bufferSize = AudioRecord.getMinBufferSize(sampleRate, channelConfig, audioFormat)*2;        buffer=new byte[bufferSize];        mRecorder=new AudioRecord(MediaRecorder.AudioSource.MIC,sampleRate,channelConfig,                audioFormat,bufferSize);    }    public void start() throws InterruptedException {        mEnc.start();        mRecorder.startRecording();        if(mThread!=null&&mThread.isAlive()){            isRecording=false;            mThread.join();        }        isRecording=true;        mThread=new Thread(this);        mThread.start();    }    private ByteBuffer getInputBuffer(int index){        if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.LOLLIPOP) {            return mEnc.getInputBuffer(index);        }else{            return mEnc.getInputBuffers()[index];        }    }    private ByteBuffer getOutputBuffer(int index){        if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.LOLLIPOP) {            return mEnc.getOutputBuffer(index);        }else{            return mEnc.getOutputBuffers()[index];        }    }    //TODO Add End Flag    private void readOutputData() throws IOException{        int index=mEnc.dequeueInputBuffer(-1);        if(index>=0){            final ByteBuffer buffer=getInputBuffer(index);            buffer.clear();            int length=mRecorder.read(buffer,bufferSize);            if(length>0){                mEnc.queueInputBuffer(index,0,length,System.nanoTime()/1000,0);            }else{                Log.e(TAG,"length-->"+length);            }        }        MediaCodec.BufferInfo mInfo=new MediaCodec.BufferInfo();        int outIndex;        do{            outIndex=mEnc.dequeueOutputBuffer(mInfo,0);            Log.e(TAG,"audio flag---->"+mInfo.flags+"/"+outIndex);            if(outIndex>=0){                ByteBuffer buffer=getOutputBuffer(outIndex);                buffer.position(mInfo.offset);                byte[] temp=new byte[mInfo.size+7];                buffer.get(temp,7,mInfo.size);                addADTStoPacket(temp,temp.length);                Log.d(TAG, "readOutputData: temp.length-->"+temp.length);                fos.write(temp);                audioEnncoderListener.getAudioData(temp);                mEnc.releaseOutputBuffer(outIndex,false);            }else if(outIndex ==MediaCodec.INFO_TRY_AGAIN_LATER){            }else if(outIndex==MediaCodec.INFO_OUTPUT_FORMAT_CHANGED){            }        }while (outIndex>=0);    }    /**     * 给编码出的aac裸流添加adts头字段     * @param packet 要空出前7个字节，否则会搞乱数据     * @param packetLen     */    private void addADTStoPacket(byte[] packet, int packetLen) {        int profile = 2;  //AAC LC        int freqIdx = 8;  //44.1KHz--4  这个参数跟采样率有关sampleRate，8000-->11 16000-->8 44100-->4        int chanCfg = 2;  //CPE  这个参数跟通道数有关channelCount   chanCfg = 这个参数跟通道数有关channelCount        packet[0] = (byte)0xFF;        packet[1] = (byte)0xF9;        packet[2] = (byte)(((profile-1)<<6) + (freqIdx<<2) +(chanCfg>>2));        packet[3] = (byte)(((chanCfg&3)<<6) + (packetLen>>11));        packet[4] = (byte)((packetLen&0x7FF) >> 3);        packet[5] = (byte)(((packetLen&7)<<5) + 0x1F);        packet[6] = (byte)0xFC;    }    /**     * 停止录制     */    public void stop(){        try {            isRecording=false;            mThread.join();            mRecorder.stop();            mEnc.stop();            mEnc.release();            fos.flush();            fos.close();        } catch (Exception e){            e.printStackTrace();        }    }    @Override    public void run() {        while (isRecording){            try {                readOutputData();//                fos.write(buffer,0,length);            } catch (IOException e) {                e.printStackTrace();            }        }    }    public interface AudioEnncoderListener{        void getAudioData(byte[] temp);    }}

Mp4v2Helper.java

/** * 描述： * 作者：chezi008 on 2017/4/12 16:14 * 邮箱：chezi008@163.com */public class Mp4v2Helper {    public static native int initMp4Encoder(String outputFilePath,int width,int height);    public static native int mp4VEncode(byte[] data, int size);    public static native int mp4AEncode(byte[] data, int size);    public static native void closeMp4Encoder();    static {        Log.i("NativeClass", "before load library");        System.loadLibrary("Mp4v2Helper");//注意这里为自己指定的.so文件，无lib前缀，亦无后缀        Log.i("NativeClass", "after load library");    }}

Mp4v2Helper.c

//// Created by Administrator on 2017/4/12.//#include <com_seuic_jni_Mp4v2Helper.h>#include "mp4record.h"#include "mp4record.c"MP4V2_CONTEXT *_mp4Handle;/** * 初始化 * @param env * @param jclass * @param path * @param width * @param height * @return */jint JNICALL Java_com_seuic_jni_Mp4v2Helper_initMp4Encoder        (JNIEnv *env, jclass jclass, jstring path, jint width, jint height) {    const char *local_title = (*env)->GetStringUTFChars(env, path, NULL);    int m_width = width;    int m_height = height;    _mp4Handle = initMp4Encoder(local_title, m_width, m_height);    return 0;}/** * 添加视频帧的方法 */JNIEXPORT jint JNICALL Java_com_seuic_jni_Mp4v2Helper_mp4VEncode        (JNIEnv *env, jclass clz, jbyteArray data, jint size) {    unsigned char *buf = (unsigned char *) (*env)->GetByteArrayElements(env, data, JNI_FALSE);    int nalsize = size;    int reseult = mp4VEncode(_mp4Handle, buf, nalsize);    return reseult;}/** * 添加音频数据 */JNIEXPORT jint JNICALL Java_com_seuic_jni_Mp4v2Helper_mp4AEncode        (JNIEnv *env, jclass clz, jbyteArray data, jint size) {    unsigned char *buf = (unsigned char *) (*env)->GetByteArrayElements(env, data, JNI_FALSE);    int nalsize = size;    int reseult = mp4AEncode(_mp4Handle, buf, nalsize);    return reseult;}/** * 释放 */JNIEXPORT void JNICALL Java_com_seuic_jni_Mp4v2Helper_closeMp4Encoder        (JNIEnv *env, jclass clz) {    closeMp4Encoder(_mp4Handle);}

CMakeLists.txt

cmake_minimum_required(VERSION 3.4.1)set(distribution_DIR ${CMAKE_SOURCE_DIR}/../../../../libs)add_library( mp4v2             SHARED             IMPORTED )set_target_properties( mp4v2                       PROPERTIES IMPORTED_LOCATION                       ../../../../libs/armeabi/libmp4v2.so )include_directories(libs/include)set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -std=gnu++11")add_library(Mp4v2Helper SHARED        src/main/cpp/Mp4v2Helper.c)find_library( # Sets the name of the path variable.              log-lib              # Specifies the name of the NDK library that              # you want CMake to locate.              log )target_link_libraries( Mp4v2Helper mp4v2 ${log-lib} )

5 0