使用Opengl ES GlSurfaceView Render Camera preview

Camera 是手机当中最重要的多媒体模块之一，基于Camera可以做出许多非常有意思的feature，比如各种滤镜、图像识别以及最近非常热门的AR、VR。这些都涉及到图像处理，为了更加高效的处理和现实camera的图像，GPU是一个非常不错的选择，OpenGL接口是GPU硬件的访问接口。本文就介绍如何使用Opengl来render Camera的preview 数据。

一、Android Camera API

使用Android Camera API接口可以非常方便写出camera应用程序，在Android L之前可以使用android.hardware.camera API来编写Camera相关的应用程序，在Android L以后，google推出了android.hardware.camera2 API。相比较而言，camera2提供了更加灵活的API来访问camera底层硬件，能够控制并每一帧的参数（AEC/AWB/AF/等参数）。不过Camera2使用起来要稍微复杂一些，这里就使用camera API写camera。关键两点：open camera和start preview。这里参照了这个blog http://blog.csdn.net/yanzi1225627/article/details/33339965/ 写了camera 应用相关的类

package com.wfly.kui.magiccamera.camera;import android.graphics.Bitmap;import android.graphics.BitmapFactory;import android.graphics.PixelFormat;import android.graphics.SurfaceTexture;import android.hardware.Camera;import android.util.Log;import android.view.SurfaceHolder;import com.wfly.kui.magiccamera.utils.CamParaUtil;import com.wfly.kui.magiccamera.utils.FileUtil;import com.wfly.kui.magiccamera.utils.ImageUtil;import java.io.IOException;import java.util.List;/** * Created by kuiw on 11/3/2016. */public class CameraInstance {    private static final String TAG="CameraInstance";    private Camera mCamera;    private Camera.Parameters mParams;    private static CameraInstance mCameraInstance;    private boolean isPreviewing=false;    private boolean isOpened=false;    private Camera.Size mPreviewSize;    private Camera.Size mPictureSize;    public interface CamOpenedCallback{        public void cameraHasOpened();    }    public boolean isPreviewing(){        return isPreviewing;    }    public boolean isOpened(){        return isOpened;    }    public Camera.Size getmPreviewSize(){return mPreviewSize;}    private CameraInstance(){    }    public static synchronized CameraInstance getInstance(){        if(mCameraInstance == null){            mCameraInstance = new CameraInstance();        }        return mCameraInstance;    }    public void doOpenCamera(CamOpenedCallback callback){        Log.i(TAG, "doOpenCamera....");        if(mCamera == null){            mCamera = Camera.open();            isOpened=true;            Log.i(TAG, "Camera open over....");            if(callback != null){                callback.cameraHasOpened();            }        }else{            Log.i(TAG, "Camera is in open status");        }    }    public void doStartPreview(SurfaceTexture surface, float previewRate){        Log.i(TAG, "doStartPreview...");        if(isPreviewing){            Log.e(TAG,"camera is in previewing state");            return ;        }        if(mCamera != null){            try {                mCamera.setPreviewTexture(surface);            } catch (IOException e) {                e.printStackTrace();            }            initCameraParams();        }    }    /**     * 停止预览，释放Camera     */    public void doStopCamera(){        if(null != mCamera)        {            isOpened=false;            mCamera.setPreviewCallback(null);            mCamera.stopPreview();            isPreviewing = false;            mCamera.release();            mCamera = null;        }    }    public void doStopPreview(){        Log.e(TAG,"doStopPreview....");        if (isPreviewing && null!=mCamera){            mCamera.stopPreview();            isPreviewing=false;        }else{            Log.e(TAG,"camera is in not in previewing status");        }    }    public void doTakePicture(){        if(isPreviewing && (mCamera != null)){            mCamera.takePicture(mShutterCallback, null, mJpegPictureCallback);        }    }    private void initCameraParams(){        if(mCamera != null){            mParams = mCamera.getParameters();            mParams.setPictureFormat(PixelFormat.JPEG);            mPictureSize = CamParaUtil.getInstance().getPropPictureSize(                    mParams.getSupportedPictureSizes(),30, 800);            mParams.setPictureSize(mPictureSize.width, mPictureSize.height);            mPreviewSize = CamParaUtil.getInstance().getPropPreviewSize(                    mParams.getSupportedPreviewSizes(), 30, 800);            mParams.setPreviewSize(mPreviewSize.width, mPreviewSize.height);            mCamera.setDisplayOrientation(90);            List<String> focusModes = mParams.getSupportedFocusModes();            if(focusModes.contains("continuous-video")){                mParams.setFocusMode(Camera.Parameters.FOCUS_MODE_CONTINUOUS_VIDEO);            }            mCamera.setParameters(mParams);            mCamera.startPreview();            isPreviewing = true;            mParams = mCamera.getParameters();        }    }    Camera.ShutterCallback mShutterCallback = new Camera.ShutterCallback()            //快门按下的回调，在这里我们可以设置类似播放“咔嚓”声之类的操作。默认的就是咔嚓。    {        public void onShutter() {            // TODO Auto-generated method stub            Log.i(TAG, "myShutterCallback:onShutter...");        }    };    Camera.PictureCallback mRawCallback = new Camera.PictureCallback()            // 拍摄的未压缩原数据的回调,可以为null    {        public void onPictureTaken(byte[] data, Camera camera) {            // TODO Auto-generated method stub            Log.i(TAG, "myRawCallback:onPictureTaken...");        }    };    Camera.PictureCallback mJpegPictureCallback = new Camera.PictureCallback ()            //对jpeg图像数据的回调,最重要的一个回调    {        public void onPictureTaken(byte[] data, Camera camera) {            // TODO Auto-generated method stub            Log.i(TAG, "myJpegCallback:onPictureTaken...");            Bitmap b = null;            if(null != data){                b = BitmapFactory.decodeByteArray(data, 0, data.length);                mCamera.stopPreview();                isPreviewing = false;            }            if(null != b)            {                Bitmap rotaBitmap = ImageUtil.getRotateBitmap(b, 90.0f);                FileUtil.saveBitmap(rotaBitmap);            }            mCamera.startPreview();            isPreviewing = true;        }    };}

二、GLSurfaceView

主要实现三个接口onSurfaceCreated，onSurfaceChanged，onDrawFrame。在surface创建成功后，调用camera start preview。

这里主要用三个类实现相关的调用:CameraSurfaceView, FilterRender, GLUtils

package com.wfly.kui.magiccamera.camera;import android.content.Context;import android.graphics.SurfaceTexture;import android.opengl.GLES11Ext;import android.opengl.GLES20;import android.opengl.GLSurfaceView;import android.opengl.GLSurfaceView;import android.util.AttributeSet;import android.util.Log;import javax.microedition.khronos.egl.EGLConfig;import javax.microedition.khronos.opengles.GL10;/** * Created by kuiw on 11/3/2016. */public class CameraSurfaceView extends GLSurfaceView implements GLSurfaceView.Renderer,SurfaceTexture.OnFrameAvailableListener, CameraInstance.CamOpenedCallback {    public static final String TAG="CameraSurfaceView";    private Context mContext;    private SurfaceTexture mSurface;    private int mTextureID = -1;    private DirectDrawer mDirectDrawer;    private FilterRender mFilterDrawer;    public CameraSurfaceView(Context context, AttributeSet attrs) {        super(context, attrs);        setEGLContextClientVersion(2);        setRenderer(this);        setRenderMode(RENDERMODE_WHEN_DIRTY);        this.mContext=context;    }    @Override    public void onPause() {        super.onPause();        CameraInstance.getInstance().doStopPreview();    }    @Override    public void onResume() {        super.onResume();    }    @Override    public void onSurfaceCreated(GL10 gl10, EGLConfig eglConfig) {        Log.i(TAG, "onSurfaceCreated...");        //mTextureID = createTextureID();        //mDirectDrawer = new DirectDrawer(mTextureID);        mFilterDrawer=new FilterRender(mContext);        mTextureID=mFilterDrawer.getmTexture();        mSurface = new SurfaceTexture(mTextureID);        mSurface.setOnFrameAvailableListener(this);        //CameraInstance.getInstance().doOpenCamera(null);    }    @Override    public void onSurfaceChanged(GL10 gl10, int width, int height) {        GLES20.glViewport(0, 0, width, height);        if(!CameraInstance.getInstance().isPreviewing()){            CameraInstance.getInstance().doStartPreview(mSurface, 1.33f);        }    }    @Override    public void onDrawFrame(GL10 gl10) {        GLES20.glClearColor(1.0f, 1.0f, 1.0f, 1.0f);        GLES20.glClear(GLES20.GL_COLOR_BUFFER_BIT | GLES20.GL_DEPTH_BUFFER_BIT);        Log.i(TAG, "onDrawFrame...");        mSurface.updateTexImage();        mFilterDrawer.drawTexture();        /*        float[] mtx = new float[16];        mSurface.getTransformMatrix(mtx);        mDirectDrawer.draw(mtx);*/    }    @Override    public void onFrameAvailable(SurfaceTexture surfaceTexture) {        this.requestRender();    }    @Override    public void cameraHasOpened() {    }    private int createTextureID()    {        int[] texture = new int[1];        GLES20.glGenTextures(1, texture, 0);        GLES20.glBindTexture(GLES11Ext.GL_TEXTURE_EXTERNAL_OES, texture[0]);        GLES20.glTexParameterf(GLES11Ext.GL_TEXTURE_EXTERNAL_OES,                GL10.GL_TEXTURE_MIN_FILTER,GL10.GL_LINEAR);        GLES20.glTexParameterf(GLES11Ext.GL_TEXTURE_EXTERNAL_OES,                GL10.GL_TEXTURE_MAG_FILTER, GL10.GL_LINEAR);        GLES20.glTexParameteri(GLES11Ext.GL_TEXTURE_EXTERNAL_OES,                GL10.GL_TEXTURE_WRAP_S, GL10.GL_CLAMP_TO_EDGE);        GLES20.glTexParameteri(GLES11Ext.GL_TEXTURE_EXTERNAL_OES,                GL10.GL_TEXTURE_WRAP_T, GL10.GL_CLAMP_TO_EDGE);        return texture[0];    }}

package com.wfly.kui.magiccamera.camera;import android.content.Context;import android.hardware.Camera;import android.media.effect.Effect;import android.media.effect.EffectContext;import android.media.effect.EffectFactory;import android.opengl.GLES11Ext;import android.opengl.GLES20;import com.wfly.kui.magiccamera.utils.GLUtils;import java.nio.ByteBuffer;import java.nio.ByteOrder;import java.nio.FloatBuffer;import javax.microedition.khronos.opengles.GL10;import static android.R.attr.bitmap;/** * Created by kuiw on 11/4/2016. */public class FilterRender {    public static final String TAG="FilterRender";    private int mTexture=-1;    private Context mContext;    private final float vertices[] = {            -1f, -1f,            1f, -1f,            -1f, 1f,            1f, 1f,            -1f,0f,            0f,0f,            -1f,1f,            0f,1f,            0f,0f,            1f,0f,            0f,1f,            1f,1f,    };    private final float texturevertices[]={            0f,1f,            1f,1f,            0f,0f,            1f,0f,            0f,1f,            1f,1f,            0f,0f,            1f,0f,            0f,1f,            1f,1f,            0f,0f,            1f,0f,    };    private FloatBuffer verticesbBuf;    private FloatBuffer textureVerticesBuf;    private String vertexShader;    private String fragmentShader;    private int aPositionHandle;    private int uTextureHandle;    private int aTexPositionHandle;    private int program;    private EffectContext effectContext;    private Effect effect;    private int mEffectTexture;    private Camera.Size mPreviewSize;    private void applyEffectForTexture(){        effect=effectContext.getFactory().createEffect(EffectFactory.EFFECT_BRIGHTNESS);        //effect.setParameter("scale", .5f);;        effect.setParameter("brightness", 4.0f);        effect.apply(mTexture,CameraInstance.getInstance().getmPreviewSize().width,CameraInstance.getInstance().getmPreviewSize().height,mEffectTexture);    }    private void initFloatBuffer(){        ByteBuffer byteBuffer = ByteBuffer.allocateDirect(vertices.length*4);        byteBuffer.order(ByteOrder.nativeOrder());        verticesbBuf=byteBuffer.asFloatBuffer();        verticesbBuf.put(vertices);        verticesbBuf.position(0);        byteBuffer=ByteBuffer.allocateDirect(texturevertices.length*4);        byteBuffer.order(ByteOrder.nativeOrder());        textureVerticesBuf=byteBuffer.asFloatBuffer();        textureVerticesBuf.put(texturevertices);        textureVerticesBuf.position(0);    }    public void drawTexture(){       // GLES20.glBindFramebuffer(GLES20.GL_FRAMEBUFFER,0);        GLES20.glUseProgram(program);        //GLES20.glDisable(GLES20.GL_BLEND);        //applyEffectForTexture();        aPositionHandle = GLES20.glGetAttribLocation(program, "aPosition");        aTexPositionHandle = GLES20.glGetAttribLocation(program, "aTexPosition");        uTextureHandle = GLES20.glGetUniformLocation(program, "uTexture");        GLES20.glVertexAttribPointer(aPositionHandle,2,GLES20.GL_FLOAT,false,0,verticesbBuf);        GLES20.glEnableVertexAttribArray(aPositionHandle);        GLES20.glActiveTexture(GLES20.GL_TEXTURE0);        GLES20.glBindTexture(GLES11Ext.GL_TEXTURE_EXTERNAL_OES,mTexture);        GLES20.glUniform1i(uTextureHandle,0);        GLES20.glVertexAttribPointer(aTexPositionHandle, 2, GLES20.GL_FLOAT, false, 0, textureVerticesBuf);        GLES20.glEnableVertexAttribArray(aTexPositionHandle);        GLES20.glClear(GLES20.GL_COLOR_BUFFER_BIT);        for(int index=0;index<3;index++){            GLES20.glDrawArrays(GLES20.GL_TRIANGLE_STRIP, index*4, 4);        }        GLES20.glDisableVertexAttribArray(aTexPositionHandle);        GLES20.glDisableVertexAttribArray(aPositionHandle);    }    private int createTextureID()    {        int[] texture = new int[1];        GLES20.glGenTextures(1, texture, 0);        GLES20.glBindTexture(GLES11Ext.GL_TEXTURE_EXTERNAL_OES, texture[0]);        GLES20.glTexParameterf(GLES11Ext.GL_TEXTURE_EXTERNAL_OES,                GL10.GL_TEXTURE_MIN_FILTER,GL10.GL_LINEAR);        GLES20.glTexParameterf(GLES11Ext.GL_TEXTURE_EXTERNAL_OES,                GL10.GL_TEXTURE_MAG_FILTER, GL10.GL_LINEAR);        GLES20.glTexParameteri(GLES11Ext.GL_TEXTURE_EXTERNAL_OES,                GL10.GL_TEXTURE_WRAP_S, GL10.GL_CLAMP_TO_EDGE);        GLES20.glTexParameteri(GLES11Ext.GL_TEXTURE_EXTERNAL_OES,                GL10.GL_TEXTURE_WRAP_T, GL10.GL_CLAMP_TO_EDGE);        return texture[0];    }    public FilterRender(Context context){        this.mTexture=createTextureID();        this.mContext=context;        effectContext=EffectContext.createWithCurrentGlContext();        mEffectTexture=createTextureID();        initFloatBuffer();        mPreviewSize=CameraInstance.getInstance().getmPreviewSize();        vertexShader= GLUtils.loadFromAssetsFile("vertexshader.vs",mContext.getAssets());        fragmentShader=GLUtils.loadFromAssetsFile("fragmentshader.vs",mContext.getAssets());        program=GLUtils.createProgram(vertexShader,fragmentShader);    }    public int getmTexture(){return mTexture;}}


package com.wfly.kui.magiccamera.utils;import android.content.Context;import android.content.res.AssetManager;import android.content.res.Resources;import android.graphics.Bitmap;import android.graphics.BitmapFactory;import android.opengl.GLES20;import android.util.Log;import java.io.ByteArrayOutputStream;import java.io.InputStream;/** * Created by Administrator on 2016/10/22. */public class GLUtils {    public static final String TAG="GLUtils";    /**     * 加载着色器方法     *     * 流程 :     *     * ① 创建着色器     * ② 加载着色器脚本     * ③ 编译着色器     * ④ 获取着色器编译结果     *     * @param shaderType 着色器类型,顶点着色器(GLES20.GL_FRAGMENT_SHADER), 片元着色器(GLES20.GL_FRAGMENT_SHADER)     * @param source 着色脚本字符串     * @return 返回的是着色器的引用, 返回值可以代表加载的着色器     */    public static int loadShader(int shaderType , String source){        //1.创建一个着色器, 并记录所创建的着色器的id, 如果id==0, 那么创建失败        int shader = GLES20.glCreateShader(shaderType);        if(shader != 0){            //2.如果着色器创建成功, 为创建的着色器加载脚本代码            GLES20.glShaderSource(shader, source);            //3.编译已经加载脚本代码的着色器            GLES20.glCompileShader(shader);            int[] compiled = new int[1];            //4.获取着色器的编译情况, 如果结果为0, 说明编译失败            GLES20.glGetShaderiv(shader, GLES20.GL_COMPILE_STATUS, compiled, 0);            if(compiled[0] == 0){                Log.e("ES20_ERROR", "Could not compile shader " + shaderType + ":");                Log.e("ES20_ERROR", GLES20.glGetShaderInfoLog(shader));                //编译失败的话, 删除着色器, 并显示log                GLES20.glDeleteShader(shader);                shader = 0;            }        }else{            Log.e(TAG,"create shader failed:");        }        return shader;    }    /**     * 检查每一步的操作是否正确     *     * 使用GLES20.glGetError()方法可以获取错误代码, 如果错误代码为0, 那么就没有错误     *     * @param op 具体执行的方法名, 比如执行向着色程序中加入着色器,     * 使glAttachShader()方法, 那么这个参数就是"glAttachShader"     */    public static void checkGLError(String op){        int error;        //错误代码不为0, 就打印错误日志, 并抛出异常        while( (error = GLES20.glGetError()) != GLES20.GL_NO_ERROR ){            Log.e("ES20_ERROR", op + ": glError " + error);            throw new RuntimeException(op + ": glError " + error);        }    }    /**     * 创建着色程序     *     * ① 加载顶点着色器     * ② 加载片元着色器     * ③ 创建着色程序     * ④ 向着色程序中加入顶点着色器     * ⑤ 向着色程序中加入片元着色器     * ⑥ 链接程序     * ⑦ 获取链接程序结果     *     * @param vertexSource定点着色器脚本字符串     * @param fragmentSource片元着色器脚本字符串     * @return     */    public static int createProgram(String vertexSource , String fragmentSource){        //1. 加载顶点着色器, 返回0说明加载失败        int vertexShader = loadShader(GLES20.GL_VERTEX_SHADER, vertexSource);        if(vertexShader == 0) {            Log.e(TAG,"load vertex shader failed!");            return 0;        }        //2. 加载片元着色器, 返回0说明加载失败        int fragShader = loadShader(GLES20.GL_FRAGMENT_SHADER, fragmentSource);        if(fragShader == 0) {            Log.e(TAG,"load fragment shader failed!");            return 0;        }        //3. 创建着色程序, 返回0说明创建失败        int program = GLES20.glCreateProgram();        if(program != 0){            //4. 向着色程序中加入顶点着色器            GLES20.glAttachShader(program, vertexShader);            checkGLError("glAttachShader");            //5. 向着色程序中加入片元着色器            GLES20.glAttachShader(program, fragShader);            checkGLError("glAttachShader");            //6. 链接程序            GLES20.glLinkProgram(program);            int[] linkStatus = new int[1];            //获取链接程序结果            GLES20.glGetProgramiv(program, GLES20.GL_LINK_STATUS, linkStatus, 0);            if(linkStatus[0] != GLES20.GL_TRUE){                Log.e("ES20.ERROR", "链接程序失败 : ");                Log.e("ES20.ERROR", GLES20.glGetProgramInfoLog(program));                //如果链接程序失败删除程序                GLES20.glDeleteProgram(program);                program = 0;            }            GLES20.glDeleteShader(vertexShader);            GLES20.glDeleteShader(fragShader);        }        Log.e(TAG,"Create Programe Successfully");        return program;    }    /**     * 从assets中加载着色脚本     *     * ① 打开assets目录中的文件输入流     * ② 创建带缓冲区的输出流     * ③ 逐个字节读取文件数据, 放入缓冲区     * ④ 将缓冲区中的数据转为字符串     *     * @param fileName assets目录中的着色脚本文件名     * @param resources应用的资源     * @return     */    public static String loadFromAssetsFile(String fileName, Resources resources){        String result = null;        try {            //1. 打开assets目录中读取文件的输入流, 相当于创建了一个文件的字节输入流            InputStream is = resources.getAssets().open(fileName);            int ch = 0;            //2. 创建一个带缓冲区的输出流, 每次读取一个字节, 注意这里字节读取用的是int类型            ByteArrayOutputStream baos = new ByteArrayOutputStream();            //3. 逐个字节读取数据, 并将读取的数据放入缓冲器中            while((ch = is.read()) != -1){                baos.write(ch);            }            //4. 将缓冲区中的数据转为字节数组, 并将字节数组转换为字符串            byte[] buffer = baos.toByteArray();            baos.close();            is.close();            result = new String(buffer, "UTF-8");            result = result.replaceAll("\\r\\n", "\n");        } catch (Exception e) {            e.printStackTrace();        }        return result;    }    public static String loadFromAssetsFile(String fileName, AssetManager assetManager){        String result = null;        try {            //1. 打开assets目录中读取文件的输入流, 相当于创建了一个文件的字节输入流            InputStream is = assetManager.open(fileName);            int ch = 0;            //2. 创建一个带缓冲区的输出流, 每次读取一个字节, 注意这里字节读取用的是int类型            ByteArrayOutputStream baos = new ByteArrayOutputStream();            //3. 逐个字节读取数据, 并将读取的数据放入缓冲器中            while((ch = is.read()) != -1){                baos.write(ch);            }            //4. 将缓冲区中的数据转为字节数组, 并将字节数组转换为字符串            byte[] buffer = baos.toByteArray();            baos.close();            is.close();            result = new String(buffer, "UTF-8");            result = result.replaceAll("\\r\\n", "\n");        } catch (Exception e) {            e.printStackTrace();        }        return result;    }    public static Bitmap loadBitmapFromAssetsFile(String fileName,Resources resources){        Bitmap bitmap=null;        try {            //1. 打开assets目录中读取文件的输入流, 相当于创建了一个文件的字节输入流            InputStream is = resources.getAssets().open(fileName);            BitmapFactory.Options options = new BitmapFactory.Options();            bitmap = BitmapFactory.decodeStream(is, null, options);            if(bitmap==null){                Log.e(TAG,"load image from asset file:"+fileName);            }        } catch (Exception e) {            e.printStackTrace();        }        return bitmap;    }}