Refactor EglViewport

pull/421/head
Giacomo Randazzo 7 years ago
parent 2ca572e57d
commit 076c53aab1
  1. 187
      cameraview/src/main/gles/com/otaliastudios/cameraview/EglViewport.java
  2. 6
      cameraview/src/main/gles/com/otaliastudios/cameraview/TextureMediaEncoder.java
  3. 11
      cameraview/src/main/java/com/otaliastudios/cameraview/SnapshotPictureRecorder.java
  4. 19
      cameraview/src/main/views/com/otaliastudios/cameraview/GlCameraPreview.java

@ -4,7 +4,6 @@ package com.otaliastudios.cameraview;
import android.opengl.GLES11Ext;
import android.opengl.GLES20;
import android.opengl.Matrix;
import android.util.Log;
import java.nio.FloatBuffer;
@ -27,18 +26,6 @@ class EglViewport extends EglElement {
" vTextureCoord = (uTexMatrix * aTextureCoord).xy;\n" +
"}\n";
// Simple vertex shader.
private static final String OVERLAY_VERTEX_SHADER =
"uniform mat4 uMVPMatrix;\n" +
"uniform mat4 uOverlayTexMatrix;\n" +
"attribute vec4 aPosition;\n" +
"attribute vec4 aTextureCoord;\n" +
"varying vec2 vOverlayTextureCoord;\n" +
"void main() {\n" +
" gl_Position = uMVPMatrix * aPosition;\n" +
" vOverlayTextureCoord = (uOverlayTexMatrix * aTextureCoord).xy;\n" +
"}\n";
// Simple fragment shader for use with external 2D textures
private static final String SIMPLE_FRAGMENT_SHADER =
"#extension GL_OES_EGL_image_external : require\n" +
@ -49,21 +36,6 @@ class EglViewport extends EglElement {
" gl_FragColor = texture2D(sTexture, vTextureCoord);\n" +
"}\n";
// Fragment shader for use with two external 2D textures.
// A overlay texture will act as a layer on top of the camera texture,
// it covers the preview when alpha is 1 and lets the camera texture through when alpha is less
// than 1.
private static final String OVERLAY_FRAGMENT_SHADER =
"#extension GL_OES_EGL_image_external : require\n" +
"precision mediump float;\n" +
"varying vec2 vOverlayTextureCoord;\n" +
"uniform samplerExternalOES overlayTexture;\n" +
"void main() {\n" +
" lowp vec4 c1 = texture2D(overlayTexture, vOverlayTextureCoord);\n" +
" if (c1.a == 0.0){ discard; }\n" +
" gl_FragColor = c1;\n" +
"}\n";
// Stuff from Drawable2d.FULL_RECTANGLE
// A full square, extending from -1 to +1 in both dimensions.
// When the model/view/projection matrix is identity, this will exactly cover the viewport.
@ -90,38 +62,20 @@ class EglViewport extends EglElement {
// Stuff from Texture2dProgram
private int mProgramHandle;
private int mProgramHandleOverlay;
private int mTextureTarget;
// Program attributes
private int muMVPMatrixLocation;
private int muOverlayMVPMatrixLocation;
private int muTexMatrixLocation;
private int muOverlayTexMatrixLocation;
private int muTextureLocation;
private int muOverlayTextureLocation;
private int maPositionLocation;
private int maOverlayPositionLocation;
private int maTextureCoordLocation;
private int maOverlayTextureCoordLocation;
// private int muKernelLoc; // Used for filtering
// private int muTexOffsetLoc; // Used for filtering
// private int muColorAdjustLoc; // Used for filtering
private final boolean mHasOverlay;
EglViewport() {
this(false);
}
EglViewport(boolean hasOverlay) {
mHasOverlay = hasOverlay;
mTextureTarget = GLES11Ext.GL_TEXTURE_EXTERNAL_OES;
mProgramHandle = createProgram(SIMPLE_VERTEX_SHADER, SIMPLE_FRAGMENT_SHADER);
mProgramHandleOverlay = createProgram(OVERLAY_VERTEX_SHADER, OVERLAY_FRAGMENT_SHADER);
maPositionLocation = GLES20.glGetAttribLocation(mProgramHandle, "aPosition");
checkLocation(maPositionLocation, "aPosition");
maTextureCoordLocation = GLES20.glGetAttribLocation(mProgramHandle, "aTextureCoord");
@ -131,22 +85,6 @@ class EglViewport extends EglElement {
muTexMatrixLocation = GLES20.glGetUniformLocation(mProgramHandle, "uTexMatrix");
checkLocation(muTexMatrixLocation, "uTexMatrix");
maOverlayPositionLocation = GLES20.glGetAttribLocation(mProgramHandleOverlay, "aPosition");
checkLocation(maOverlayPositionLocation, "aPosition");
maOverlayTextureCoordLocation = GLES20.glGetAttribLocation(mProgramHandleOverlay, "aTextureCoord");
checkLocation(maOverlayTextureCoordLocation, "aTextureCoord");
muOverlayMVPMatrixLocation = GLES20.glGetUniformLocation(mProgramHandleOverlay, "uMVPMatrix");
checkLocation(muOverlayMVPMatrixLocation, "uMVPMatrix");
if (mHasOverlay) {
muOverlayTexMatrixLocation = GLES20.glGetUniformLocation(mProgramHandleOverlay, "uOverlayTexMatrix");
checkLocation(muOverlayTexMatrixLocation, "uOverlayTexMatrix");
muOverlayTextureLocation = GLES20.glGetUniformLocation(mProgramHandleOverlay, "overlayTexture");
checkLocation(muOverlayTextureLocation, "overlayTexture");
}
muTextureLocation = GLES20.glGetUniformLocation(mProgramHandle, "sTexture");
checkLocation(muTextureLocation, "sTexture");
// Stuff from Drawable2d.FULL_RECTANGLE
}
@ -161,20 +99,13 @@ class EglViewport extends EglElement {
}
int createTexture() {
return createTextures()[0];
}
int[] createTextures() {
// index 0 is reserved for the camera texture, index 1 is reserved for the overlay texture
int numTextures = mHasOverlay ? 2 : 1;
int[] textures = new int[numTextures];
int[] textures = new int[1];
GLES20.glGenTextures(1, textures, 0);
GLES20.glGenTextures(1, textures, 1);
check("glGenTextures");
// camera texture
GLES20.glBindTexture(mTextureTarget, textures[0]);
check("glBindTexture " + textures[0]);
int texId = textures[0];
GLES20.glBindTexture(mTextureTarget, texId);
check("glBindTexture " + texId);
GLES20.glTexParameterf(GLES11Ext.GL_TEXTURE_EXTERNAL_OES, GLES20.GL_TEXTURE_MIN_FILTER, GLES20.GL_NEAREST);
GLES20.glTexParameterf(GLES11Ext.GL_TEXTURE_EXTERNAL_OES, GLES20.GL_TEXTURE_MAG_FILTER, GLES20.GL_LINEAR);
@ -182,19 +113,7 @@ class EglViewport extends EglElement {
GLES20.glTexParameteri(GLES11Ext.GL_TEXTURE_EXTERNAL_OES, GLES20.GL_TEXTURE_WRAP_T, GLES20.GL_CLAMP_TO_EDGE);
check("glTexParameter");
// overlay texture
if (mHasOverlay) {
GLES20.glBindTexture(mTextureTarget, textures[1]);
check("glBindTexture " + textures[1]);
GLES20.glTexParameterf(GLES11Ext.GL_TEXTURE_EXTERNAL_OES, GLES20.GL_TEXTURE_MIN_FILTER, GLES20.GL_NEAREST);
GLES20.glTexParameterf(GLES11Ext.GL_TEXTURE_EXTERNAL_OES, GLES20.GL_TEXTURE_MAG_FILTER, GLES20.GL_LINEAR);
GLES20.glTexParameteri(GLES11Ext.GL_TEXTURE_EXTERNAL_OES, GLES20.GL_TEXTURE_WRAP_S, GLES20.GL_CLAMP_TO_EDGE);
GLES20.glTexParameteri(GLES11Ext.GL_TEXTURE_EXTERNAL_OES, GLES20.GL_TEXTURE_WRAP_T, GLES20.GL_CLAMP_TO_EDGE);
check("glTexParameter");
}
return textures;
return texId;
}
void drawFrame(int textureId, float[] textureMatrix) {
@ -203,33 +122,11 @@ class EglViewport extends EglElement {
mTextureCoordinatesArray);
}
private void drawFrame(int textureId, float[] textureMatrix,
FloatBuffer vertexBuffer,
FloatBuffer texBuffer) {
// 0 is not a valid texture id
drawFrame(textureId, 0, textureMatrix, null,
vertexBuffer,
texBuffer);
}
void drawFrameOverlay(int overlayTextureId, float[] overlayTextureMatrix) {
drawFrameOverlay(overlayTextureId, overlayTextureMatrix,
mVertexCoordinatesArray,
mTextureCoordinatesArray);
}
void drawFrame(int textureId, int overlayTextureId, float[] textureMatrix, float[] overlayTextureMatrix) {
drawFrame(textureId, overlayTextureId, textureMatrix, overlayTextureMatrix,
mVertexCoordinatesArray,
mTextureCoordinatesArray);
}
/**
* The issue with the CIRCLE shader is that if the textureMatrix has a scale value,
* it fails miserably, not taking the scale into account.
* So what we can do here is
* <p>
*
* - read textureMatrix scaleX and scaleY values. This is pretty much impossible to do from the matrix itself
* without making risky assumptions over the order of operations.
* https://www.opengl.org/discussion_boards/showthread.php/159215-Is-it-possible-to-extract-rotation-translation-scale-given-a-matrix
@ -238,7 +135,7 @@ class EglViewport extends EglElement {
* - pass them to the fragment shader
* - in the fragment shader, take this scale value into account
*/
private void drawFrame(int textureId, int overlayTextureId, float[] textureMatrix, float[] overlayTextureMatrix,
private void drawFrame(int textureId, float[] textureMatrix,
FloatBuffer vertexBuffer,
FloatBuffer texBuffer) {
check("draw start");
@ -247,10 +144,15 @@ class EglViewport extends EglElement {
GLES20.glUseProgram(mProgramHandle);
check("glUseProgram");
// Set the camera texture.
// enable blending, from: http://www.learnopengles.com/android-lesson-five-an-introduction-to-blending/
GLES20.glDisable(GLES20.GL_CULL_FACE);
GLES20.glDisable(GLES20.GL_DEPTH_TEST);
GLES20.glEnable(GLES20.GL_BLEND);
GLES20.glBlendFunc(GLES20.GL_SRC_ALPHA, GLES20.GL_ONE_MINUS_SRC_ALPHA);
// Set the texture.
GLES20.glActiveTexture(GLES20.GL_TEXTURE0);
GLES20.glBindTexture(mTextureTarget, textureId);
GLES20.glUniform1i(muTextureLocation, 0);
// Copy the model / view / projection matrix over.
GLES20.glUniformMatrix4fv(muMVPMatrixLocation, 1, false, IDENTITY_MATRIX, 0);
@ -274,72 +176,15 @@ class EglViewport extends EglElement {
GLES20.glVertexAttribPointer(maTextureCoordLocation, 2, GLES20.GL_FLOAT, false, 8, texBuffer);
check("glVertexAttribPointer");
// Draw the rect.
GLES20.glDrawArrays(GLES20.GL_TRIANGLE_STRIP, 0, VERTEX_COUNT);
check("glDrawArrays");
// Done -- disable vertex array, textures, and program.
// Done -- disable vertex array, texture, and program.
GLES20.glDisableVertexAttribArray(maPositionLocation);
GLES20.glDisableVertexAttribArray(maTextureCoordLocation);
GLES20.glBindTexture(mTextureTarget, 0);
GLES20.glUseProgram(0);
}
private void drawFrameOverlay(int overlayTextureId, float[] overlayTextureMatrix,
FloatBuffer vertexBuffer,
FloatBuffer texBuffer) {
check("draw start");
// Select the program.
GLES20.glUseProgram(mProgramHandleOverlay);
check("glUseProgram");
// No culling of back faces
GLES20.glDisable(GLES20.GL_CULL_FACE);
// No depth testing
GLES20.glDisable(GLES20.GL_DEPTH_TEST);
// Enable blending
GLES20.glEnable(GLES20.GL_BLEND);
GLES20.glBlendFunc(GLES20.GL_SRC_ALPHA, GLES20.GL_ONE_MINUS_SRC_ALPHA);
GLES20.glActiveTexture(GLES20.GL_TEXTURE1);
GLES20.glBindTexture(mTextureTarget, overlayTextureId);
GLES20.glUniform1i(muOverlayTextureLocation, 1);
// Copy the model / view / projection matrix over.
GLES20.glUniformMatrix4fv(muOverlayMVPMatrixLocation, 1, false, IDENTITY_MATRIX, 0);
check("glUniformMatrix4fv");
// Copy the texture transformation matrix over.
GLES20.glUniformMatrix4fv(muOverlayTexMatrixLocation, 1, false, overlayTextureMatrix, 0);
check("glUniformMatrix4fv");
// Enable the "aPosition" vertex attribute.
// Connect vertexBuffer to "aPosition".
GLES20.glEnableVertexAttribArray(maOverlayPositionLocation);
check("glEnableVertexAttribArray");
GLES20.glVertexAttribPointer(maOverlayPositionLocation, 2, GLES20.GL_FLOAT, false, 8, vertexBuffer);
check("glVertexAttribPointer");
// Enable the "aTextureCoord" vertex attribute.
// Connect texBuffer to "aTextureCoord".
GLES20.glEnableVertexAttribArray(maOverlayTextureCoordLocation);
check("glEnableVertexAttribArray");
GLES20.glVertexAttribPointer(maOverlayTextureCoordLocation, 2, GLES20.GL_FLOAT, false, 8, texBuffer);
check("glVertexAttribPointer");
// Draw the rect.
GLES20.glDrawArrays(GLES20.GL_TRIANGLE_STRIP, 0, VERTEX_COUNT);
check("glDrawArrays");
// Done -- disable vertex array, textures, and program.
GLES20.glDisableVertexAttribArray(maOverlayPositionLocation);
GLES20.glDisableVertexAttribArray(maOverlayTextureCoordLocation);
GLES20.glBindTexture(mTextureTarget, 0);
GLES20.glUseProgram(0);
}
}

@ -88,7 +88,8 @@ class TextureMediaEncoder extends VideoMediaEncoder<TextureMediaEncoder.Config>
mWindow = new EglWindowSurface(mEglCore, mSurface, true);
mWindow.makeCurrent(); // drawing will happen on the InputWindowSurface, which
// is backed by mVideoEncoder.getInputSurface()
mViewport = new EglViewport(mConfig.overlayTextureId != 0);
// mViewport = new EglViewport(mConfig.overlayTextureId != 0);
mViewport = new EglViewport();
}
@EncoderThread
@ -141,7 +142,8 @@ class TextureMediaEncoder extends VideoMediaEncoder<TextureMediaEncoder.Config>
drainOutput(false);
// Future note: passing scale values to the viewport? They are scaleX and scaleY,
// but flipped based on the mConfig.scaleFlipped boolean.
mViewport.drawFrame(mConfig.textureId, mConfig.overlayTextureId, transform, overlayTransform);
// mViewport.drawFrame(mConfig.textureId, mConfig.overlayTextureId, transform, overlayTransform);
mViewport.drawFrame(mConfig.textureId, transform);
mWindow.setPresentationTime(frame.timestamp);
mWindow.swapBuffers();
mFramePool.recycle(frame);

@ -74,7 +74,8 @@ class SnapshotPictureRecorder extends PictureRecorder {
}
mSurfaceTexture = new SurfaceTexture(mTextureId, true);
if (mOverlayTextureId != 0) {
mOverlaySurfaceTexture = new SurfaceTexture(mOverlayTextureId, true);
mOverlaySurfaceTexture = null;
// mOverlaySurfaceTexture = new SurfaceTexture(mOverlayTextureId, true);
}
// Need to crop the size.
Rect crop = CropHelper.computeCrop(mResult.size, mOutputRatio);
@ -91,7 +92,7 @@ class SnapshotPictureRecorder extends PictureRecorder {
@RendererThread
@Override
public void onRendererFrame(SurfaceTexture surfaceTexture, SurfaceTexture overlaySurfaceTexture, final float scaleX, final float scaleY) {
public void onRendererFrame(SurfaceTexture surfaceTexture, final SurfaceTexture overlaySurfaceTexture, final float scaleX, final float scaleY) {
preview.removeRendererFrameCallback(this);
// This kinda work but has drawbacks:
@ -120,7 +121,7 @@ class SnapshotPictureRecorder extends PictureRecorder {
public void run() {
EglWindowSurface surface = new EglWindowSurface(core, mSurfaceTexture);
surface.makeCurrent();
EglViewport viewport = new EglViewport(mOverlaySurfaceTexture != null);
EglViewport viewport = new EglViewport();
mSurfaceTexture.updateTexImage();
mSurfaceTexture.getTransformMatrix(mTransform);
if (mOverlaySurfaceTexture != null) {
@ -177,9 +178,9 @@ class SnapshotPictureRecorder extends PictureRecorder {
// Future note: passing scale values to the viewport?
// They are simply realScaleX and realScaleY.
viewport.drawFrame(mTextureId, mOverlayTextureId, mTransform, mOverlayTransform);
viewport.drawFrame(mTextureId, mTransform);
if (mOverlayTransform != null) {
viewport.drawFrameOverlay(mOverlayTextureId, mOverlayTransform);
// viewport.drawFrameOverlay(mOverlayTextureId, mOverlayTransform);
}
// don't - surface.swapBuffers();
mResult.data = surface.saveFrameTo(Bitmap.CompressFormat.JPEG);

@ -64,6 +64,7 @@ class GlCameraPreview extends CameraPreview<GLSurfaceView, SurfaceTexture> imple
private SurfaceTexture mOverlaySurfaceTexture;
private Surface mOverlaySurface;
private EglViewport mOutputViewport;
private EglViewport mOutputOverlayViewport;
private Set<RendererFrameCallback> mRendererFrameCallbacks = Collections.synchronizedSet(new HashSet<RendererFrameCallback>());
/* for tests */ float mScaleX = 1F;
/* for tests */ float mScaleY = 1F;
@ -144,14 +145,20 @@ class GlCameraPreview extends CameraPreview<GLSurfaceView, SurfaceTexture> imple
mOutputViewport.release();
mOutputViewport = null;
}
if (mOutputOverlayViewport != null) {
mOutputOverlayViewport.release();
mOutputOverlayViewport = null;
}
}
@RendererThread
@Override
public void onSurfaceCreated(GL10 gl, EGLConfig config) {
mOutputViewport = new EglViewport(hasOverlay());
mOutputViewport = new EglViewport();
mOutputOverlayViewport = new EglViewport();
mOutputTextureIds[0] = mOutputViewport.createTexture();
if (hasOverlay()) {
mOutputTextureIds = mOutputViewport.createTextures();
mOutputTextureIds[1] = mOutputOverlayViewport.createTexture();
mOverlaySurfaceTexture = new SurfaceTexture(mOutputTextureIds[1]);
mOverlaySurface = new Surface(mOverlaySurfaceTexture);
for (OverlayInputSurfaceListener listener : overlayListeners) {
@ -161,8 +168,6 @@ class GlCameraPreview extends CameraPreview<GLSurfaceView, SurfaceTexture> imple
}
mOverlaySurfaceTexture.setDefaultBufferSize(mInputStreamWidth, mInputStreamHeight);
} else {
mOutputTextureIds[0] = mOutputViewport.createTexture();
}
mInputSurfaceTexture = new SurfaceTexture(mOutputTextureIds[0]);
getView().queueEvent(new Runnable() {
@ -244,11 +249,9 @@ class GlCameraPreview extends CameraPreview<GLSurfaceView, SurfaceTexture> imple
}
// Future note: passing scale to the viewport?
// They are scaleX an scaleY, but flipped based on mInputFlipped.
if (hasOverlay() && mOutputTextureIds[1] != 0) {
mOutputViewport.drawFrame(mOutputTextureIds[0], mOutputTextureIds[1], mTransformMatrix, mOverlayTransformMatrix);
mOutputViewport.drawFrameOverlay(mOutputTextureIds[1], mOverlayTransformMatrix);
} else {
mOutputViewport.drawFrame(mOutputTextureIds[0], mTransformMatrix);
if (hasOverlay() && mOutputTextureIds[1] != 0) {
mOutputOverlayViewport.drawFrame(mOutputTextureIds[1], mOverlayTransformMatrix);
}
for (RendererFrameCallback callback : mRendererFrameCallbacks) {
callback.onRendererFrame(mInputSurfaceTexture, mOverlaySurfaceTexture, mScaleX, mScaleY);

Loading…
Cancel
Save