diff --git a/cameraview/src/main/gles/com/otaliastudios/cameraview/EglViewport.java b/cameraview/src/main/gles/com/otaliastudios/cameraview/EglViewport.java index 25a5da28..1892a0e2 100644 --- a/cameraview/src/main/gles/com/otaliastudios/cameraview/EglViewport.java +++ b/cameraview/src/main/gles/com/otaliastudios/cameraview/EglViewport.java @@ -4,6 +4,7 @@ package com.otaliastudios.cameraview; import android.opengl.GLES11Ext; import android.opengl.GLES20; import android.opengl.Matrix; +import android.util.Log; import java.nio.FloatBuffer; @@ -26,6 +27,21 @@ class EglViewport extends EglElement { " vTextureCoord = (uTexMatrix * aTextureCoord).xy;\n" + "}\n"; + // Simple vertex shader. + private static final String OVERLAY_VERTEX_SHADER = + "uniform mat4 uMVPMatrix;\n" + + "uniform mat4 uTexMatrix;\n" + + "uniform mat4 uOverlayTexMatrix;\n" + + "attribute vec4 aPosition;\n" + + "attribute vec4 aTextureCoord;\n" + + "varying vec2 vTextureCoord;\n" + + "varying vec2 vOverlayTextureCoord;\n" + + "void main() {\n" + + " gl_Position = uMVPMatrix * aPosition;\n" + + " vTextureCoord = (uTexMatrix * aTextureCoord).xy;\n" + + " vOverlayTextureCoord = (uOverlayTexMatrix * aTextureCoord).xy;\n" + + "}\n"; + // Simple fragment shader for use with external 2D textures private static final String SIMPLE_FRAGMENT_SHADER = "#extension GL_OES_EGL_image_external : require\n" + @@ -36,6 +52,28 @@ class EglViewport extends EglElement { " gl_FragColor = texture2D(sTexture, vTextureCoord);\n" + "}\n"; + // Fragment shader for use with two external 2D textures. + // A overlay texture will act as a layer on top of the camera texture, + // it covers the preview when alpha is 1 and lets the camera texture through when alpha is less + // than 1. + private static final String OVERLAY_FRAGMENT_SHADER = + "#extension GL_OES_EGL_image_external : require\n" + + "precision mediump float;\n" + + "varying vec2 vTextureCoord;\n" + + "varying vec2 vOverlayTextureCoord;\n" + + "uniform samplerExternalOES sTexture;\n" + + "uniform samplerExternalOES overlayTexture;\n" + + "void main() {\n" + + " lowp vec4 c2 = texture2D(sTexture, vTextureCoord);\n" + + " lowp vec4 c1 = texture2D(overlayTexture, vOverlayTextureCoord);\n" + + " lowp vec4 outputColor;\n" + + " outputColor.r = c1.r + c2.r * c2.a * (1.0 - c1.a);\n" + + " outputColor.g = c1.g + c2.g * c2.a * (1.0 - c1.a);\n" + + " outputColor.b = c1.b + c2.b * c2.a * (1.0 - c1.a);\n" + + " outputColor.a = c1.a + c2.a * (1.0 - c1.a);\n" + + " gl_FragColor = outputColor;\n" + + "}\n"; + // Stuff from Drawable2d.FULL_RECTANGLE // A full square, extending from -1 to +1 in both dimensions. // When the model/view/projection matrix is identity, this will exactly cover the viewport. @@ -66,6 +104,9 @@ class EglViewport extends EglElement { // Program attributes private int muMVPMatrixLocation; private int muTexMatrixLocation; + private int muOverlayTexMatrixLocation; + private int muTextureLocation; + private int muOverlayTextureLocation; private int maPositionLocation; private int maTextureCoordLocation; @@ -73,9 +114,22 @@ class EglViewport extends EglElement { // private int muTexOffsetLoc; // Used for filtering // private int muColorAdjustLoc; // Used for filtering + private final boolean mHasOverlay; + + EglViewport() { + this(false); + } + + EglViewport(boolean hasOverlay) { + mHasOverlay = hasOverlay; + mTextureTarget = GLES11Ext.GL_TEXTURE_EXTERNAL_OES; - mProgramHandle = createProgram(SIMPLE_VERTEX_SHADER, SIMPLE_FRAGMENT_SHADER); + if (mHasOverlay) { + mProgramHandle = createProgram(OVERLAY_VERTEX_SHADER, OVERLAY_FRAGMENT_SHADER); + } else { + mProgramHandle = createProgram(SIMPLE_VERTEX_SHADER, SIMPLE_FRAGMENT_SHADER); + } maPositionLocation = GLES20.glGetAttribLocation(mProgramHandle, "aPosition"); checkLocation(maPositionLocation, "aPosition"); maTextureCoordLocation = GLES20.glGetAttribLocation(mProgramHandle, "aTextureCoord"); @@ -84,6 +138,14 @@ class EglViewport extends EglElement { checkLocation(muMVPMatrixLocation, "uMVPMatrix"); muTexMatrixLocation = GLES20.glGetUniformLocation(mProgramHandle, "uTexMatrix"); checkLocation(muTexMatrixLocation, "uTexMatrix"); + muTextureLocation = GLES20.glGetUniformLocation(mProgramHandle, "sTexture"); + checkLocation(muTextureLocation, "sTexture"); + if (mHasOverlay) { + muOverlayTexMatrixLocation = GLES20.glGetUniformLocation(mProgramHandle, "uOverlayTexMatrix"); + checkLocation(muOverlayTexMatrixLocation, "uOverlayTexMatrix"); + muOverlayTextureLocation = GLES20.glGetUniformLocation(mProgramHandle, "overlayTexture"); + checkLocation(muOverlayTextureLocation, "overlayTexture"); + } // Stuff from Drawable2d.FULL_RECTANGLE @@ -99,13 +161,19 @@ class EglViewport extends EglElement { } int createTexture() { - int[] textures = new int[1]; - GLES20.glGenTextures(1, textures, 0); + return createTextures()[0]; + } + + int[] createTextures() { + // index 0 is reserved for the camera texture, index 1 is reserved for the overlay texture + int numTextures = mHasOverlay ? 2 : 1; + int[] textures = new int[numTextures]; + GLES20.glGenTextures(numTextures, textures, 0); check("glGenTextures"); - int texId = textures[0]; - GLES20.glBindTexture(mTextureTarget, texId); - check("glBindTexture " + texId); + // camera texture + GLES20.glBindTexture(mTextureTarget, textures[0]); + check("glBindTexture " + textures[0]); GLES20.glTexParameterf(GLES11Ext.GL_TEXTURE_EXTERNAL_OES, GLES20.GL_TEXTURE_MIN_FILTER, GLES20.GL_NEAREST); GLES20.glTexParameterf(GLES11Ext.GL_TEXTURE_EXTERNAL_OES, GLES20.GL_TEXTURE_MAG_FILTER, GLES20.GL_LINEAR); @@ -113,7 +181,19 @@ class EglViewport extends EglElement { GLES20.glTexParameteri(GLES11Ext.GL_TEXTURE_EXTERNAL_OES, GLES20.GL_TEXTURE_WRAP_T, GLES20.GL_CLAMP_TO_EDGE); check("glTexParameter"); - return texId; + // overlay texture + if (mHasOverlay) { + GLES20.glBindTexture(mTextureTarget, textures[1]); + check("glBindTexture " + textures[1]); + + GLES20.glTexParameterf(GLES11Ext.GL_TEXTURE_EXTERNAL_OES, GLES20.GL_TEXTURE_MIN_FILTER, GLES20.GL_NEAREST); + GLES20.glTexParameterf(GLES11Ext.GL_TEXTURE_EXTERNAL_OES, GLES20.GL_TEXTURE_MAG_FILTER, GLES20.GL_LINEAR); + GLES20.glTexParameteri(GLES11Ext.GL_TEXTURE_EXTERNAL_OES, GLES20.GL_TEXTURE_WRAP_S, GLES20.GL_CLAMP_TO_EDGE); + GLES20.glTexParameteri(GLES11Ext.GL_TEXTURE_EXTERNAL_OES, GLES20.GL_TEXTURE_WRAP_T, GLES20.GL_CLAMP_TO_EDGE); + check("glTexParameter"); + } + + return textures; } void drawFrame(int textureId, float[] textureMatrix) { @@ -122,6 +202,22 @@ class EglViewport extends EglElement { mTextureCoordinatesArray); } + + private void drawFrame(int textureId, float[] textureMatrix, + FloatBuffer vertexBuffer, + FloatBuffer texBuffer) { + // 0 is not a valid texture id + drawFrame(textureId, 0, textureMatrix, null, + vertexBuffer, + texBuffer); + } + + void drawFrame(int textureId, int overlayTextureId, float[] textureMatrix, float[] overlayTextureMatrix) { + drawFrame(textureId, overlayTextureId, textureMatrix, overlayTextureMatrix, + mVertexCoordinatesArray, + mTextureCoordinatesArray); + } + /** * The issue with the CIRCLE shader is that if the textureMatrix has a scale value, * it fails miserably, not taking the scale into account. @@ -135,7 +231,7 @@ class EglViewport extends EglElement { * - pass them to the fragment shader * - in the fragment shader, take this scale value into account */ - private void drawFrame(int textureId, float[] textureMatrix, + private void drawFrame(int textureId, int overlayTextureId, float[] textureMatrix, float[] overlayTextureMatrix, FloatBuffer vertexBuffer, FloatBuffer texBuffer) { check("draw start"); @@ -144,9 +240,10 @@ class EglViewport extends EglElement { GLES20.glUseProgram(mProgramHandle); check("glUseProgram"); - // Set the texture. + // Set the camera texture. GLES20.glActiveTexture(GLES20.GL_TEXTURE0); GLES20.glBindTexture(mTextureTarget, textureId); + GLES20.glUniform1i(muTextureLocation, 0); // Copy the model / view / projection matrix over. GLES20.glUniformMatrix4fv(muMVPMatrixLocation, 1, false, IDENTITY_MATRIX, 0); @@ -170,12 +267,22 @@ class EglViewport extends EglElement { GLES20.glVertexAttribPointer(maTextureCoordLocation, 2, GLES20.GL_FLOAT, false, 8, texBuffer); check("glVertexAttribPointer"); + // Set the overlay texture. + if (mHasOverlay) { + GLES20.glActiveTexture(GLES20.GL_TEXTURE1); + GLES20.glBindTexture(mTextureTarget, overlayTextureId); + GLES20.glUniform1i(muOverlayTextureLocation, 1); + + // Copy the texture transformation matrix over. + GLES20.glUniformMatrix4fv(muOverlayTexMatrixLocation, 1, false, overlayTextureMatrix, 0); + check("glUniformMatrix4fv"); + } // Draw the rect. GLES20.glDrawArrays(GLES20.GL_TRIANGLE_STRIP, 0, VERTEX_COUNT); check("glDrawArrays"); - // Done -- disable vertex array, texture, and program. + // Done -- disable vertex array, textures, and program. GLES20.glDisableVertexAttribArray(maPositionLocation); GLES20.glDisableVertexAttribArray(maTextureCoordLocation); GLES20.glBindTexture(mTextureTarget, 0); diff --git a/cameraview/src/main/gles/com/otaliastudios/cameraview/TextureMediaEncoder.java b/cameraview/src/main/gles/com/otaliastudios/cameraview/TextureMediaEncoder.java index 5e72a878..2dc7ddad 100644 --- a/cameraview/src/main/gles/com/otaliastudios/cameraview/TextureMediaEncoder.java +++ b/cameraview/src/main/gles/com/otaliastudios/cameraview/TextureMediaEncoder.java @@ -17,10 +17,12 @@ class TextureMediaEncoder extends VideoMediaEncoder static class Frame { float[] transform; + float[] overlayTransform; long timestamp; } static class Config extends VideoMediaEncoder.Config { int textureId; + int overlayTextureId; float scaleX; float scaleY; boolean scaleFlipped; @@ -29,11 +31,17 @@ class TextureMediaEncoder extends VideoMediaEncoder Config(int width, int height, int bitRate, int frameRate, int rotation, String mimeType, int textureId, float scaleX, float scaleY, boolean scaleFlipped, EGLContext eglContext) { + this(width, height, bitRate, frameRate, rotation, mimeType, textureId, 0, scaleX, scaleY, scaleFlipped, eglContext); + } + + Config(int width, int height, int bitRate, int frameRate, int rotation, String mimeType, + int textureId, int overlayTextureId, float scaleX, float scaleY, boolean scaleFlipped, EGLContext eglContext) { // We rotate the texture using transformRotation. Pass rotation=0 to super so that // no rotation metadata is written into the output file. super(width, height, bitRate, frameRate, 0, mimeType); this.transformRotation = rotation; this.textureId = textureId; + this.overlayTextureId = overlayTextureId; this.scaleX = scaleX; this.scaleY = scaleY; this.scaleFlipped = scaleFlipped; @@ -57,7 +65,7 @@ class TextureMediaEncoder extends VideoMediaEncoder mWindow = new EglWindowSurface(mEglCore, mSurface, true); mWindow.makeCurrent(); // drawing will happen on the InputWindowSurface, which // is backed by mVideoEncoder.getInputSurface() - mViewport = new EglViewport(); + mViewport = new EglViewport(mConfig.overlayTextureId != 0); } @EncoderThread @@ -126,7 +134,7 @@ class TextureMediaEncoder extends VideoMediaEncoder drain(false); // Future note: passing scale values to the viewport? They are scaleX and scaleY, // but flipped based on the mConfig.scaleFlipped boolean. - mViewport.drawFrame(mConfig.textureId, transform); + mViewport.drawFrame(mConfig.textureId, mConfig.overlayTextureId, transform, frame.overlayTransform); mWindow.setPresentationTime(timestamp); mWindow.swapBuffers(); } diff --git a/cameraview/src/main/java/com/otaliastudios/cameraview/CameraView.java b/cameraview/src/main/java/com/otaliastudios/cameraview/CameraView.java index ea7a8429..fabe1c82 100644 --- a/cameraview/src/main/java/com/otaliastudios/cameraview/CameraView.java +++ b/cameraview/src/main/java/com/otaliastudios/cameraview/CameraView.java @@ -24,13 +24,17 @@ import androidx.annotation.ColorInt; import androidx.annotation.NonNull; import androidx.annotation.Nullable; import android.util.AttributeSet; +import android.util.Log; import android.view.MotionEvent; +import android.view.Surface; +import android.view.View; import android.view.ViewGroup; import android.widget.FrameLayout; import java.io.File; import java.util.ArrayList; import java.util.HashMap; +import java.util.LinkedList; import java.util.List; import java.util.concurrent.CopyOnWriteArrayList; @@ -249,7 +253,7 @@ public class CameraView extends FrameLayout implements LifecycleObserver { } case GL_SURFACE: default: { mPreview = Preview.GL_SURFACE; - return new GlCameraPreview(context, container, null); + return new GlCameraPreview(context, container, null, true); } } } @@ -259,6 +263,9 @@ public class CameraView extends FrameLayout implements LifecycleObserver { mCameraController.setPreview(mCameraPreview); } + private OverlayLayout mPreviewOverlayLayout; + private OverlayLayout mNoPreviewOverlayLayout; + @Override protected void onAttachedToWindow() { super.onAttachedToWindow(); @@ -266,6 +273,40 @@ public class CameraView extends FrameLayout implements LifecycleObserver { // isHardwareAccelerated will return the real value only after we are // attached. That's why we instantiate the preview here. instantiatePreview(); + + mPreviewOverlayLayout = findViewById(R.id.preview_overlay_layout); + mNoPreviewOverlayLayout = findViewById(R.id.no_preview_overlay_layout); + mNoPreviewOverlayLayout.setDrawOnScreen(false); + + ((GlCameraPreview) mCameraPreview).addOverlayInputSurfaceListener(new GlCameraPreview.OverlayInputSurfaceListener() { + @Override + public void onSurface(@NonNull Surface surface) { + mPreviewOverlayLayout.setOutputSurface(surface); + mNoPreviewOverlayLayout.setOutputSurface(surface); + } + }); + + LinkedList indexesOverlayViews = new LinkedList<>(); + for (int i = 0; i < getChildCount(); i++) { + View view = getChildAt(i); + if (view.getLayoutParams() instanceof OverlayLayoutParams && + ((OverlayLayoutParams) view.getLayoutParams()).isOverlay) { + indexesOverlayViews.add(i); + } + } + for (Integer i : indexesOverlayViews) { + if (i != null && getChildAt(i) != null) { + View view = getChildAt(i); + + removeViewAt(i); + if (((OverlayLayoutParams) view.getLayoutParams()).showInPreview) { + mPreviewOverlayLayout.addView(view); + } else { + mNoPreviewOverlayLayout.addView(view); + } + + } + } } if (!isInEditMode()) { mOrientationHelper.enable(getContext()); @@ -280,6 +321,26 @@ public class CameraView extends FrameLayout implements LifecycleObserver { super.onDetachedFromWindow(); } + @Override + protected boolean checkLayoutParams(ViewGroup.LayoutParams p) { + return p instanceof OverlayLayoutParams; + } + + @Override + protected OverlayLayoutParams generateDefaultLayoutParams() { + return new OverlayLayoutParams(OverlayLayoutParams.WRAP_CONTENT, OverlayLayoutParams.WRAP_CONTENT); + } + + @Override + public OverlayLayoutParams generateLayoutParams(AttributeSet attributeSet) { + return new OverlayLayoutParams(this.getContext(), attributeSet); + } + + @Override + protected OverlayLayoutParams generateLayoutParams(ViewGroup.LayoutParams p) { + return new OverlayLayoutParams(p); + } + //endregion //region Measuring behavior @@ -1824,6 +1885,71 @@ public class CameraView extends FrameLayout implements LifecycleObserver { } } + public static class OverlayLayoutParams extends FrameLayout.LayoutParams { + + private boolean isOverlay = false; + private boolean showInPreview = false; + private boolean showInPicture = false; + private boolean showInVideo = false; + + public OverlayLayoutParams(Context context, AttributeSet attributeSet) { + super(context, attributeSet); + this.readStyleParameters(context, attributeSet); + } + + public OverlayLayoutParams(int width, int height) { + super(width, height); + } + + public OverlayLayoutParams(ViewGroup.LayoutParams layoutParams) { + super(layoutParams); + } + + private void readStyleParameters(Context context, AttributeSet attributeSet) { + TypedArray a = context.obtainStyledAttributes(attributeSet, R.styleable.CameraView_Layout); + try { + this.isOverlay = a.getBoolean(R.styleable.CameraView_Layout_layout_overlay, false); + this.showInPreview = a.getBoolean(R.styleable.CameraView_Layout_layout_showInPreview, false); + this.showInPicture = a.getBoolean(R.styleable.CameraView_Layout_layout_showInPicture, false); + this.showInVideo = a.getBoolean(R.styleable.CameraView_Layout_layout_showInVideo, false); + } finally { + a.recycle(); + } + } + + public boolean isOverlay() { + return isOverlay; + } + + public void setOverlay(boolean overlay) { + isOverlay = overlay; + } + + public boolean isShowInPreview() { + return showInPreview; + } + + public void setShowInPreview(boolean showInPreview) { + this.showInPreview = showInPreview; + } + + public boolean isShowInPicture() { + return showInPicture; + } + + public void setShowInPicture(boolean showInPicture) { + this.showInPicture = showInPicture; + } + + public boolean isShowInVideo() { + return showInVideo; + } + + public void setShowInVideo(boolean showInVideo) { + this.showInVideo = showInVideo; + } + } + //endregion //region deprecated APIs diff --git a/cameraview/src/main/java/com/otaliastudios/cameraview/OverlayLayout.java b/cameraview/src/main/java/com/otaliastudios/cameraview/OverlayLayout.java new file mode 100644 index 00000000..b30a843d --- /dev/null +++ b/cameraview/src/main/java/com/otaliastudios/cameraview/OverlayLayout.java @@ -0,0 +1,115 @@ +package com.otaliastudios.cameraview; + +import android.content.Context; +import android.graphics.Canvas; +import android.graphics.Color; +import android.graphics.Paint; +import android.util.AttributeSet; +import android.util.Log; +import android.view.Surface; +import android.view.View; +import android.view.ViewGroup; +import android.widget.FrameLayout; + +import androidx.annotation.NonNull; +import androidx.annotation.Nullable; + +public class OverlayLayout extends FrameLayout { + + private Surface outputSurface = null; + private boolean drawOnScreen = true; + + public OverlayLayout(@NonNull Context context) { + super(context); + setWillNotDraw(false); + } + + public OverlayLayout(@NonNull Context context, @Nullable AttributeSet attrs) { + super(context, attrs); + setWillNotDraw(false); + } + + @Override + public void draw(Canvas canvas) { + if (outputSurface != null ) { + // Requires a try/catch for .lockCanvas( null ) + try { + final Canvas surfaceCanvas = outputSurface.lockCanvas(null); + + Paint drawPaint = new Paint(); + drawPaint.setColor(Color.GREEN); + drawPaint.setStrokeWidth(5); + drawPaint.setStyle(Paint.Style.FILL_AND_STROKE); + surfaceCanvas.drawCircle(200, 200, 40, drawPaint); + + float xScale = surfaceCanvas.getWidth() / (float) canvas.getWidth(); + surfaceCanvas.scale(xScale, xScale); + super.draw(surfaceCanvas); + + outputSurface.unlockCanvasAndPost(surfaceCanvas); + } catch (Surface.OutOfResourcesException e) { + e.printStackTrace(); + } + } + + if (drawOnScreen) { + // draw the view normally + super.draw(canvas); + } + } + +// @Override +// protected void onDraw(Canvas canvas) { +// if (outputSurface != null ) { +// // Requires a try/catch for .lockCanvas( null ) +// try { +// final Canvas surfaceCanvas = outputSurface.lockCanvas(null); +// super.onDraw(surfaceCanvas); +// +// Log.d("ciao", "onDraw: canvas size: " + surfaceCanvas.getWidth() + " " + surfaceCanvas.getHeight()); +// Log.d("ciao", "onDraw: w h: " + getWidth() + " " + getHeight()); +// +// Paint drawPaint = new Paint(); +// drawPaint.setColor(Color.GREEN); +// drawPaint.setStrokeWidth(5); +// drawPaint.setStyle(Paint.Style.FILL_AND_STROKE); +// +// surfaceCanvas.drawCircle(200, 200, 40, drawPaint); +// +// outputSurface.unlockCanvasAndPost(surfaceCanvas); +// } catch (Surface.OutOfResourcesException e) { +// e.printStackTrace(); +// } +// } +// +// if (drawOnScreen) { +// // draw the view normally +// super.onDraw(canvas); +// } +// } + + public void setOutputSurface(Surface outputSurface) { + this.outputSurface = outputSurface; + + // force redrawing + postInvalidate(); + postInvalidateRecursive(this); + } + + public void setDrawOnScreen(boolean drawOnScreen) { + this.drawOnScreen = drawOnScreen; + } + + // invalidates children (and nested children) recursively + private void postInvalidateRecursive(ViewGroup layout) { + int count = layout.getChildCount(); + View child; + for (int i = 0; i < count; i++) { + child = layout.getChildAt(i); + if(child instanceof ViewGroup) + postInvalidateRecursive((ViewGroup) child); + else + child.postInvalidate(); + } + } +} diff --git a/cameraview/src/main/java/com/otaliastudios/cameraview/SnapshotPictureRecorder.java b/cameraview/src/main/java/com/otaliastudios/cameraview/SnapshotPictureRecorder.java index 8dd3e7c9..22990200 100644 --- a/cameraview/src/main/java/com/otaliastudios/cameraview/SnapshotPictureRecorder.java +++ b/cameraview/src/main/java/com/otaliastudios/cameraview/SnapshotPictureRecorder.java @@ -10,6 +10,8 @@ import android.hardware.Camera; import android.opengl.EGL14; import android.opengl.EGLContext; import android.opengl.Matrix; +import android.util.Log; + import androidx.annotation.NonNull; import java.io.ByteArrayOutputStream; @@ -54,23 +56,38 @@ class SnapshotPictureRecorder extends PictureRecorder { preview.addRendererFrameCallback(new GlCameraPreview.RendererFrameCallback() { int mTextureId; + int mOverlayTextureId = 0; SurfaceTexture mSurfaceTexture; + SurfaceTexture mOverlaySurfaceTexture; float[] mTransform; + float[] mOverlayTransform; @RendererThread - public void onRendererTextureCreated(int textureId) { - mTextureId = textureId; + public void onRendererTextureCreated(int... textureIds) { + mTextureId = textureIds[0]; + if (textureIds.length > 1) { + mOverlayTextureId = textureIds[1]; + } mSurfaceTexture = new SurfaceTexture(mTextureId, true); + if (mOverlayTextureId != 0) { + mOverlaySurfaceTexture = new SurfaceTexture(mOverlayTextureId, true); + } // Need to crop the size. Rect crop = CropHelper.computeCrop(mResult.size, mOutputRatio); mResult.size = new Size(crop.width(), crop.height()); mSurfaceTexture.setDefaultBufferSize(mResult.size.getWidth(), mResult.size.getHeight()); + if (mOverlaySurfaceTexture != null) { + mSurfaceTexture.setDefaultBufferSize(mResult.size.getWidth(), mResult.size.getHeight()); + } mTransform = new float[16]; + if (mOverlaySurfaceTexture != null) { + mOverlayTransform = new float[16]; + } } @RendererThread @Override - public void onRendererFrame(SurfaceTexture surfaceTexture, final float scaleX, final float scaleY) { + public void onRendererFrame(SurfaceTexture surfaceTexture, SurfaceTexture overlaySurfaceTexture, final float scaleX, final float scaleY) { preview.removeRendererFrameCallback(this); // This kinda work but has drawbacks: @@ -99,9 +116,13 @@ class SnapshotPictureRecorder extends PictureRecorder { public void run() { EglWindowSurface surface = new EglWindowSurface(core, mSurfaceTexture); surface.makeCurrent(); - EglViewport viewport = new EglViewport(); + EglViewport viewport = new EglViewport(mOverlaySurfaceTexture != null); mSurfaceTexture.updateTexImage(); mSurfaceTexture.getTransformMatrix(mTransform); + if (mOverlaySurfaceTexture != null) { + mOverlaySurfaceTexture.updateTexImage(); + mOverlaySurfaceTexture.getTransformMatrix(mOverlayTransform); + } // Apply scale and crop: // NOTE: scaleX and scaleY are in REF_VIEW, while our input appears to be in REF_SENSOR. @@ -124,16 +145,22 @@ class SnapshotPictureRecorder extends PictureRecorder { // Future note: passing scale values to the viewport? // They are simply realScaleX and realScaleY. - viewport.drawFrame(mTextureId, mTransform); + viewport.drawFrame(mTextureId, mOverlayTextureId, mTransform, mOverlayTransform); // don't - surface.swapBuffers(); mResult.data = surface.saveFrameTo(Bitmap.CompressFormat.JPEG); mResult.format = PictureResult.FORMAT_JPEG; mSurfaceTexture.releaseTexImage(); + if (mOverlaySurfaceTexture != null) { + mOverlaySurfaceTexture.releaseTexImage(); + } // EGL14.eglMakeCurrent(oldDisplay, oldSurface, oldSurface, eglContext); surface.release(); viewport.release(); mSurfaceTexture.release(); + if (mOverlaySurfaceTexture != null) { + mOverlaySurfaceTexture.release(); + } core.release(); dispatchResult(); } diff --git a/cameraview/src/main/java/com/otaliastudios/cameraview/SnapshotVideoRecorder.java b/cameraview/src/main/java/com/otaliastudios/cameraview/SnapshotVideoRecorder.java index c3003f18..9cf92952 100644 --- a/cameraview/src/main/java/com/otaliastudios/cameraview/SnapshotVideoRecorder.java +++ b/cameraview/src/main/java/com/otaliastudios/cameraview/SnapshotVideoRecorder.java @@ -31,6 +31,7 @@ class SnapshotVideoRecorder extends VideoRecorder implements GlCameraPreview.Ren private int mCurrentState = STATE_NOT_RECORDING; private int mDesiredState = STATE_NOT_RECORDING; private int mTextureId = 0; + private int mOverlayTextureId = 0; SnapshotVideoRecorder(@NonNull VideoResult stub, @Nullable VideoResultListener listener, @NonNull GlCameraPreview preview) { super(stub, listener); @@ -50,13 +51,16 @@ class SnapshotVideoRecorder extends VideoRecorder implements GlCameraPreview.Ren @RendererThread @Override - public void onRendererTextureCreated(int textureId) { - mTextureId = textureId; + public void onRendererTextureCreated(int... textureId) { + mTextureId = textureId[0]; + if (textureId.length > 1) { + mOverlayTextureId = textureId[1]; + } } @RendererThread @Override - public void onRendererFrame(@NonNull SurfaceTexture surfaceTexture, float scaleX, float scaleY) { + public void onRendererFrame(@NonNull SurfaceTexture surfaceTexture, SurfaceTexture overlaySurfaceTexture, float scaleX, float scaleY) { if (mCurrentState == STATE_NOT_RECORDING && mDesiredState == STATE_RECORDING) { Size size = mResult.getSize(); // Ensure width and height are divisible by 2, as I have read somewhere. @@ -79,6 +83,7 @@ class SnapshotVideoRecorder extends VideoRecorder implements GlCameraPreview.Ren mResult.videoFrameRate, mResult.rotation, type, mTextureId, + mOverlayTextureId, scaleX, scaleY, mPreview.mInputFlipped, EGL14.eglGetCurrentContext() @@ -99,7 +104,11 @@ class SnapshotVideoRecorder extends VideoRecorder implements GlCameraPreview.Ren TextureMediaEncoder.Frame frame = new TextureMediaEncoder.Frame(); frame.timestamp = surfaceTexture.getTimestamp(); frame.transform = new float[16]; // TODO would be cool to avoid this at every frame. But it's not easy. + frame.overlayTransform = new float[16]; surfaceTexture.getTransformMatrix(frame.transform); + if (overlaySurfaceTexture != null) { + overlaySurfaceTexture.getTransformMatrix(frame.overlayTransform); + } mEncoderEngine.notify(TextureMediaEncoder.FRAME_EVENT, frame); } diff --git a/cameraview/src/main/res/layout/cameraview_gl_view.xml b/cameraview/src/main/res/layout/cameraview_gl_view.xml index 0525ee6e..f18da9ec 100644 --- a/cameraview/src/main/res/layout/cameraview_gl_view.xml +++ b/cameraview/src/main/res/layout/cameraview_gl_view.xml @@ -6,9 +6,19 @@ android:layout_gravity="center" android:gravity="center"> + + + + \ No newline at end of file diff --git a/cameraview/src/main/res/values/attrs.xml b/cameraview/src/main/res/values/attrs.xml index f25628fe..72e46ff8 100644 --- a/cameraview/src/main/res/values/attrs.xml +++ b/cameraview/src/main/res/values/attrs.xml @@ -122,4 +122,16 @@ + + + + + + + + + + + + \ No newline at end of file diff --git a/cameraview/src/main/views/com/otaliastudios/cameraview/CameraPreview.java b/cameraview/src/main/views/com/otaliastudios/cameraview/CameraPreview.java index 43bb0ec2..237297c1 100644 --- a/cameraview/src/main/views/com/otaliastudios/cameraview/CameraPreview.java +++ b/cameraview/src/main/views/com/otaliastudios/cameraview/CameraPreview.java @@ -34,9 +34,16 @@ abstract class CameraPreview { protected int mInputStreamHeight; protected boolean mInputFlipped; + private final boolean mHasOverlay; + CameraPreview(@NonNull Context context, @NonNull ViewGroup parent, @Nullable SurfaceCallback callback) { + this(context, parent, callback, false); + } + + CameraPreview(@NonNull Context context, @NonNull ViewGroup parent, @Nullable SurfaceCallback callback, boolean hasOverlay) { mView = onCreateView(context, parent); mSurfaceCallback = callback; + mHasOverlay = hasOverlay; } @NonNull @@ -162,4 +169,8 @@ abstract class CameraPreview { /* not final for tests */ boolean isCropping() { return mCropping; } + + protected final boolean hasOverlay() { + return mHasOverlay; + } } diff --git a/cameraview/src/main/views/com/otaliastudios/cameraview/GlCameraPreview.java b/cameraview/src/main/views/com/otaliastudios/cameraview/GlCameraPreview.java index 44ece2a9..7091baed 100644 --- a/cameraview/src/main/views/com/otaliastudios/cameraview/GlCameraPreview.java +++ b/cameraview/src/main/views/com/otaliastudios/cameraview/GlCameraPreview.java @@ -7,11 +7,14 @@ import android.opengl.Matrix; import androidx.annotation.NonNull; import androidx.annotation.Nullable; +import android.util.Log; import android.view.LayoutInflater; +import android.view.Surface; import android.view.SurfaceHolder; import android.view.View; import android.view.ViewGroup; +import java.util.ArrayList; import java.util.Collections; import java.util.HashSet; import java.util.Set; @@ -55,8 +58,11 @@ class GlCameraPreview extends CameraPreview imple private boolean mDispatched; private final float[] mTransformMatrix = new float[16]; - private int mOutputTextureId = 0; + private final float[] mOverlayTransformMatrix = new float[16]; + private int[] mOutputTextureIds = new int[]{0, 0}; private SurfaceTexture mInputSurfaceTexture; + private SurfaceTexture mOverlaySurfaceTexture; + private Surface mOverlaySurface; private EglViewport mOutputViewport; private Set mRendererFrameCallbacks = Collections.synchronizedSet(new HashSet()); /* for tests */ float mScaleX = 1F; @@ -64,8 +70,10 @@ class GlCameraPreview extends CameraPreview imple private View mRootView; - GlCameraPreview(@NonNull Context context, @NonNull ViewGroup parent, @Nullable SurfaceCallback callback) { - super(context, parent, callback); + private ArrayList overlayListeners = new ArrayList<>(); + + GlCameraPreview(@NonNull Context context, @NonNull ViewGroup parent, @Nullable SurfaceCallback callback, boolean hasOverlay) { + super(context, parent, callback, hasOverlay); } @NonNull @@ -122,7 +130,16 @@ class GlCameraPreview extends CameraPreview imple mInputSurfaceTexture.release(); mInputSurfaceTexture = null; } - mOutputTextureId = 0; + if (mOverlaySurface != null) { + mOverlaySurface.release(); + mOverlaySurface = null; + } + if (mOverlaySurfaceTexture != null) { + mOverlaySurfaceTexture.release(); + mOverlaySurfaceTexture = null; + } + mOutputTextureIds[0] = 0; + mOutputTextureIds[1] = 0; if (mOutputViewport != null) { mOutputViewport.release(); mOutputViewport = null; @@ -132,14 +149,31 @@ class GlCameraPreview extends CameraPreview imple @RendererThread @Override public void onSurfaceCreated(GL10 gl, EGLConfig config) { - mOutputViewport = new EglViewport(); - mOutputTextureId = mOutputViewport.createTexture(); - mInputSurfaceTexture = new SurfaceTexture(mOutputTextureId); + mOutputViewport = new EglViewport(hasOverlay()); + if (hasOverlay()) { + mOutputTextureIds = mOutputViewport.createTextures(); + mOverlaySurfaceTexture = new SurfaceTexture(mOutputTextureIds[1]); + mOverlaySurface = new Surface(mOverlaySurfaceTexture); + for (OverlayInputSurfaceListener listener : overlayListeners) { + if (listener != null) { + listener.onSurface(mOverlaySurface); + } + } + + mOverlaySurfaceTexture.setDefaultBufferSize(mInputStreamWidth, mInputStreamHeight); + } else { + mOutputTextureIds[0] = mOutputViewport.createTexture(); + } + mInputSurfaceTexture = new SurfaceTexture(mOutputTextureIds[0]); getView().queueEvent(new Runnable() { @Override public void run() { for (RendererFrameCallback callback : mRendererFrameCallbacks) { - callback.onRendererTextureCreated(mOutputTextureId); + if (mOutputTextureIds[1] != 0) { + callback.onRendererTextureCreated(mOutputTextureIds[0], mOutputTextureIds[1]); + } else { + callback.onRendererTextureCreated(mOutputTextureIds[0]); + } } } }); @@ -186,12 +220,18 @@ class GlCameraPreview extends CameraPreview imple // Latch the latest frame. If there isn't anything new, // we'll just re-use whatever was there before. mInputSurfaceTexture.updateTexImage(); + if (hasOverlay()) { + mOverlaySurfaceTexture.updateTexImage(); + } if (mInputStreamWidth <= 0 || mInputStreamHeight <= 0) { // Skip drawing. Camera was not opened. return; } mInputSurfaceTexture.getTransformMatrix(mTransformMatrix); + if (hasOverlay()) { + mOverlaySurfaceTexture.getTransformMatrix(mOverlayTransformMatrix); + } if (isCropping()) { // Scaling is easy. However: // If the view is 10x1000 (very tall), it will show only the left strip of the preview (not the center one). @@ -201,12 +241,21 @@ class GlCameraPreview extends CameraPreview imple float translY = (1F - mScaleY) / 2F; Matrix.translateM(mTransformMatrix, 0, translX, translY, 0); Matrix.scaleM(mTransformMatrix, 0, mScaleX, mScaleY, 1); + + if (hasOverlay()) { + Matrix.translateM(mOverlayTransformMatrix, 0, translX, translY, 0); + Matrix.scaleM(mOverlayTransformMatrix, 0, mScaleX, mScaleY, 1); + } } // Future note: passing scale to the viewport? // They are scaleX an scaleY, but flipped based on mInputFlipped. - mOutputViewport.drawFrame(mOutputTextureId, mTransformMatrix); + if (hasOverlay() && mOutputTextureIds[1] != 0) { + mOutputViewport.drawFrame(mOutputTextureIds[0], mOutputTextureIds[1], mTransformMatrix, mOverlayTransformMatrix); + } else { + mOutputViewport.drawFrame(mOutputTextureIds[0], mTransformMatrix); + } for (RendererFrameCallback callback : mRendererFrameCallbacks) { - callback.onRendererFrame(mInputSurfaceTexture, mScaleX, mScaleY); + callback.onRendererFrame(mInputSurfaceTexture, mOverlaySurfaceTexture, mScaleX, mScaleY); } } @@ -267,10 +316,10 @@ class GlCameraPreview extends CameraPreview imple * Called on the renderer thread, hopefully only once, to notify that * the texture was created (or to inform a new callback of the old texture). * - * @param textureId the GL texture linked to the image stream + * @param textureId the GL textures linked to the image stream */ @RendererThread - void onRendererTextureCreated(int textureId); + void onRendererTextureCreated(int... textureId); /** * Called on the renderer thread after each frame was drawn. @@ -282,7 +331,7 @@ class GlCameraPreview extends CameraPreview imple * @param scaleY the scaleY (in REF_VIEW) value */ @RendererThread - void onRendererFrame(SurfaceTexture surfaceTexture, float scaleX, float scaleY); + void onRendererFrame(SurfaceTexture surfaceTexture, SurfaceTexture overlaySurfaceTexture, float scaleX, float scaleY); } void addRendererFrameCallback(@NonNull final RendererFrameCallback callback) { @@ -290,7 +339,10 @@ class GlCameraPreview extends CameraPreview imple @Override public void run() { mRendererFrameCallbacks.add(callback); - if (mOutputTextureId != 0) callback.onRendererTextureCreated(mOutputTextureId); + if (mOutputTextureIds[0] != 0 && mOutputTextureIds[1] != 0) { + callback.onRendererTextureCreated(mOutputTextureIds); + } + else if (mOutputTextureIds[0] != 0) callback.onRendererTextureCreated(mOutputTextureIds[0]); } }); } @@ -298,4 +350,15 @@ class GlCameraPreview extends CameraPreview imple void removeRendererFrameCallback(@NonNull final RendererFrameCallback callback) { mRendererFrameCallbacks.remove(callback); } + + public void addOverlayInputSurfaceListener(@NonNull OverlayInputSurfaceListener listener) { + overlayListeners.add(listener); + if (mOverlaySurface != null) { + listener.onSurface(mOverlaySurface); + } + } + + interface OverlayInputSurfaceListener { + void onSurface(@NonNull Surface surface); + } } diff --git a/demo/src/main/res/layout/activity_camera.xml b/demo/src/main/res/layout/activity_camera.xml index 91c43953..76b0251d 100644 --- a/demo/src/main/res/layout/activity_camera.xml +++ b/demo/src/main/res/layout/activity_camera.xml @@ -26,7 +26,21 @@ app:cameraGesturePinch="zoom" app:cameraGestureScrollHorizontal="exposureCorrection" app:cameraGestureScrollVertical="none" - app:cameraMode="picture" /> + app:cameraMode="picture"> + + + +