From d6eea6e08f7168d05cbf7f72e9d5998a7cd52113 Mon Sep 17 00:00:00 2001 From: Mattia Iavarone Date: Wed, 31 Jul 2019 13:52:31 +0200 Subject: [PATCH] Draw overlays on the encoder thread --- .../internal/Issue514Workaround.java | 8 +- .../cameraview/overlay/OverlayDrawer.java | 79 +++++++++++++++++++ .../picture/SnapshotGlPictureRecorder.java | 43 +++------- .../video/SnapshotVideoRecorder.java | 54 +++---------- .../video/encoding/TextureConfig.java | 16 ++-- .../video/encoding/TextureMediaEncoder.java | 16 ++-- 6 files changed, 113 insertions(+), 103 deletions(-) create mode 100644 cameraview/src/main/java/com/otaliastudios/cameraview/overlay/OverlayDrawer.java diff --git a/cameraview/src/main/java/com/otaliastudios/cameraview/internal/Issue514Workaround.java b/cameraview/src/main/java/com/otaliastudios/cameraview/internal/Issue514Workaround.java index aae0468a..62733a5a 100644 --- a/cameraview/src/main/java/com/otaliastudios/cameraview/internal/Issue514Workaround.java +++ b/cameraview/src/main/java/com/otaliastudios/cameraview/internal/Issue514Workaround.java @@ -99,11 +99,9 @@ import com.otaliastudios.cameraview.preview.RendererThread; public class Issue514Workaround { private final int cameraTextureId; - private final boolean hasOverlay; - public Issue514Workaround(int cameraTextureId, boolean hasOverlay) { + public Issue514Workaround(int cameraTextureId) { this.cameraTextureId = cameraTextureId; - this.hasOverlay = hasOverlay; } public void beforeOverlayUpdateTexImage() { @@ -119,8 +117,6 @@ public class Issue514Workaround { } private void bindTexture(int textureId) { - if (hasOverlay) { - GLES20.glBindTexture(GLES11Ext.GL_TEXTURE_EXTERNAL_OES, textureId); - } + GLES20.glBindTexture(GLES11Ext.GL_TEXTURE_EXTERNAL_OES, textureId); } } diff --git a/cameraview/src/main/java/com/otaliastudios/cameraview/overlay/OverlayDrawer.java b/cameraview/src/main/java/com/otaliastudios/cameraview/overlay/OverlayDrawer.java new file mode 100644 index 00000000..06bd74c9 --- /dev/null +++ b/cameraview/src/main/java/com/otaliastudios/cameraview/overlay/OverlayDrawer.java @@ -0,0 +1,79 @@ +package com.otaliastudios.cameraview.overlay; + +import android.graphics.Canvas; +import android.graphics.Color; +import android.graphics.PorterDuff; +import android.graphics.SurfaceTexture; +import android.view.Surface; + +import androidx.annotation.NonNull; + +import com.otaliastudios.cameraview.CameraLogger; +import com.otaliastudios.cameraview.internal.Issue514Workaround; +import com.otaliastudios.cameraview.internal.egl.EglViewport; +import com.otaliastudios.cameraview.size.Size; + +public class OverlayDrawer { + + private static final String TAG = OverlayDrawer.class.getSimpleName(); + private static final CameraLogger LOG = CameraLogger.create(TAG); + + private Overlay mOverlay; + private int mTextureId; + private SurfaceTexture mSurfaceTexture; + private Surface mSurface; + private EglViewport mViewport; + private Issue514Workaround mIssue514Workaround; + private float[] mTransform = new float[16]; + + public OverlayDrawer(@NonNull Overlay overlay, @NonNull Size size, int cameraTextureId) { + mOverlay = overlay; + mViewport = new EglViewport(); + mTextureId = mViewport.createTexture(); + mSurfaceTexture = new SurfaceTexture(mTextureId); + mSurfaceTexture.setDefaultBufferSize(size.getWidth(), size.getHeight()); + mSurface = new Surface(mSurfaceTexture); + mIssue514Workaround = new Issue514Workaround(cameraTextureId); + } + + + public void draw(@NonNull Overlay.Target target) { + try { + final Canvas surfaceCanvas = mSurface.lockCanvas(null); + surfaceCanvas.drawColor(Color.TRANSPARENT, PorterDuff.Mode.CLEAR); + mOverlay.drawOn(target, surfaceCanvas); + mSurface.unlockCanvasAndPost(surfaceCanvas); + } catch (Surface.OutOfResourcesException e) { + LOG.w("Got Surface.OutOfResourcesException while drawing video overlays", e); + } + mIssue514Workaround.beforeOverlayUpdateTexImage(); + mSurfaceTexture.updateTexImage(); + mSurfaceTexture.getTransformMatrix(mTransform); + } + + public float[] getTransform() { + return mTransform; + } + + public void render() { + mIssue514Workaround.afterOverlayGlDrawn(); + mViewport.drawFrame(mTextureId, mTransform); + mIssue514Workaround.afterOverlayGlDrawn(); + } + + public void release() { + mIssue514Workaround.end(); + if (mSurfaceTexture != null) { + mSurfaceTexture.release(); + mSurfaceTexture = null; + } + if (mSurface != null) { + mSurface.release(); + mSurface = null; + } + if (mViewport != null) { + mViewport.release(); + mViewport = null; + } + } +} diff --git a/cameraview/src/main/java/com/otaliastudios/cameraview/picture/SnapshotGlPictureRecorder.java b/cameraview/src/main/java/com/otaliastudios/cameraview/picture/SnapshotGlPictureRecorder.java index bd10e876..c4c63045 100644 --- a/cameraview/src/main/java/com/otaliastudios/cameraview/picture/SnapshotGlPictureRecorder.java +++ b/cameraview/src/main/java/com/otaliastudios/cameraview/picture/SnapshotGlPictureRecorder.java @@ -26,6 +26,7 @@ import com.otaliastudios.cameraview.internal.egl.EglViewport; import com.otaliastudios.cameraview.internal.egl.EglWindowSurface; import com.otaliastudios.cameraview.internal.utils.CropHelper; import com.otaliastudios.cameraview.internal.utils.WorkerHandler; +import com.otaliastudios.cameraview.overlay.OverlayDrawer; import com.otaliastudios.cameraview.preview.GlCameraPreview; import com.otaliastudios.cameraview.preview.RendererFrameCallback; import com.otaliastudios.cameraview.preview.RendererThread; @@ -65,14 +66,11 @@ public class SnapshotGlPictureRecorder extends PictureRecorder { private Overlay mOverlay; private boolean mHasOverlay; + private OverlayDrawer mOverlayDrawer; private int mTextureId; private float[] mTransform; - private int mOverlayTextureId = 0; - private SurfaceTexture mOverlaySurfaceTexture; - private Surface mOverlaySurface; - private float[] mOverlayTransform; private EglViewport mViewport; @@ -121,11 +119,7 @@ public class SnapshotGlPictureRecorder extends PictureRecorder { Matrix.setIdentityM(mTransform, 0); if (mHasOverlay) { - mOverlayTextureId = mViewport.createTexture(); - mOverlaySurfaceTexture = new SurfaceTexture(mOverlayTextureId, true); - mOverlaySurfaceTexture.setDefaultBufferSize(mResult.size.getWidth(), mResult.size.getHeight()); - mOverlaySurface = new Surface(mOverlaySurfaceTexture); - mOverlayTransform = new float[16]; + mOverlayDrawer = new OverlayDrawer(mOverlay, mResult.size, textureId); } } @@ -169,7 +163,6 @@ public class SnapshotGlPictureRecorder extends PictureRecorder { final int fakeOutputTextureId = 9999; SurfaceTexture fakeOutputSurface = new SurfaceTexture(fakeOutputTextureId); fakeOutputSurface.setDefaultBufferSize(mResult.size.getWidth(), mResult.size.getHeight()); - final Issue514Workaround issue514Workaround = new Issue514Workaround(mTextureId, mHasOverlay); // 1. Create an EGL surface final EglCore core = new EglCore(eglContext, EglCore.FLAG_RECORDABLE); @@ -195,46 +188,30 @@ public class SnapshotGlPictureRecorder extends PictureRecorder { Matrix.translateM(mTransform, 0, -0.5F, -0.5F, 0); // Go back to old position // 4. Do pretty much the same for overlays - issue514Workaround.beforeOverlayUpdateTexImage(); if (mHasOverlay) { // 1. First we must draw on the texture and get latest image - try { - final Canvas surfaceCanvas = mOverlaySurface.lockCanvas(null); - surfaceCanvas.drawColor(Color.TRANSPARENT, PorterDuff.Mode.CLEAR); - mOverlay.drawOn(Overlay.Target.PICTURE_SNAPSHOT, surfaceCanvas); - mOverlaySurface.unlockCanvasAndPost(surfaceCanvas); - } catch (Surface.OutOfResourcesException e) { - LOG.w("Got Surface.OutOfResourcesException while drawing picture overlays", e); - } - mOverlaySurfaceTexture.updateTexImage(); - mOverlaySurfaceTexture.getTransformMatrix(mOverlayTransform); + mOverlayDrawer.draw(Overlay.Target.PICTURE_SNAPSHOT); // 2. Then we can apply the transformations int rotation = mEngine.getAngles().offset(Reference.VIEW, Reference.OUTPUT, Axis.ABSOLUTE); - Matrix.translateM(mOverlayTransform, 0, 0.5F, 0.5F, 0); - Matrix.rotateM(mOverlayTransform, 0, rotation, 0, 0, 1); + Matrix.translateM(mOverlayDrawer.getTransform(), 0, 0.5F, 0.5F, 0); + Matrix.rotateM(mOverlayDrawer.getTransform(), 0, rotation, 0, 0, 1); // No need to flip the x axis for front camera, but need to flip the y axis always. - Matrix.scaleM(mOverlayTransform, 0, 1, -1, 1); - Matrix.translateM(mOverlayTransform, 0, -0.5F, -0.5F, 0); + Matrix.scaleM(mOverlayDrawer.getTransform(), 0, 1, -1, 1); + Matrix.translateM(mOverlayDrawer.getTransform(), 0, -0.5F, -0.5F, 0); } // 5. Draw and save mViewport.drawFrame(mTextureId, mTransform); - if (mHasOverlay) mViewport.drawFrame(mOverlayTextureId, mOverlayTransform); - issue514Workaround.afterOverlayGlDrawn(); + if (mHasOverlay) mOverlayDrawer.render(); mResult.format = PictureResult.FORMAT_JPEG; mResult.data = eglSurface.saveFrameTo(Bitmap.CompressFormat.JPEG); // 6. Cleanup - issue514Workaround.end(); eglSurface.releaseEglSurface(); mViewport.release(); fakeOutputSurface.release(); - if (mHasOverlay) { - mOverlaySurfaceTexture.releaseTexImage(); - mOverlaySurface.release(); - mOverlaySurfaceTexture.release(); - } + if (mHasOverlay) mOverlayDrawer.release(); core.release(); dispatchResult(); } diff --git a/cameraview/src/main/java/com/otaliastudios/cameraview/video/SnapshotVideoRecorder.java b/cameraview/src/main/java/com/otaliastudios/cameraview/video/SnapshotVideoRecorder.java index ed93501f..37b7d1cb 100644 --- a/cameraview/src/main/java/com/otaliastudios/cameraview/video/SnapshotVideoRecorder.java +++ b/cameraview/src/main/java/com/otaliastudios/cameraview/video/SnapshotVideoRecorder.java @@ -15,6 +15,7 @@ import com.otaliastudios.cameraview.VideoResult; import com.otaliastudios.cameraview.controls.Audio; import com.otaliastudios.cameraview.engine.CameraEngine; import com.otaliastudios.cameraview.internal.egl.EglViewport; +import com.otaliastudios.cameraview.overlay.OverlayDrawer; import com.otaliastudios.cameraview.preview.GlCameraPreview; import com.otaliastudios.cameraview.preview.RendererFrameCallback; import com.otaliastudios.cameraview.preview.RendererThread; @@ -60,15 +61,11 @@ public class SnapshotVideoRecorder extends VideoRecorder implements RendererFram private int mDesiredState = STATE_NOT_RECORDING; private int mTextureId = 0; - private int mOverlayTextureId = 0; - private SurfaceTexture mOverlaySurfaceTexture; - private Surface mOverlaySurface; private Overlay mOverlay; + private OverlayDrawer mOverlayDrawer; private boolean mHasOverlay; private int mOverlayRotation; - private Issue514Workaround mIssue514Workaround; - public SnapshotVideoRecorder(@NonNull CameraEngine engine, @NonNull GlCameraPreview preview, @Nullable Overlay overlay, @@ -96,14 +93,8 @@ public class SnapshotVideoRecorder extends VideoRecorder implements RendererFram public void onRendererTextureCreated(int textureId) { mTextureId = textureId; if (mHasOverlay) { - EglViewport temp = new EglViewport(); - mOverlayTextureId = temp.createTexture(); - mOverlaySurfaceTexture = new SurfaceTexture(mOverlayTextureId); - mOverlaySurfaceTexture.setDefaultBufferSize(mResult.size.getWidth(), mResult.size.getHeight()); - mOverlaySurface = new Surface(mOverlaySurfaceTexture); - temp.release(true); + mOverlayDrawer = new OverlayDrawer(mOverlay, mResult.size, textureId); } - mIssue514Workaround = new Issue514Workaround(textureId, mHasOverlay); } @RendererThread @@ -140,13 +131,13 @@ public class SnapshotVideoRecorder extends VideoRecorder implements RendererFram videoConfig.textureId = mTextureId; videoConfig.scaleX = scaleX; videoConfig.scaleY = scaleY; - videoConfig.issue514Workaround = mIssue514Workaround; // Get egl context from the RendererThread, which is the one in which we have created // the textureId and the overlayTextureId, managed by the GlSurfaceView. // Next operations can then be performed on different threads using this handle. videoConfig.eglContext = EGL14.eglGetCurrentContext(); if (mHasOverlay) { - videoConfig.overlayTextureId = mOverlayTextureId; + videoConfig.overlayTarget = Overlay.Target.VIDEO_SNAPSHOT; + videoConfig.overlayDrawer = mOverlayDrawer; videoConfig.overlayRotation = mOverlayRotation; } TextureMediaEncoder videoEncoder = new TextureMediaEncoder(videoConfig); @@ -176,26 +167,7 @@ public class SnapshotVideoRecorder extends VideoRecorder implements RendererFram frame.timestamp = surfaceTexture.getTimestamp(); frame.timestampMillis = System.currentTimeMillis(); // NOTE: this is an approximation but it seems to work. surfaceTexture.getTransformMatrix(frame.transform); - - // get overlay - if (mHasOverlay) { - try { - final Canvas surfaceCanvas = mOverlaySurface.lockCanvas(null); - surfaceCanvas.drawColor(Color.TRANSPARENT, PorterDuff.Mode.CLEAR); - mOverlay.drawOn(Overlay.Target.VIDEO_SNAPSHOT, surfaceCanvas); - mOverlaySurface.unlockCanvasAndPost(surfaceCanvas); - } catch (Surface.OutOfResourcesException e) { - LOG.w("Got Surface.OutOfResourcesException while drawing video overlays", e); - } - mIssue514Workaround.beforeOverlayUpdateTexImage(); - mOverlaySurfaceTexture.updateTexImage(); - mOverlaySurfaceTexture.getTransformMatrix(frame.overlayTransform); - } - - if (mEncoderEngine != null) { - // Can happen on teardown. At least it used to. - // NOTE: If this still happens, I would say we can still crash on mOverlaySurface - // calls above. We might have to add some synchronization. + if (mEncoderEngine != null) { // Can happen on teardown. At least it used to. mEncoderEngine.notify(TextureMediaEncoder.FRAME_EVENT, frame); } } @@ -243,17 +215,9 @@ public class SnapshotVideoRecorder extends VideoRecorder implements RendererFram mDesiredState = STATE_NOT_RECORDING; mPreview.removeRendererFrameCallback(SnapshotVideoRecorder.this); mPreview = null; - if (mIssue514Workaround != null) { - mIssue514Workaround.end(); - mIssue514Workaround = null; - } - if (mOverlaySurfaceTexture != null) { - mOverlaySurfaceTexture.release(); - mOverlaySurfaceTexture = null; - } - if (mOverlaySurface != null) { - mOverlaySurface.release(); - mOverlaySurface = null; + if (mOverlayDrawer != null) { + mOverlayDrawer.release(); + mOverlayDrawer = null; } mEncoderEngine = null; dispatchResult(); diff --git a/cameraview/src/main/java/com/otaliastudios/cameraview/video/encoding/TextureConfig.java b/cameraview/src/main/java/com/otaliastudios/cameraview/video/encoding/TextureConfig.java index 247a6e55..67ada078 100644 --- a/cameraview/src/main/java/com/otaliastudios/cameraview/video/encoding/TextureConfig.java +++ b/cameraview/src/main/java/com/otaliastudios/cameraview/video/encoding/TextureConfig.java @@ -5,6 +5,8 @@ import android.opengl.EGLContext; import androidx.annotation.NonNull; import com.otaliastudios.cameraview.internal.Issue514Workaround; +import com.otaliastudios.cameraview.overlay.Overlay; +import com.otaliastudios.cameraview.overlay.OverlayDrawer; /** * Video configuration to be passed as input to the constructor @@ -12,31 +14,29 @@ import com.otaliastudios.cameraview.internal.Issue514Workaround; */ public class TextureConfig extends VideoConfig { - private final static int NO_TEXTURE = Integer.MIN_VALUE; - - public int textureId = NO_TEXTURE; - public int overlayTextureId = NO_TEXTURE; + public int textureId; + public Overlay.Target overlayTarget; + public OverlayDrawer overlayDrawer; public int overlayRotation; public float scaleX; public float scaleY; public EGLContext eglContext; - public Issue514Workaround issue514Workaround; @NonNull TextureConfig copy() { TextureConfig copy = new TextureConfig(); copy(copy); copy.textureId = this.textureId; - copy.overlayTextureId = this.overlayTextureId; + copy.overlayDrawer = this.overlayDrawer; + copy.overlayTarget = this.overlayTarget; copy.overlayRotation = this.overlayRotation; copy.scaleX = this.scaleX; copy.scaleY = this.scaleY; copy.eglContext = this.eglContext; - copy.issue514Workaround = this.issue514Workaround; return copy; } boolean hasOverlay() { - return overlayTextureId != NO_TEXTURE; + return overlayDrawer != null; } } diff --git a/cameraview/src/main/java/com/otaliastudios/cameraview/video/encoding/TextureMediaEncoder.java b/cameraview/src/main/java/com/otaliastudios/cameraview/video/encoding/TextureMediaEncoder.java index 799efad2..f3bf63bc 100644 --- a/cameraview/src/main/java/com/otaliastudios/cameraview/video/encoding/TextureMediaEncoder.java +++ b/cameraview/src/main/java/com/otaliastudios/cameraview/video/encoding/TextureMediaEncoder.java @@ -65,11 +65,6 @@ public class TextureMediaEncoder extends VideoMediaEncoder { * The transformation matrix for the base texture. */ public float[] transform = new float[16]; - - /** - * The transformation matrix for the overlay texture, if any. - */ - public float[] overlayTransform = new float[16]; } /** @@ -131,7 +126,6 @@ public class TextureMediaEncoder extends VideoMediaEncoder { // 1. We must scale this matrix like GlCameraPreview does, because it might have some cropping. // Scaling takes place with respect to the (0, 0, 0) point, so we must apply a Translation to compensate. float[] transform = frame.transform; - float[] overlayTransform = frame.overlayTransform; float scaleX = mConfig.scaleX; float scaleY = mConfig.scaleY; float scaleTranslX = (1F - scaleX) / 2F; @@ -149,14 +143,14 @@ public class TextureMediaEncoder extends VideoMediaEncoder { // 3. Do the same for overlays with their own rotation. if (mConfig.hasOverlay()) { - Matrix.translateM(overlayTransform, 0, 0.5F, 0.5F, 0); - Matrix.rotateM(overlayTransform, 0, mConfig.overlayRotation, 0, 0, 1); - Matrix.translateM(overlayTransform, 0, -0.5F, -0.5F, 0); + mConfig.overlayDrawer.draw(mConfig.overlayTarget); + Matrix.translateM(mConfig.overlayDrawer.getTransform(), 0, 0.5F, 0.5F, 0); + Matrix.rotateM(mConfig.overlayDrawer.getTransform(), 0, mConfig.overlayRotation, 0, 0, 1); + Matrix.translateM(mConfig.overlayDrawer.getTransform(), 0, -0.5F, -0.5F, 0); } mViewport.drawFrame(mConfig.textureId, transform); if (mConfig.hasOverlay()) { - mViewport.drawFrame(mConfig.overlayTextureId, overlayTransform); - mConfig.issue514Workaround.afterOverlayGlDrawn(); + mConfig.overlayDrawer.render(); } mWindow.setPresentationTime(frame.timestamp); mWindow.swapBuffers();