diff --git a/cameraview/src/main/java/com/otaliastudios/cameraview/internal/Issue514Workaround.java b/cameraview/src/main/java/com/otaliastudios/cameraview/internal/Issue514Workaround.java index f59fc3b5..aae0468a 100644 --- a/cameraview/src/main/java/com/otaliastudios/cameraview/internal/Issue514Workaround.java +++ b/cameraview/src/main/java/com/otaliastudios/cameraview/internal/Issue514Workaround.java @@ -6,6 +6,7 @@ import android.graphics.SurfaceTexture; import android.opengl.GLES11Ext; import android.opengl.GLES20; import android.view.Surface; +import com.otaliastudios.cameraview.internal.egl.EglViewport; import com.otaliastudios.cameraview.preview.RendererThread; @@ -13,20 +14,11 @@ import com.otaliastudios.cameraview.preview.RendererThread; * Fixes an issue for some devices with snapshot picture and video recording. * This is so dirty and totally unclear that I wanted to have a separate class. * - * WHAT WE DO - * What we do here is, assuming that we have overlays, is to create a useless - * {@link SurfaceTexture} that receives NO input, because we create no Surface out of it. - * However, it MUST be created using the camera frames texture ID. - * After creation, and before drawing on the overlay Surface, users can call {@link #start()}: - * this will call {@link SurfaceTexture#updateTexImage()}. Since this has NO input, this - * call should be doing NOTHING! However, it fixes the bug #514. Keep reading! - * After the snapshot has been taken (either picture or video), this can be released - * using {@link #end()}. - * * WHY IT FIXES THAT ISSUE - * I have no answer, this is pure magic to me. There is actually no need of this class in some cases: - * - when we don't have overlays - * - on the majority of devices + * I have no answer, this is pure magic to me. + * There is actually no need of this class in some cases: + * - when we don't have overlays, everything works + * - on the majority of devices, everything works * But some devices will show the issue #514 and so they need this class to fix it. * I believe that this has no performance impact but it could be measured and the use of this * class could be restricted to those specific devices that need it. @@ -81,44 +73,54 @@ import com.otaliastudios.cameraview.preview.RendererThread; * and it is implementation specific. * * THE MAGIC - * Impossible to say why, but using this class fixes the described issue. - * - Creating an useless {@link SurfaceTexture} - * - Calling a useless {@link SurfaceTexture#updateTexImage()} before locking the overlay canvas - * - Releasing the useless {@link SurfaceTexture} - * This must be done using the video frame textureId, not the overlayTextureId nor any other id. + * Hard to say why, but using this class fixes the described issue. + * It seems that when the {@link SurfaceTexture#updateTexImage()} method for the overlay surface + * is called - the one that updates the overlayTextureId - we must ensure that the currently + * bound texture is the cameraTextureId. + * + * This makes no sense, since overlaySurfaceTexture.updateTexImage() is setting it to overlayTextureId, + * but it fixes the issue. Specifically, after any draw operation with {@link EglViewport}, the bound + * texture is reset to 0. + * + * So, since updating and drawing can happen on different threads, to maximize the chances that + * when updateTexImage() is called we have bound the cameraTextureId, and to avoid using a lock, + * we require to call + * - {@link #beforeOverlayUpdateTexImage()} right before the {@link SurfaceTexture#updateTexImage()} call + * - {@link #afterOverlayGlDrawn()} right after the last {@link EglViewport#drawFrame(int, float[])} call + * - {@link #end()} to release * + * If filling the texture and rendering happen on two different threads (with a shared EGL context) + * there is still a chance that updateTexImage() is called with a current texture that creates this + * issue (0?), but the alternative would be creating a lock. + * + * REFERENCES * https://android.googlesource.com/platform/frameworks/native/+/5c1139f/libs/gui/SurfaceTexture.cpp */ public class Issue514Workaround { - private SurfaceTexture surfaceTexture = null; private final int cameraTextureId; private final boolean hasOverlay; public Issue514Workaround(int cameraTextureId, boolean hasOverlay) { this.cameraTextureId = cameraTextureId; this.hasOverlay = hasOverlay; - if (this.hasOverlay) { - try { - // surfaceTexture = new SurfaceTexture(cameraTextureId); - } catch (Exception ignore) { } - } } - public void start() { - if (hasOverlay) { - try { - GLES20.glBindTexture(GLES11Ext.GL_TEXTURE_EXTERNAL_OES, cameraTextureId); - // surfaceTexture.updateTexImage(); - } catch (Exception ignore) {} - } + public void beforeOverlayUpdateTexImage() { + bindTexture(cameraTextureId); + } + + public void afterOverlayGlDrawn() { + bindTexture(cameraTextureId); } public void end() { + bindTexture(0); + } + + private void bindTexture(int textureId) { if (hasOverlay) { - try { - // surfaceTexture.release(); - } catch (Exception ignore) {} + GLES20.glBindTexture(GLES11Ext.GL_TEXTURE_EXTERNAL_OES, textureId); } } } diff --git a/cameraview/src/main/java/com/otaliastudios/cameraview/internal/egl/EglViewport.java b/cameraview/src/main/java/com/otaliastudios/cameraview/internal/egl/EglViewport.java index dde679f8..dcb9a991 100644 --- a/cameraview/src/main/java/com/otaliastudios/cameraview/internal/egl/EglViewport.java +++ b/cameraview/src/main/java/com/otaliastudios/cameraview/internal/egl/EglViewport.java @@ -77,14 +77,10 @@ public class EglViewport extends EglElement { // private int muColorAdjustLoc; // Used for filtering public EglViewport() { - this(false); + this(GLES20.GL_TEXTURE0, GLES11Ext.GL_TEXTURE_EXTERNAL_OES); } - public EglViewport(boolean overlay) { - this(GLES20.GL_TEXTURE0, overlay ? GLES20.GL_TEXTURE_2D : GLES11Ext.GL_TEXTURE_EXTERNAL_OES); - } - - public EglViewport(int textureUnit, int textureTarget) { + private EglViewport(int textureUnit, int textureTarget) { mTextureUnit = textureUnit; mTextureTarget = textureTarget; mProgramHandle = createProgram(SIMPLE_VERTEX_SHADER, SIMPLE_FRAGMENT_SHADER); @@ -197,7 +193,7 @@ public class EglViewport extends EglElement { // Done -- disable vertex array, texture, and program. GLES20.glDisableVertexAttribArray(maPositionLocation); GLES20.glDisableVertexAttribArray(maTextureCoordLocation); - // GLES20.glBindTexture(mTextureTarget, 0); + GLES20.glBindTexture(mTextureTarget, 0); GLES20.glUseProgram(0); } } diff --git a/cameraview/src/main/java/com/otaliastudios/cameraview/picture/SnapshotGlPictureRecorder.java b/cameraview/src/main/java/com/otaliastudios/cameraview/picture/SnapshotGlPictureRecorder.java index fc36c05f..bd10e876 100644 --- a/cameraview/src/main/java/com/otaliastudios/cameraview/picture/SnapshotGlPictureRecorder.java +++ b/cameraview/src/main/java/com/otaliastudios/cameraview/picture/SnapshotGlPictureRecorder.java @@ -195,6 +195,7 @@ public class SnapshotGlPictureRecorder extends PictureRecorder { Matrix.translateM(mTransform, 0, -0.5F, -0.5F, 0); // Go back to old position // 4. Do pretty much the same for overlays + issue514Workaround.beforeOverlayUpdateTexImage(); if (mHasOverlay) { // 1. First we must draw on the texture and get latest image try { @@ -205,7 +206,6 @@ public class SnapshotGlPictureRecorder extends PictureRecorder { } catch (Surface.OutOfResourcesException e) { LOG.w("Got Surface.OutOfResourcesException while drawing picture overlays", e); } - issue514Workaround.start(); mOverlaySurfaceTexture.updateTexImage(); mOverlaySurfaceTexture.getTransformMatrix(mOverlayTransform); @@ -221,6 +221,7 @@ public class SnapshotGlPictureRecorder extends PictureRecorder { // 5. Draw and save mViewport.drawFrame(mTextureId, mTransform); if (mHasOverlay) mViewport.drawFrame(mOverlayTextureId, mOverlayTransform); + issue514Workaround.afterOverlayGlDrawn(); mResult.format = PictureResult.FORMAT_JPEG; mResult.data = eglSurface.saveFrameTo(Bitmap.CompressFormat.JPEG); diff --git a/cameraview/src/main/java/com/otaliastudios/cameraview/video/SnapshotVideoRecorder.java b/cameraview/src/main/java/com/otaliastudios/cameraview/video/SnapshotVideoRecorder.java index 9ce821c5..ed93501f 100644 --- a/cameraview/src/main/java/com/otaliastudios/cameraview/video/SnapshotVideoRecorder.java +++ b/cameraview/src/main/java/com/otaliastudios/cameraview/video/SnapshotVideoRecorder.java @@ -140,6 +140,7 @@ public class SnapshotVideoRecorder extends VideoRecorder implements RendererFram videoConfig.textureId = mTextureId; videoConfig.scaleX = scaleX; videoConfig.scaleY = scaleY; + videoConfig.issue514Workaround = mIssue514Workaround; // Get egl context from the RendererThread, which is the one in which we have created // the textureId and the overlayTextureId, managed by the GlSurfaceView. // Next operations can then be performed on different threads using this handle. @@ -186,7 +187,7 @@ public class SnapshotVideoRecorder extends VideoRecorder implements RendererFram } catch (Surface.OutOfResourcesException e) { LOG.w("Got Surface.OutOfResourcesException while drawing video overlays", e); } - mIssue514Workaround.start(); + mIssue514Workaround.beforeOverlayUpdateTexImage(); mOverlaySurfaceTexture.updateTexImage(); mOverlaySurfaceTexture.getTransformMatrix(frame.overlayTransform); } diff --git a/cameraview/src/main/java/com/otaliastudios/cameraview/video/encoding/TextureConfig.java b/cameraview/src/main/java/com/otaliastudios/cameraview/video/encoding/TextureConfig.java index 4f022713..247a6e55 100644 --- a/cameraview/src/main/java/com/otaliastudios/cameraview/video/encoding/TextureConfig.java +++ b/cameraview/src/main/java/com/otaliastudios/cameraview/video/encoding/TextureConfig.java @@ -4,6 +4,8 @@ import android.opengl.EGLContext; import androidx.annotation.NonNull; +import com.otaliastudios.cameraview.internal.Issue514Workaround; + /** * Video configuration to be passed as input to the constructor * of a {@link TextureMediaEncoder}. @@ -18,6 +20,7 @@ public class TextureConfig extends VideoConfig { public float scaleX; public float scaleY; public EGLContext eglContext; + public Issue514Workaround issue514Workaround; @NonNull TextureConfig copy() { @@ -29,6 +32,7 @@ public class TextureConfig extends VideoConfig { copy.scaleX = this.scaleX; copy.scaleY = this.scaleY; copy.eglContext = this.eglContext; + copy.issue514Workaround = this.issue514Workaround; return copy; } diff --git a/cameraview/src/main/java/com/otaliastudios/cameraview/video/encoding/TextureMediaEncoder.java b/cameraview/src/main/java/com/otaliastudios/cameraview/video/encoding/TextureMediaEncoder.java index 4bfc4b9e..799efad2 100644 --- a/cameraview/src/main/java/com/otaliastudios/cameraview/video/encoding/TextureMediaEncoder.java +++ b/cameraview/src/main/java/com/otaliastudios/cameraview/video/encoding/TextureMediaEncoder.java @@ -156,6 +156,7 @@ public class TextureMediaEncoder extends VideoMediaEncoder { mViewport.drawFrame(mConfig.textureId, transform); if (mConfig.hasOverlay()) { mViewport.drawFrame(mConfig.overlayTextureId, overlayTransform); + mConfig.issue514Workaround.afterOverlayGlDrawn(); } mWindow.setPresentationTime(frame.timestamp); mWindow.swapBuffers(); diff --git a/demo/src/main/res/layout/activity_camera.xml b/demo/src/main/res/layout/activity_camera.xml index f36eac91..eb5e9092 100644 --- a/demo/src/main/res/layout/activity_camera.xml +++ b/demo/src/main/res/layout/activity_camera.xml @@ -21,7 +21,7 @@ app:cameraPlaySounds="true" app:cameraGrid="off" app:cameraFlash="off" - app:cameraAudio="on" + app:cameraAudio="off" app:cameraFacing="back" app:cameraGestureTap="autoFocus" app:cameraGestureLongTap="none"