Draw overlays on the encoder thread

pull/528/head
Mattia Iavarone 6 years ago
parent 356b52de36
commit d6eea6e08f
  1. 6
      cameraview/src/main/java/com/otaliastudios/cameraview/internal/Issue514Workaround.java
  2. 79
      cameraview/src/main/java/com/otaliastudios/cameraview/overlay/OverlayDrawer.java
  3. 43
      cameraview/src/main/java/com/otaliastudios/cameraview/picture/SnapshotGlPictureRecorder.java
  4. 54
      cameraview/src/main/java/com/otaliastudios/cameraview/video/SnapshotVideoRecorder.java
  5. 16
      cameraview/src/main/java/com/otaliastudios/cameraview/video/encoding/TextureConfig.java
  6. 16
      cameraview/src/main/java/com/otaliastudios/cameraview/video/encoding/TextureMediaEncoder.java

@ -99,11 +99,9 @@ import com.otaliastudios.cameraview.preview.RendererThread;
public class Issue514Workaround { public class Issue514Workaround {
private final int cameraTextureId; private final int cameraTextureId;
private final boolean hasOverlay;
public Issue514Workaround(int cameraTextureId, boolean hasOverlay) { public Issue514Workaround(int cameraTextureId) {
this.cameraTextureId = cameraTextureId; this.cameraTextureId = cameraTextureId;
this.hasOverlay = hasOverlay;
} }
public void beforeOverlayUpdateTexImage() { public void beforeOverlayUpdateTexImage() {
@ -119,8 +117,6 @@ public class Issue514Workaround {
} }
private void bindTexture(int textureId) { private void bindTexture(int textureId) {
if (hasOverlay) {
GLES20.glBindTexture(GLES11Ext.GL_TEXTURE_EXTERNAL_OES, textureId); GLES20.glBindTexture(GLES11Ext.GL_TEXTURE_EXTERNAL_OES, textureId);
} }
} }
}

@ -0,0 +1,79 @@
package com.otaliastudios.cameraview.overlay;
import android.graphics.Canvas;
import android.graphics.Color;
import android.graphics.PorterDuff;
import android.graphics.SurfaceTexture;
import android.view.Surface;
import androidx.annotation.NonNull;
import com.otaliastudios.cameraview.CameraLogger;
import com.otaliastudios.cameraview.internal.Issue514Workaround;
import com.otaliastudios.cameraview.internal.egl.EglViewport;
import com.otaliastudios.cameraview.size.Size;
public class OverlayDrawer {
private static final String TAG = OverlayDrawer.class.getSimpleName();
private static final CameraLogger LOG = CameraLogger.create(TAG);
private Overlay mOverlay;
private int mTextureId;
private SurfaceTexture mSurfaceTexture;
private Surface mSurface;
private EglViewport mViewport;
private Issue514Workaround mIssue514Workaround;
private float[] mTransform = new float[16];
public OverlayDrawer(@NonNull Overlay overlay, @NonNull Size size, int cameraTextureId) {
mOverlay = overlay;
mViewport = new EglViewport();
mTextureId = mViewport.createTexture();
mSurfaceTexture = new SurfaceTexture(mTextureId);
mSurfaceTexture.setDefaultBufferSize(size.getWidth(), size.getHeight());
mSurface = new Surface(mSurfaceTexture);
mIssue514Workaround = new Issue514Workaround(cameraTextureId);
}
public void draw(@NonNull Overlay.Target target) {
try {
final Canvas surfaceCanvas = mSurface.lockCanvas(null);
surfaceCanvas.drawColor(Color.TRANSPARENT, PorterDuff.Mode.CLEAR);
mOverlay.drawOn(target, surfaceCanvas);
mSurface.unlockCanvasAndPost(surfaceCanvas);
} catch (Surface.OutOfResourcesException e) {
LOG.w("Got Surface.OutOfResourcesException while drawing video overlays", e);
}
mIssue514Workaround.beforeOverlayUpdateTexImage();
mSurfaceTexture.updateTexImage();
mSurfaceTexture.getTransformMatrix(mTransform);
}
public float[] getTransform() {
return mTransform;
}
public void render() {
mIssue514Workaround.afterOverlayGlDrawn();
mViewport.drawFrame(mTextureId, mTransform);
mIssue514Workaround.afterOverlayGlDrawn();
}
public void release() {
mIssue514Workaround.end();
if (mSurfaceTexture != null) {
mSurfaceTexture.release();
mSurfaceTexture = null;
}
if (mSurface != null) {
mSurface.release();
mSurface = null;
}
if (mViewport != null) {
mViewport.release();
mViewport = null;
}
}
}

@ -26,6 +26,7 @@ import com.otaliastudios.cameraview.internal.egl.EglViewport;
import com.otaliastudios.cameraview.internal.egl.EglWindowSurface; import com.otaliastudios.cameraview.internal.egl.EglWindowSurface;
import com.otaliastudios.cameraview.internal.utils.CropHelper; import com.otaliastudios.cameraview.internal.utils.CropHelper;
import com.otaliastudios.cameraview.internal.utils.WorkerHandler; import com.otaliastudios.cameraview.internal.utils.WorkerHandler;
import com.otaliastudios.cameraview.overlay.OverlayDrawer;
import com.otaliastudios.cameraview.preview.GlCameraPreview; import com.otaliastudios.cameraview.preview.GlCameraPreview;
import com.otaliastudios.cameraview.preview.RendererFrameCallback; import com.otaliastudios.cameraview.preview.RendererFrameCallback;
import com.otaliastudios.cameraview.preview.RendererThread; import com.otaliastudios.cameraview.preview.RendererThread;
@ -65,14 +66,11 @@ public class SnapshotGlPictureRecorder extends PictureRecorder {
private Overlay mOverlay; private Overlay mOverlay;
private boolean mHasOverlay; private boolean mHasOverlay;
private OverlayDrawer mOverlayDrawer;
private int mTextureId; private int mTextureId;
private float[] mTransform; private float[] mTransform;
private int mOverlayTextureId = 0;
private SurfaceTexture mOverlaySurfaceTexture;
private Surface mOverlaySurface;
private float[] mOverlayTransform;
private EglViewport mViewport; private EglViewport mViewport;
@ -121,11 +119,7 @@ public class SnapshotGlPictureRecorder extends PictureRecorder {
Matrix.setIdentityM(mTransform, 0); Matrix.setIdentityM(mTransform, 0);
if (mHasOverlay) { if (mHasOverlay) {
mOverlayTextureId = mViewport.createTexture(); mOverlayDrawer = new OverlayDrawer(mOverlay, mResult.size, textureId);
mOverlaySurfaceTexture = new SurfaceTexture(mOverlayTextureId, true);
mOverlaySurfaceTexture.setDefaultBufferSize(mResult.size.getWidth(), mResult.size.getHeight());
mOverlaySurface = new Surface(mOverlaySurfaceTexture);
mOverlayTransform = new float[16];
} }
} }
@ -169,7 +163,6 @@ public class SnapshotGlPictureRecorder extends PictureRecorder {
final int fakeOutputTextureId = 9999; final int fakeOutputTextureId = 9999;
SurfaceTexture fakeOutputSurface = new SurfaceTexture(fakeOutputTextureId); SurfaceTexture fakeOutputSurface = new SurfaceTexture(fakeOutputTextureId);
fakeOutputSurface.setDefaultBufferSize(mResult.size.getWidth(), mResult.size.getHeight()); fakeOutputSurface.setDefaultBufferSize(mResult.size.getWidth(), mResult.size.getHeight());
final Issue514Workaround issue514Workaround = new Issue514Workaround(mTextureId, mHasOverlay);
// 1. Create an EGL surface // 1. Create an EGL surface
final EglCore core = new EglCore(eglContext, EglCore.FLAG_RECORDABLE); final EglCore core = new EglCore(eglContext, EglCore.FLAG_RECORDABLE);
@ -195,46 +188,30 @@ public class SnapshotGlPictureRecorder extends PictureRecorder {
Matrix.translateM(mTransform, 0, -0.5F, -0.5F, 0); // Go back to old position Matrix.translateM(mTransform, 0, -0.5F, -0.5F, 0); // Go back to old position
// 4. Do pretty much the same for overlays // 4. Do pretty much the same for overlays
issue514Workaround.beforeOverlayUpdateTexImage();
if (mHasOverlay) { if (mHasOverlay) {
// 1. First we must draw on the texture and get latest image // 1. First we must draw on the texture and get latest image
try { mOverlayDrawer.draw(Overlay.Target.PICTURE_SNAPSHOT);
final Canvas surfaceCanvas = mOverlaySurface.lockCanvas(null);
surfaceCanvas.drawColor(Color.TRANSPARENT, PorterDuff.Mode.CLEAR);
mOverlay.drawOn(Overlay.Target.PICTURE_SNAPSHOT, surfaceCanvas);
mOverlaySurface.unlockCanvasAndPost(surfaceCanvas);
} catch (Surface.OutOfResourcesException e) {
LOG.w("Got Surface.OutOfResourcesException while drawing picture overlays", e);
}
mOverlaySurfaceTexture.updateTexImage();
mOverlaySurfaceTexture.getTransformMatrix(mOverlayTransform);
// 2. Then we can apply the transformations // 2. Then we can apply the transformations
int rotation = mEngine.getAngles().offset(Reference.VIEW, Reference.OUTPUT, Axis.ABSOLUTE); int rotation = mEngine.getAngles().offset(Reference.VIEW, Reference.OUTPUT, Axis.ABSOLUTE);
Matrix.translateM(mOverlayTransform, 0, 0.5F, 0.5F, 0); Matrix.translateM(mOverlayDrawer.getTransform(), 0, 0.5F, 0.5F, 0);
Matrix.rotateM(mOverlayTransform, 0, rotation, 0, 0, 1); Matrix.rotateM(mOverlayDrawer.getTransform(), 0, rotation, 0, 0, 1);
// No need to flip the x axis for front camera, but need to flip the y axis always. // No need to flip the x axis for front camera, but need to flip the y axis always.
Matrix.scaleM(mOverlayTransform, 0, 1, -1, 1); Matrix.scaleM(mOverlayDrawer.getTransform(), 0, 1, -1, 1);
Matrix.translateM(mOverlayTransform, 0, -0.5F, -0.5F, 0); Matrix.translateM(mOverlayDrawer.getTransform(), 0, -0.5F, -0.5F, 0);
} }
// 5. Draw and save // 5. Draw and save
mViewport.drawFrame(mTextureId, mTransform); mViewport.drawFrame(mTextureId, mTransform);
if (mHasOverlay) mViewport.drawFrame(mOverlayTextureId, mOverlayTransform); if (mHasOverlay) mOverlayDrawer.render();
issue514Workaround.afterOverlayGlDrawn();
mResult.format = PictureResult.FORMAT_JPEG; mResult.format = PictureResult.FORMAT_JPEG;
mResult.data = eglSurface.saveFrameTo(Bitmap.CompressFormat.JPEG); mResult.data = eglSurface.saveFrameTo(Bitmap.CompressFormat.JPEG);
// 6. Cleanup // 6. Cleanup
issue514Workaround.end();
eglSurface.releaseEglSurface(); eglSurface.releaseEglSurface();
mViewport.release(); mViewport.release();
fakeOutputSurface.release(); fakeOutputSurface.release();
if (mHasOverlay) { if (mHasOverlay) mOverlayDrawer.release();
mOverlaySurfaceTexture.releaseTexImage();
mOverlaySurface.release();
mOverlaySurfaceTexture.release();
}
core.release(); core.release();
dispatchResult(); dispatchResult();
} }

@ -15,6 +15,7 @@ import com.otaliastudios.cameraview.VideoResult;
import com.otaliastudios.cameraview.controls.Audio; import com.otaliastudios.cameraview.controls.Audio;
import com.otaliastudios.cameraview.engine.CameraEngine; import com.otaliastudios.cameraview.engine.CameraEngine;
import com.otaliastudios.cameraview.internal.egl.EglViewport; import com.otaliastudios.cameraview.internal.egl.EglViewport;
import com.otaliastudios.cameraview.overlay.OverlayDrawer;
import com.otaliastudios.cameraview.preview.GlCameraPreview; import com.otaliastudios.cameraview.preview.GlCameraPreview;
import com.otaliastudios.cameraview.preview.RendererFrameCallback; import com.otaliastudios.cameraview.preview.RendererFrameCallback;
import com.otaliastudios.cameraview.preview.RendererThread; import com.otaliastudios.cameraview.preview.RendererThread;
@ -60,15 +61,11 @@ public class SnapshotVideoRecorder extends VideoRecorder implements RendererFram
private int mDesiredState = STATE_NOT_RECORDING; private int mDesiredState = STATE_NOT_RECORDING;
private int mTextureId = 0; private int mTextureId = 0;
private int mOverlayTextureId = 0;
private SurfaceTexture mOverlaySurfaceTexture;
private Surface mOverlaySurface;
private Overlay mOverlay; private Overlay mOverlay;
private OverlayDrawer mOverlayDrawer;
private boolean mHasOverlay; private boolean mHasOverlay;
private int mOverlayRotation; private int mOverlayRotation;
private Issue514Workaround mIssue514Workaround;
public SnapshotVideoRecorder(@NonNull CameraEngine engine, public SnapshotVideoRecorder(@NonNull CameraEngine engine,
@NonNull GlCameraPreview preview, @NonNull GlCameraPreview preview,
@Nullable Overlay overlay, @Nullable Overlay overlay,
@ -96,14 +93,8 @@ public class SnapshotVideoRecorder extends VideoRecorder implements RendererFram
public void onRendererTextureCreated(int textureId) { public void onRendererTextureCreated(int textureId) {
mTextureId = textureId; mTextureId = textureId;
if (mHasOverlay) { if (mHasOverlay) {
EglViewport temp = new EglViewport(); mOverlayDrawer = new OverlayDrawer(mOverlay, mResult.size, textureId);
mOverlayTextureId = temp.createTexture();
mOverlaySurfaceTexture = new SurfaceTexture(mOverlayTextureId);
mOverlaySurfaceTexture.setDefaultBufferSize(mResult.size.getWidth(), mResult.size.getHeight());
mOverlaySurface = new Surface(mOverlaySurfaceTexture);
temp.release(true);
} }
mIssue514Workaround = new Issue514Workaround(textureId, mHasOverlay);
} }
@RendererThread @RendererThread
@ -140,13 +131,13 @@ public class SnapshotVideoRecorder extends VideoRecorder implements RendererFram
videoConfig.textureId = mTextureId; videoConfig.textureId = mTextureId;
videoConfig.scaleX = scaleX; videoConfig.scaleX = scaleX;
videoConfig.scaleY = scaleY; videoConfig.scaleY = scaleY;
videoConfig.issue514Workaround = mIssue514Workaround;
// Get egl context from the RendererThread, which is the one in which we have created // Get egl context from the RendererThread, which is the one in which we have created
// the textureId and the overlayTextureId, managed by the GlSurfaceView. // the textureId and the overlayTextureId, managed by the GlSurfaceView.
// Next operations can then be performed on different threads using this handle. // Next operations can then be performed on different threads using this handle.
videoConfig.eglContext = EGL14.eglGetCurrentContext(); videoConfig.eglContext = EGL14.eglGetCurrentContext();
if (mHasOverlay) { if (mHasOverlay) {
videoConfig.overlayTextureId = mOverlayTextureId; videoConfig.overlayTarget = Overlay.Target.VIDEO_SNAPSHOT;
videoConfig.overlayDrawer = mOverlayDrawer;
videoConfig.overlayRotation = mOverlayRotation; videoConfig.overlayRotation = mOverlayRotation;
} }
TextureMediaEncoder videoEncoder = new TextureMediaEncoder(videoConfig); TextureMediaEncoder videoEncoder = new TextureMediaEncoder(videoConfig);
@ -176,26 +167,7 @@ public class SnapshotVideoRecorder extends VideoRecorder implements RendererFram
frame.timestamp = surfaceTexture.getTimestamp(); frame.timestamp = surfaceTexture.getTimestamp();
frame.timestampMillis = System.currentTimeMillis(); // NOTE: this is an approximation but it seems to work. frame.timestampMillis = System.currentTimeMillis(); // NOTE: this is an approximation but it seems to work.
surfaceTexture.getTransformMatrix(frame.transform); surfaceTexture.getTransformMatrix(frame.transform);
if (mEncoderEngine != null) { // Can happen on teardown. At least it used to.
// get overlay
if (mHasOverlay) {
try {
final Canvas surfaceCanvas = mOverlaySurface.lockCanvas(null);
surfaceCanvas.drawColor(Color.TRANSPARENT, PorterDuff.Mode.CLEAR);
mOverlay.drawOn(Overlay.Target.VIDEO_SNAPSHOT, surfaceCanvas);
mOverlaySurface.unlockCanvasAndPost(surfaceCanvas);
} catch (Surface.OutOfResourcesException e) {
LOG.w("Got Surface.OutOfResourcesException while drawing video overlays", e);
}
mIssue514Workaround.beforeOverlayUpdateTexImage();
mOverlaySurfaceTexture.updateTexImage();
mOverlaySurfaceTexture.getTransformMatrix(frame.overlayTransform);
}
if (mEncoderEngine != null) {
// Can happen on teardown. At least it used to.
// NOTE: If this still happens, I would say we can still crash on mOverlaySurface
// calls above. We might have to add some synchronization.
mEncoderEngine.notify(TextureMediaEncoder.FRAME_EVENT, frame); mEncoderEngine.notify(TextureMediaEncoder.FRAME_EVENT, frame);
} }
} }
@ -243,17 +215,9 @@ public class SnapshotVideoRecorder extends VideoRecorder implements RendererFram
mDesiredState = STATE_NOT_RECORDING; mDesiredState = STATE_NOT_RECORDING;
mPreview.removeRendererFrameCallback(SnapshotVideoRecorder.this); mPreview.removeRendererFrameCallback(SnapshotVideoRecorder.this);
mPreview = null; mPreview = null;
if (mIssue514Workaround != null) { if (mOverlayDrawer != null) {
mIssue514Workaround.end(); mOverlayDrawer.release();
mIssue514Workaround = null; mOverlayDrawer = null;
}
if (mOverlaySurfaceTexture != null) {
mOverlaySurfaceTexture.release();
mOverlaySurfaceTexture = null;
}
if (mOverlaySurface != null) {
mOverlaySurface.release();
mOverlaySurface = null;
} }
mEncoderEngine = null; mEncoderEngine = null;
dispatchResult(); dispatchResult();

@ -5,6 +5,8 @@ import android.opengl.EGLContext;
import androidx.annotation.NonNull; import androidx.annotation.NonNull;
import com.otaliastudios.cameraview.internal.Issue514Workaround; import com.otaliastudios.cameraview.internal.Issue514Workaround;
import com.otaliastudios.cameraview.overlay.Overlay;
import com.otaliastudios.cameraview.overlay.OverlayDrawer;
/** /**
* Video configuration to be passed as input to the constructor * Video configuration to be passed as input to the constructor
@ -12,31 +14,29 @@ import com.otaliastudios.cameraview.internal.Issue514Workaround;
*/ */
public class TextureConfig extends VideoConfig { public class TextureConfig extends VideoConfig {
private final static int NO_TEXTURE = Integer.MIN_VALUE; public int textureId;
public Overlay.Target overlayTarget;
public int textureId = NO_TEXTURE; public OverlayDrawer overlayDrawer;
public int overlayTextureId = NO_TEXTURE;
public int overlayRotation; public int overlayRotation;
public float scaleX; public float scaleX;
public float scaleY; public float scaleY;
public EGLContext eglContext; public EGLContext eglContext;
public Issue514Workaround issue514Workaround;
@NonNull @NonNull
TextureConfig copy() { TextureConfig copy() {
TextureConfig copy = new TextureConfig(); TextureConfig copy = new TextureConfig();
copy(copy); copy(copy);
copy.textureId = this.textureId; copy.textureId = this.textureId;
copy.overlayTextureId = this.overlayTextureId; copy.overlayDrawer = this.overlayDrawer;
copy.overlayTarget = this.overlayTarget;
copy.overlayRotation = this.overlayRotation; copy.overlayRotation = this.overlayRotation;
copy.scaleX = this.scaleX; copy.scaleX = this.scaleX;
copy.scaleY = this.scaleY; copy.scaleY = this.scaleY;
copy.eglContext = this.eglContext; copy.eglContext = this.eglContext;
copy.issue514Workaround = this.issue514Workaround;
return copy; return copy;
} }
boolean hasOverlay() { boolean hasOverlay() {
return overlayTextureId != NO_TEXTURE; return overlayDrawer != null;
} }
} }

@ -65,11 +65,6 @@ public class TextureMediaEncoder extends VideoMediaEncoder<TextureConfig> {
* The transformation matrix for the base texture. * The transformation matrix for the base texture.
*/ */
public float[] transform = new float[16]; public float[] transform = new float[16];
/**
* The transformation matrix for the overlay texture, if any.
*/
public float[] overlayTransform = new float[16];
} }
/** /**
@ -131,7 +126,6 @@ public class TextureMediaEncoder extends VideoMediaEncoder<TextureConfig> {
// 1. We must scale this matrix like GlCameraPreview does, because it might have some cropping. // 1. We must scale this matrix like GlCameraPreview does, because it might have some cropping.
// Scaling takes place with respect to the (0, 0, 0) point, so we must apply a Translation to compensate. // Scaling takes place with respect to the (0, 0, 0) point, so we must apply a Translation to compensate.
float[] transform = frame.transform; float[] transform = frame.transform;
float[] overlayTransform = frame.overlayTransform;
float scaleX = mConfig.scaleX; float scaleX = mConfig.scaleX;
float scaleY = mConfig.scaleY; float scaleY = mConfig.scaleY;
float scaleTranslX = (1F - scaleX) / 2F; float scaleTranslX = (1F - scaleX) / 2F;
@ -149,14 +143,14 @@ public class TextureMediaEncoder extends VideoMediaEncoder<TextureConfig> {
// 3. Do the same for overlays with their own rotation. // 3. Do the same for overlays with their own rotation.
if (mConfig.hasOverlay()) { if (mConfig.hasOverlay()) {
Matrix.translateM(overlayTransform, 0, 0.5F, 0.5F, 0); mConfig.overlayDrawer.draw(mConfig.overlayTarget);
Matrix.rotateM(overlayTransform, 0, mConfig.overlayRotation, 0, 0, 1); Matrix.translateM(mConfig.overlayDrawer.getTransform(), 0, 0.5F, 0.5F, 0);
Matrix.translateM(overlayTransform, 0, -0.5F, -0.5F, 0); Matrix.rotateM(mConfig.overlayDrawer.getTransform(), 0, mConfig.overlayRotation, 0, 0, 1);
Matrix.translateM(mConfig.overlayDrawer.getTransform(), 0, -0.5F, -0.5F, 0);
} }
mViewport.drawFrame(mConfig.textureId, transform); mViewport.drawFrame(mConfig.textureId, transform);
if (mConfig.hasOverlay()) { if (mConfig.hasOverlay()) {
mViewport.drawFrame(mConfig.overlayTextureId, overlayTransform); mConfig.overlayDrawer.render();
mConfig.issue514Workaround.afterOverlayGlDrawn();
} }
mWindow.setPresentationTime(frame.timestamp); mWindow.setPresentationTime(frame.timestamp);
mWindow.swapBuffers(); mWindow.swapBuffers();

Loading…
Cancel
Save