Draw overlays on the encoder thread

pull/528/head
Mattia Iavarone 6 years ago
parent 356b52de36
commit d6eea6e08f
  1. 8
      cameraview/src/main/java/com/otaliastudios/cameraview/internal/Issue514Workaround.java
  2. 79
      cameraview/src/main/java/com/otaliastudios/cameraview/overlay/OverlayDrawer.java
  3. 43
      cameraview/src/main/java/com/otaliastudios/cameraview/picture/SnapshotGlPictureRecorder.java
  4. 54
      cameraview/src/main/java/com/otaliastudios/cameraview/video/SnapshotVideoRecorder.java
  5. 16
      cameraview/src/main/java/com/otaliastudios/cameraview/video/encoding/TextureConfig.java
  6. 16
      cameraview/src/main/java/com/otaliastudios/cameraview/video/encoding/TextureMediaEncoder.java

@ -99,11 +99,9 @@ import com.otaliastudios.cameraview.preview.RendererThread;
public class Issue514Workaround {
private final int cameraTextureId;
private final boolean hasOverlay;
public Issue514Workaround(int cameraTextureId, boolean hasOverlay) {
public Issue514Workaround(int cameraTextureId) {
this.cameraTextureId = cameraTextureId;
this.hasOverlay = hasOverlay;
}
public void beforeOverlayUpdateTexImage() {
@ -119,8 +117,6 @@ public class Issue514Workaround {
}
private void bindTexture(int textureId) {
if (hasOverlay) {
GLES20.glBindTexture(GLES11Ext.GL_TEXTURE_EXTERNAL_OES, textureId);
}
GLES20.glBindTexture(GLES11Ext.GL_TEXTURE_EXTERNAL_OES, textureId);
}
}

@ -0,0 +1,79 @@
package com.otaliastudios.cameraview.overlay;
import android.graphics.Canvas;
import android.graphics.Color;
import android.graphics.PorterDuff;
import android.graphics.SurfaceTexture;
import android.view.Surface;
import androidx.annotation.NonNull;
import com.otaliastudios.cameraview.CameraLogger;
import com.otaliastudios.cameraview.internal.Issue514Workaround;
import com.otaliastudios.cameraview.internal.egl.EglViewport;
import com.otaliastudios.cameraview.size.Size;
public class OverlayDrawer {
private static final String TAG = OverlayDrawer.class.getSimpleName();
private static final CameraLogger LOG = CameraLogger.create(TAG);
private Overlay mOverlay;
private int mTextureId;
private SurfaceTexture mSurfaceTexture;
private Surface mSurface;
private EglViewport mViewport;
private Issue514Workaround mIssue514Workaround;
private float[] mTransform = new float[16];
public OverlayDrawer(@NonNull Overlay overlay, @NonNull Size size, int cameraTextureId) {
mOverlay = overlay;
mViewport = new EglViewport();
mTextureId = mViewport.createTexture();
mSurfaceTexture = new SurfaceTexture(mTextureId);
mSurfaceTexture.setDefaultBufferSize(size.getWidth(), size.getHeight());
mSurface = new Surface(mSurfaceTexture);
mIssue514Workaround = new Issue514Workaround(cameraTextureId);
}
public void draw(@NonNull Overlay.Target target) {
try {
final Canvas surfaceCanvas = mSurface.lockCanvas(null);
surfaceCanvas.drawColor(Color.TRANSPARENT, PorterDuff.Mode.CLEAR);
mOverlay.drawOn(target, surfaceCanvas);
mSurface.unlockCanvasAndPost(surfaceCanvas);
} catch (Surface.OutOfResourcesException e) {
LOG.w("Got Surface.OutOfResourcesException while drawing video overlays", e);
}
mIssue514Workaround.beforeOverlayUpdateTexImage();
mSurfaceTexture.updateTexImage();
mSurfaceTexture.getTransformMatrix(mTransform);
}
public float[] getTransform() {
return mTransform;
}
public void render() {
mIssue514Workaround.afterOverlayGlDrawn();
mViewport.drawFrame(mTextureId, mTransform);
mIssue514Workaround.afterOverlayGlDrawn();
}
public void release() {
mIssue514Workaround.end();
if (mSurfaceTexture != null) {
mSurfaceTexture.release();
mSurfaceTexture = null;
}
if (mSurface != null) {
mSurface.release();
mSurface = null;
}
if (mViewport != null) {
mViewport.release();
mViewport = null;
}
}
}

@ -26,6 +26,7 @@ import com.otaliastudios.cameraview.internal.egl.EglViewport;
import com.otaliastudios.cameraview.internal.egl.EglWindowSurface;
import com.otaliastudios.cameraview.internal.utils.CropHelper;
import com.otaliastudios.cameraview.internal.utils.WorkerHandler;
import com.otaliastudios.cameraview.overlay.OverlayDrawer;
import com.otaliastudios.cameraview.preview.GlCameraPreview;
import com.otaliastudios.cameraview.preview.RendererFrameCallback;
import com.otaliastudios.cameraview.preview.RendererThread;
@ -65,14 +66,11 @@ public class SnapshotGlPictureRecorder extends PictureRecorder {
private Overlay mOverlay;
private boolean mHasOverlay;
private OverlayDrawer mOverlayDrawer;
private int mTextureId;
private float[] mTransform;
private int mOverlayTextureId = 0;
private SurfaceTexture mOverlaySurfaceTexture;
private Surface mOverlaySurface;
private float[] mOverlayTransform;
private EglViewport mViewport;
@ -121,11 +119,7 @@ public class SnapshotGlPictureRecorder extends PictureRecorder {
Matrix.setIdentityM(mTransform, 0);
if (mHasOverlay) {
mOverlayTextureId = mViewport.createTexture();
mOverlaySurfaceTexture = new SurfaceTexture(mOverlayTextureId, true);
mOverlaySurfaceTexture.setDefaultBufferSize(mResult.size.getWidth(), mResult.size.getHeight());
mOverlaySurface = new Surface(mOverlaySurfaceTexture);
mOverlayTransform = new float[16];
mOverlayDrawer = new OverlayDrawer(mOverlay, mResult.size, textureId);
}
}
@ -169,7 +163,6 @@ public class SnapshotGlPictureRecorder extends PictureRecorder {
final int fakeOutputTextureId = 9999;
SurfaceTexture fakeOutputSurface = new SurfaceTexture(fakeOutputTextureId);
fakeOutputSurface.setDefaultBufferSize(mResult.size.getWidth(), mResult.size.getHeight());
final Issue514Workaround issue514Workaround = new Issue514Workaround(mTextureId, mHasOverlay);
// 1. Create an EGL surface
final EglCore core = new EglCore(eglContext, EglCore.FLAG_RECORDABLE);
@ -195,46 +188,30 @@ public class SnapshotGlPictureRecorder extends PictureRecorder {
Matrix.translateM(mTransform, 0, -0.5F, -0.5F, 0); // Go back to old position
// 4. Do pretty much the same for overlays
issue514Workaround.beforeOverlayUpdateTexImage();
if (mHasOverlay) {
// 1. First we must draw on the texture and get latest image
try {
final Canvas surfaceCanvas = mOverlaySurface.lockCanvas(null);
surfaceCanvas.drawColor(Color.TRANSPARENT, PorterDuff.Mode.CLEAR);
mOverlay.drawOn(Overlay.Target.PICTURE_SNAPSHOT, surfaceCanvas);
mOverlaySurface.unlockCanvasAndPost(surfaceCanvas);
} catch (Surface.OutOfResourcesException e) {
LOG.w("Got Surface.OutOfResourcesException while drawing picture overlays", e);
}
mOverlaySurfaceTexture.updateTexImage();
mOverlaySurfaceTexture.getTransformMatrix(mOverlayTransform);
mOverlayDrawer.draw(Overlay.Target.PICTURE_SNAPSHOT);
// 2. Then we can apply the transformations
int rotation = mEngine.getAngles().offset(Reference.VIEW, Reference.OUTPUT, Axis.ABSOLUTE);
Matrix.translateM(mOverlayTransform, 0, 0.5F, 0.5F, 0);
Matrix.rotateM(mOverlayTransform, 0, rotation, 0, 0, 1);
Matrix.translateM(mOverlayDrawer.getTransform(), 0, 0.5F, 0.5F, 0);
Matrix.rotateM(mOverlayDrawer.getTransform(), 0, rotation, 0, 0, 1);
// No need to flip the x axis for front camera, but need to flip the y axis always.
Matrix.scaleM(mOverlayTransform, 0, 1, -1, 1);
Matrix.translateM(mOverlayTransform, 0, -0.5F, -0.5F, 0);
Matrix.scaleM(mOverlayDrawer.getTransform(), 0, 1, -1, 1);
Matrix.translateM(mOverlayDrawer.getTransform(), 0, -0.5F, -0.5F, 0);
}
// 5. Draw and save
mViewport.drawFrame(mTextureId, mTransform);
if (mHasOverlay) mViewport.drawFrame(mOverlayTextureId, mOverlayTransform);
issue514Workaround.afterOverlayGlDrawn();
if (mHasOverlay) mOverlayDrawer.render();
mResult.format = PictureResult.FORMAT_JPEG;
mResult.data = eglSurface.saveFrameTo(Bitmap.CompressFormat.JPEG);
// 6. Cleanup
issue514Workaround.end();
eglSurface.releaseEglSurface();
mViewport.release();
fakeOutputSurface.release();
if (mHasOverlay) {
mOverlaySurfaceTexture.releaseTexImage();
mOverlaySurface.release();
mOverlaySurfaceTexture.release();
}
if (mHasOverlay) mOverlayDrawer.release();
core.release();
dispatchResult();
}

@ -15,6 +15,7 @@ import com.otaliastudios.cameraview.VideoResult;
import com.otaliastudios.cameraview.controls.Audio;
import com.otaliastudios.cameraview.engine.CameraEngine;
import com.otaliastudios.cameraview.internal.egl.EglViewport;
import com.otaliastudios.cameraview.overlay.OverlayDrawer;
import com.otaliastudios.cameraview.preview.GlCameraPreview;
import com.otaliastudios.cameraview.preview.RendererFrameCallback;
import com.otaliastudios.cameraview.preview.RendererThread;
@ -60,15 +61,11 @@ public class SnapshotVideoRecorder extends VideoRecorder implements RendererFram
private int mDesiredState = STATE_NOT_RECORDING;
private int mTextureId = 0;
private int mOverlayTextureId = 0;
private SurfaceTexture mOverlaySurfaceTexture;
private Surface mOverlaySurface;
private Overlay mOverlay;
private OverlayDrawer mOverlayDrawer;
private boolean mHasOverlay;
private int mOverlayRotation;
private Issue514Workaround mIssue514Workaround;
public SnapshotVideoRecorder(@NonNull CameraEngine engine,
@NonNull GlCameraPreview preview,
@Nullable Overlay overlay,
@ -96,14 +93,8 @@ public class SnapshotVideoRecorder extends VideoRecorder implements RendererFram
public void onRendererTextureCreated(int textureId) {
mTextureId = textureId;
if (mHasOverlay) {
EglViewport temp = new EglViewport();
mOverlayTextureId = temp.createTexture();
mOverlaySurfaceTexture = new SurfaceTexture(mOverlayTextureId);
mOverlaySurfaceTexture.setDefaultBufferSize(mResult.size.getWidth(), mResult.size.getHeight());
mOverlaySurface = new Surface(mOverlaySurfaceTexture);
temp.release(true);
mOverlayDrawer = new OverlayDrawer(mOverlay, mResult.size, textureId);
}
mIssue514Workaround = new Issue514Workaround(textureId, mHasOverlay);
}
@RendererThread
@ -140,13 +131,13 @@ public class SnapshotVideoRecorder extends VideoRecorder implements RendererFram
videoConfig.textureId = mTextureId;
videoConfig.scaleX = scaleX;
videoConfig.scaleY = scaleY;
videoConfig.issue514Workaround = mIssue514Workaround;
// Get egl context from the RendererThread, which is the one in which we have created
// the textureId and the overlayTextureId, managed by the GlSurfaceView.
// Next operations can then be performed on different threads using this handle.
videoConfig.eglContext = EGL14.eglGetCurrentContext();
if (mHasOverlay) {
videoConfig.overlayTextureId = mOverlayTextureId;
videoConfig.overlayTarget = Overlay.Target.VIDEO_SNAPSHOT;
videoConfig.overlayDrawer = mOverlayDrawer;
videoConfig.overlayRotation = mOverlayRotation;
}
TextureMediaEncoder videoEncoder = new TextureMediaEncoder(videoConfig);
@ -176,26 +167,7 @@ public class SnapshotVideoRecorder extends VideoRecorder implements RendererFram
frame.timestamp = surfaceTexture.getTimestamp();
frame.timestampMillis = System.currentTimeMillis(); // NOTE: this is an approximation but it seems to work.
surfaceTexture.getTransformMatrix(frame.transform);
// get overlay
if (mHasOverlay) {
try {
final Canvas surfaceCanvas = mOverlaySurface.lockCanvas(null);
surfaceCanvas.drawColor(Color.TRANSPARENT, PorterDuff.Mode.CLEAR);
mOverlay.drawOn(Overlay.Target.VIDEO_SNAPSHOT, surfaceCanvas);
mOverlaySurface.unlockCanvasAndPost(surfaceCanvas);
} catch (Surface.OutOfResourcesException e) {
LOG.w("Got Surface.OutOfResourcesException while drawing video overlays", e);
}
mIssue514Workaround.beforeOverlayUpdateTexImage();
mOverlaySurfaceTexture.updateTexImage();
mOverlaySurfaceTexture.getTransformMatrix(frame.overlayTransform);
}
if (mEncoderEngine != null) {
// Can happen on teardown. At least it used to.
// NOTE: If this still happens, I would say we can still crash on mOverlaySurface
// calls above. We might have to add some synchronization.
if (mEncoderEngine != null) { // Can happen on teardown. At least it used to.
mEncoderEngine.notify(TextureMediaEncoder.FRAME_EVENT, frame);
}
}
@ -243,17 +215,9 @@ public class SnapshotVideoRecorder extends VideoRecorder implements RendererFram
mDesiredState = STATE_NOT_RECORDING;
mPreview.removeRendererFrameCallback(SnapshotVideoRecorder.this);
mPreview = null;
if (mIssue514Workaround != null) {
mIssue514Workaround.end();
mIssue514Workaround = null;
}
if (mOverlaySurfaceTexture != null) {
mOverlaySurfaceTexture.release();
mOverlaySurfaceTexture = null;
}
if (mOverlaySurface != null) {
mOverlaySurface.release();
mOverlaySurface = null;
if (mOverlayDrawer != null) {
mOverlayDrawer.release();
mOverlayDrawer = null;
}
mEncoderEngine = null;
dispatchResult();

@ -5,6 +5,8 @@ import android.opengl.EGLContext;
import androidx.annotation.NonNull;
import com.otaliastudios.cameraview.internal.Issue514Workaround;
import com.otaliastudios.cameraview.overlay.Overlay;
import com.otaliastudios.cameraview.overlay.OverlayDrawer;
/**
* Video configuration to be passed as input to the constructor
@ -12,31 +14,29 @@ import com.otaliastudios.cameraview.internal.Issue514Workaround;
*/
public class TextureConfig extends VideoConfig {
private final static int NO_TEXTURE = Integer.MIN_VALUE;
public int textureId = NO_TEXTURE;
public int overlayTextureId = NO_TEXTURE;
public int textureId;
public Overlay.Target overlayTarget;
public OverlayDrawer overlayDrawer;
public int overlayRotation;
public float scaleX;
public float scaleY;
public EGLContext eglContext;
public Issue514Workaround issue514Workaround;
@NonNull
TextureConfig copy() {
TextureConfig copy = new TextureConfig();
copy(copy);
copy.textureId = this.textureId;
copy.overlayTextureId = this.overlayTextureId;
copy.overlayDrawer = this.overlayDrawer;
copy.overlayTarget = this.overlayTarget;
copy.overlayRotation = this.overlayRotation;
copy.scaleX = this.scaleX;
copy.scaleY = this.scaleY;
copy.eglContext = this.eglContext;
copy.issue514Workaround = this.issue514Workaround;
return copy;
}
boolean hasOverlay() {
return overlayTextureId != NO_TEXTURE;
return overlayDrawer != null;
}
}

@ -65,11 +65,6 @@ public class TextureMediaEncoder extends VideoMediaEncoder<TextureConfig> {
* The transformation matrix for the base texture.
*/
public float[] transform = new float[16];
/**
* The transformation matrix for the overlay texture, if any.
*/
public float[] overlayTransform = new float[16];
}
/**
@ -131,7 +126,6 @@ public class TextureMediaEncoder extends VideoMediaEncoder<TextureConfig> {
// 1. We must scale this matrix like GlCameraPreview does, because it might have some cropping.
// Scaling takes place with respect to the (0, 0, 0) point, so we must apply a Translation to compensate.
float[] transform = frame.transform;
float[] overlayTransform = frame.overlayTransform;
float scaleX = mConfig.scaleX;
float scaleY = mConfig.scaleY;
float scaleTranslX = (1F - scaleX) / 2F;
@ -149,14 +143,14 @@ public class TextureMediaEncoder extends VideoMediaEncoder<TextureConfig> {
// 3. Do the same for overlays with their own rotation.
if (mConfig.hasOverlay()) {
Matrix.translateM(overlayTransform, 0, 0.5F, 0.5F, 0);
Matrix.rotateM(overlayTransform, 0, mConfig.overlayRotation, 0, 0, 1);
Matrix.translateM(overlayTransform, 0, -0.5F, -0.5F, 0);
mConfig.overlayDrawer.draw(mConfig.overlayTarget);
Matrix.translateM(mConfig.overlayDrawer.getTransform(), 0, 0.5F, 0.5F, 0);
Matrix.rotateM(mConfig.overlayDrawer.getTransform(), 0, mConfig.overlayRotation, 0, 0, 1);
Matrix.translateM(mConfig.overlayDrawer.getTransform(), 0, -0.5F, -0.5F, 0);
}
mViewport.drawFrame(mConfig.textureId, transform);
if (mConfig.hasOverlay()) {
mViewport.drawFrame(mConfig.overlayTextureId, overlayTransform);
mConfig.issue514Workaround.afterOverlayGlDrawn();
mConfig.overlayDrawer.render();
}
mWindow.setPresentationTime(frame.timestamp);
mWindow.swapBuffers();

Loading…
Cancel
Save