Review snapshot picture recorder again

pull/524/head
Mattia Iavarone 6 years ago
parent bcd331c495
commit 70db8763c4
  1. 161
      cameraview/src/main/java/com/otaliastudios/cameraview/picture/SnapshotGlPictureRecorder.java

@ -4,13 +4,9 @@ import android.annotation.TargetApi;
import android.graphics.Bitmap; import android.graphics.Bitmap;
import android.graphics.Canvas; import android.graphics.Canvas;
import android.graphics.Color; import android.graphics.Color;
import android.graphics.ImageFormat;
import android.graphics.PixelFormat;
import android.graphics.PorterDuff; import android.graphics.PorterDuff;
import android.graphics.Rect; import android.graphics.Rect;
import android.graphics.SurfaceTexture; import android.graphics.SurfaceTexture;
import android.media.Image;
import android.media.ImageReader;
import android.opengl.EGL14; import android.opengl.EGL14;
import android.opengl.EGLContext; import android.opengl.EGLContext;
import android.opengl.Matrix; import android.opengl.Matrix;
@ -40,8 +36,6 @@ import androidx.annotation.Nullable;
import android.view.Surface; import android.view.Surface;
import java.nio.ByteBuffer;
/** /**
* API 19. * API 19.
* Records a picture snapshots from the {@link GlCameraPreview}. It works as follows: * Records a picture snapshots from the {@link GlCameraPreview}. It works as follows:
@ -55,41 +49,15 @@ import java.nio.ByteBuffer;
* - [Optional: fill the overlayTextureId and draw it on the same surface] * - [Optional: fill the overlayTextureId and draw it on the same surface]
* - We use glReadPixels (through {@link EglBaseSurface#saveFrameTo(Bitmap.CompressFormat)}) and save to file. * - We use glReadPixels (through {@link EglBaseSurface#saveFrameTo(Bitmap.CompressFormat)}) and save to file.
* *
* We create a new surface and redraw the frame because: * We create a new EGL surface and redraw the frame because:
* 1. We want to go off the renderer thread as soon as possible * 1. We want to go off the renderer thread as soon as possible
* 2. We have overlays to be drawn - we don't want to draw them on the preview surface, not even for a frame * 2. We have overlays to be drawn - we don't want to draw them on the preview surface, not even for a frame.
*
* We can currently use two different methods:
*
* - {@link #METHOD_WINDOW_SURFACE}
* This creates an EGL window surface using a new SurfaceTexture as output,
* constructed with the preview textureId. This makes no real sense -
* the preview is already managing a window surface and using it to render,
* and we don't want to stream into that textureId: it would be both input and output.
*
* - {@link #METHOD_PBUFFER_SURFACE}
* This creates an EGL pbuffer surface, passing desired width and height.
* This is technically correct, we can just re-draw on this offscreen surface the input texture.
* However, despite being more correct, this method is significantly slower than the previous.
*
* A third method is using {@link ImageReader} to construct a surface and use that
* as the output for a new EGL window surface. This makes sense and would avoid us using glReadPixels.
* However, it's not working. It looks like ImageReader is not capable of converting the input data
* into JPEG: "RGBA override BLOB format buffer should have height == width"
*/ */
public class SnapshotGlPictureRecorder extends PictureRecorder { public class SnapshotGlPictureRecorder extends PictureRecorder {
private static final String TAG = SnapshotGlPictureRecorder.class.getSimpleName(); private static final String TAG = SnapshotGlPictureRecorder.class.getSimpleName();
private static final CameraLogger LOG = CameraLogger.create(TAG); private static final CameraLogger LOG = CameraLogger.create(TAG);
private static final int METHOD_WINDOW_SURFACE = 0;
private static final int METHOD_PBUFFER_SURFACE = 1;
private static final int METHOD_IMAGEREADER_SURFACE = 2;
// METHOD_PBUFFER_SURFACE: (409 + 420 + 394 + 424 + 437 + 482 + 420 + 463 + 451 + 378 + 416 + 429 + 459 + 361 + 466 + 517 + 492 + 435 + 585 + 427) / 20
// METHOD_WINDOW_SURFACE: (329 + 462 + 343 + 312 + 311 + 322 + 495 + 342 + 310 + 329 + 373 + 380 + 667 + 315 + 354 + 351 + 315 + 333 + 277 + 274) / 20
private static int METHOD = METHOD_WINDOW_SURFACE;
private CameraEngine mEngine; private CameraEngine mEngine;
private GlCameraPreview mPreview; private GlCameraPreview mPreview;
private AspectRatio mOutputRatio; private AspectRatio mOutputRatio;
@ -97,6 +65,17 @@ public class SnapshotGlPictureRecorder extends PictureRecorder {
private Overlay mOverlay; private Overlay mOverlay;
private boolean mHasOverlay; private boolean mHasOverlay;
private int mTextureId;
private SurfaceTexture mSurfaceTexture;
private float[] mTransform;
private int mOverlayTextureId = 0;
private SurfaceTexture mOverlaySurfaceTexture;
private Surface mOverlaySurface;
private float[] mOverlayTransform;
private EglViewport mViewport;
public SnapshotGlPictureRecorder( public SnapshotGlPictureRecorder(
@NonNull PictureResult.Stub stub, @NonNull PictureResult.Stub stub,
@NonNull CameraEngine engine, @NonNull CameraEngine engine,
@ -116,19 +95,23 @@ public class SnapshotGlPictureRecorder extends PictureRecorder {
public void take() { public void take() {
mPreview.addRendererFrameCallback(new RendererFrameCallback() { mPreview.addRendererFrameCallback(new RendererFrameCallback() {
int mTextureId; @RendererThread
SurfaceTexture mSurfaceTexture; public void onRendererTextureCreated(int textureId) {
float[] mTransform; SnapshotGlPictureRecorder.this.onRendererTextureCreated(textureId);
}
int mOverlayTextureId = 0;
SurfaceTexture mOverlaySurfaceTexture;
Surface mOverlaySurface;
float[] mOverlayTransform;
EglViewport mViewport; @RendererThread
@Override
public void onRendererFrame(@NonNull SurfaceTexture surfaceTexture, final float scaleX, final float scaleY) {
mPreview.removeRendererFrameCallback(this);
SnapshotGlPictureRecorder.this.onRendererFrame(scaleX, scaleY);
}
});
}
@RendererThread @RendererThread
public void onRendererTextureCreated(int textureId) { @TargetApi(Build.VERSION_CODES.KITKAT)
private void onRendererTextureCreated(int textureId) {
mTextureId = textureId; mTextureId = textureId;
mViewport = new EglViewport(); mViewport = new EglViewport();
mSurfaceTexture = new SurfaceTexture(mTextureId, true); mSurfaceTexture = new SurfaceTexture(mTextureId, true);
@ -147,11 +130,32 @@ public class SnapshotGlPictureRecorder extends PictureRecorder {
} }
} }
/**
* The tricky part here is the EGL surface creation.
*
* We don't have a real output window for the EGL surface - we will use glReadPixels()
* and never call swapBuffers(), so what we draw is never published.
*
* 1. One option is to use a pbuffer EGL surface. This works, we just have to pass
* the correct width and height. However, it is significantly slower than the current
* solution.
*
* 2. Another option is to create the EGL surface out of a ImageReader.getSurface()
* and use the reader to create a JPEG. In this case, we would have to publish
* the frame with swapBuffers(). However, currently ImageReader does not support
* all formats, it's risky. This is an example error that we get:
* "RGBA override BLOB format buffer should have height == width"
*
* The third option, which we are using, is to create the EGL surface using whatever
* {@link Surface} or {@link SurfaceTexture} we have at hand. Since we never call
* swapBuffers(), the frame will not actually be rendered. This is the fastest.
*
* @param scaleX frame scale x in {@link Reference#VIEW}
* @param scaleY frame scale y in {@link Reference#VIEW}
*/
@RendererThread @RendererThread
@Override @TargetApi(Build.VERSION_CODES.KITKAT)
public void onRendererFrame(@NonNull SurfaceTexture surfaceTexture, final float scaleX, final float scaleY) { private void onRendererFrame(final float scaleX, final float scaleY) {
mPreview.removeRendererFrameCallback(this);
// Get egl context from the RendererThread, which is the one in which we have created // Get egl context from the RendererThread, which is the one in which we have created
// the textureId and the overlayTextureId, managed by the GlSurfaceView. // the textureId and the overlayTextureId, managed by the GlSurfaceView.
// Next operations can then be performed on different threads using this handle. // Next operations can then be performed on different threads using this handle.
@ -160,30 +164,15 @@ public class SnapshotGlPictureRecorder extends PictureRecorder {
WorkerHandler.execute(new Runnable() { WorkerHandler.execute(new Runnable() {
@Override @Override
public void run() { public void run() {
// 0. Create the output surface. // 0. Create an EGL surface
ImageReader reader = null; EglBaseSurface eglSurface = new EglWindowSurface(core, mSurfaceTexture);
EglBaseSurface eglSurface;
if (METHOD == METHOD_WINDOW_SURFACE) {
eglSurface = new EglWindowSurface(core, mSurfaceTexture);
eglSurface.makeCurrent();
} else if (METHOD == METHOD_PBUFFER_SURFACE) {
eglSurface = new EglBaseSurface(core);
eglSurface.createOffscreenSurface(mResult.size.getWidth(), mResult.size.getHeight());
eglSurface.makeCurrent();
} else if (METHOD == METHOD_IMAGEREADER_SURFACE) {
reader = ImageReader.newInstance(mResult.size.getWidth(), mResult.size.getHeight(), ImageFormat.JPEG, 1);
eglSurface = new EglWindowSurface(core, reader.getSurface(), true);
eglSurface.makeCurrent(); eglSurface.makeCurrent();
} else {
throw new RuntimeException("Unknown method.");
}
// 1. Get latest texture // 1. Get latest texture
mSurfaceTexture.updateTexImage(); mSurfaceTexture.updateTexImage();
mSurfaceTexture.getTransformMatrix(mTransform); mSurfaceTexture.getTransformMatrix(mTransform);
// 2. Apply scale and crop: // 2. Apply scale and crop
// scaleX and scaleY are in REF_VIEW, while our input appears to be in REF_SENSOR.
boolean flip = mEngine.getAngles().flip(Reference.VIEW, Reference.SENSOR); boolean flip = mEngine.getAngles().flip(Reference.VIEW, Reference.SENSOR);
float realScaleX = flip ? scaleY : scaleX; float realScaleX = flip ? scaleY : scaleX;
float realScaleY = flip ? scaleX : scaleY; float realScaleY = flip ? scaleX : scaleY;
@ -192,26 +181,24 @@ public class SnapshotGlPictureRecorder extends PictureRecorder {
Matrix.translateM(mTransform, 0, scaleTranslX, scaleTranslY, 0); Matrix.translateM(mTransform, 0, scaleTranslX, scaleTranslY, 0);
Matrix.scaleM(mTransform, 0, realScaleX, realScaleY, 1); Matrix.scaleM(mTransform, 0, realScaleX, realScaleY, 1);
// 3. Go back to 0,0 so that rotate and flip work well. // 3. Go back to 0,0 so that rotate and flip work well
Matrix.translateM(mTransform, 0, 0.5F, 0.5F, 0); Matrix.translateM(mTransform, 0, 0.5F, 0.5F, 0);
// 4. Apply rotation: // 4. Apply rotation (not sure why we need the minus here)
// Not sure why we need the minus here.
Matrix.rotateM(mTransform, 0, -mResult.rotation, 0, 0, 1); Matrix.rotateM(mTransform, 0, -mResult.rotation, 0, 0, 1);
mResult.rotation = 0; mResult.rotation = 0;
// 5. Flip horizontally for front camera: // 5. Flip horizontally for front camera
if (mResult.facing == Facing.FRONT) { if (mResult.facing == Facing.FRONT) {
Matrix.scaleM(mTransform, 0, -1, 1, 1); Matrix.scaleM(mTransform, 0, -1, 1, 1);
} }
// 6. Go back to old position. // 6. Go back to old position
Matrix.translateM(mTransform, 0, -0.5F, -0.5F, 0); Matrix.translateM(mTransform, 0, -0.5F, -0.5F, 0);
// 7. Do pretty much the same for overlays, though with // 7. Do pretty much the same for overlays
// some differences.
if (mHasOverlay) { if (mHasOverlay) {
// 1. First we must draw on the texture and get latest image. // 1. First we must draw on the texture and get latest image
try { try {
final Canvas surfaceCanvas = mOverlaySurface.lockCanvas(null); final Canvas surfaceCanvas = mOverlaySurface.lockCanvas(null);
surfaceCanvas.drawColor(Color.TRANSPARENT, PorterDuff.Mode.CLEAR); surfaceCanvas.drawColor(Color.TRANSPARENT, PorterDuff.Mode.CLEAR);
@ -223,7 +210,7 @@ public class SnapshotGlPictureRecorder extends PictureRecorder {
mOverlaySurfaceTexture.updateTexImage(); mOverlaySurfaceTexture.updateTexImage();
mOverlaySurfaceTexture.getTransformMatrix(mOverlayTransform); mOverlaySurfaceTexture.getTransformMatrix(mOverlayTransform);
// 2. Then we can apply the transformations. // 2. Then we can apply the transformations
int rotation = mEngine.getAngles().offset(Reference.VIEW, Reference.OUTPUT, Axis.ABSOLUTE); int rotation = mEngine.getAngles().offset(Reference.VIEW, Reference.OUTPUT, Axis.ABSOLUTE);
Matrix.translateM(mOverlayTransform, 0, 0.5F, 0.5F, 0); Matrix.translateM(mOverlayTransform, 0, 0.5F, 0.5F, 0);
Matrix.rotateM(mOverlayTransform, 0, rotation, 0, 0, 1); Matrix.rotateM(mOverlayTransform, 0, rotation, 0, 0, 1);
@ -236,29 +223,7 @@ public class SnapshotGlPictureRecorder extends PictureRecorder {
mViewport.drawFrame(mTextureId, mTransform); mViewport.drawFrame(mTextureId, mTransform);
if (mHasOverlay) mViewport.drawFrame(mOverlayTextureId, mOverlayTransform); if (mHasOverlay) mViewport.drawFrame(mOverlayTextureId, mOverlayTransform);
mResult.format = PictureResult.FORMAT_JPEG; mResult.format = PictureResult.FORMAT_JPEG;
if (METHOD == METHOD_WINDOW_SURFACE) {
// We do not call swapBuffers so we do not really publish to the texture.
// We read pixels at the gl level.
mResult.data = eglSurface.saveFrameTo(Bitmap.CompressFormat.JPEG);
} else if (METHOD == METHOD_PBUFFER_SURFACE) {
// For pbuffer surfaces, swapBuffers is not needed. They already hold
// the data, so we can call glReadPixels safely.
mResult.data = eglSurface.saveFrameTo(Bitmap.CompressFormat.JPEG); mResult.data = eglSurface.saveFrameTo(Bitmap.CompressFormat.JPEG);
} else if (METHOD == METHOD_IMAGEREADER_SURFACE) {
// Publish into the output Surface.
eglSurface.swapBuffers();
Image image;
//noinspection ConstantConditions,StatementWithEmptyBody
while ((image = reader.acquireNextImage()) == null) {}
ByteBuffer buffer = image.getPlanes()[0].getBuffer();
byte[] bytes = new byte[buffer.remaining()];
buffer.get(bytes);
mResult.data = bytes;
image.close();
reader.close();
} else {
throw new RuntimeException("Unknown method.");
}
// 9. Cleanup // 9. Cleanup
mSurfaceTexture.releaseTexImage(); mSurfaceTexture.releaseTexImage();
@ -275,8 +240,6 @@ public class SnapshotGlPictureRecorder extends PictureRecorder {
} }
}); });
} }
});
}
@Override @Override
protected void dispatchResult() { protected void dispatchResult() {

Loading…
Cancel
Save