|
|
|
@ -4,13 +4,9 @@ import android.annotation.TargetApi; |
|
|
|
|
import android.graphics.Bitmap; |
|
|
|
|
import android.graphics.Canvas; |
|
|
|
|
import android.graphics.Color; |
|
|
|
|
import android.graphics.ImageFormat; |
|
|
|
|
import android.graphics.PixelFormat; |
|
|
|
|
import android.graphics.PorterDuff; |
|
|
|
|
import android.graphics.Rect; |
|
|
|
|
import android.graphics.SurfaceTexture; |
|
|
|
|
import android.media.Image; |
|
|
|
|
import android.media.ImageReader; |
|
|
|
|
import android.opengl.EGL14; |
|
|
|
|
import android.opengl.EGLContext; |
|
|
|
|
import android.opengl.Matrix; |
|
|
|
@ -40,8 +36,6 @@ import androidx.annotation.Nullable; |
|
|
|
|
|
|
|
|
|
import android.view.Surface; |
|
|
|
|
|
|
|
|
|
import java.nio.ByteBuffer; |
|
|
|
|
|
|
|
|
|
/** |
|
|
|
|
* API 19. |
|
|
|
|
* Records a picture snapshots from the {@link GlCameraPreview}. It works as follows: |
|
|
|
@ -55,41 +49,15 @@ import java.nio.ByteBuffer; |
|
|
|
|
* - [Optional: fill the overlayTextureId and draw it on the same surface] |
|
|
|
|
* - We use glReadPixels (through {@link EglBaseSurface#saveFrameTo(Bitmap.CompressFormat)}) and save to file. |
|
|
|
|
* |
|
|
|
|
* We create a new surface and redraw the frame because: |
|
|
|
|
* We create a new EGL surface and redraw the frame because: |
|
|
|
|
* 1. We want to go off the renderer thread as soon as possible |
|
|
|
|
* 2. We have overlays to be drawn - we don't want to draw them on the preview surface, not even for a frame |
|
|
|
|
* |
|
|
|
|
* We can currently use two different methods: |
|
|
|
|
* |
|
|
|
|
* - {@link #METHOD_WINDOW_SURFACE} |
|
|
|
|
* This creates an EGL window surface using a new SurfaceTexture as output, |
|
|
|
|
* constructed with the preview textureId. This makes no real sense - |
|
|
|
|
* the preview is already managing a window surface and using it to render, |
|
|
|
|
* and we don't want to stream into that textureId: it would be both input and output. |
|
|
|
|
* |
|
|
|
|
* - {@link #METHOD_PBUFFER_SURFACE} |
|
|
|
|
* This creates an EGL pbuffer surface, passing desired width and height. |
|
|
|
|
* This is technically correct, we can just re-draw on this offscreen surface the input texture. |
|
|
|
|
* However, despite being more correct, this method is significantly slower than the previous. |
|
|
|
|
* |
|
|
|
|
* A third method is using {@link ImageReader} to construct a surface and use that |
|
|
|
|
* as the output for a new EGL window surface. This makes sense and would avoid us using glReadPixels. |
|
|
|
|
* However, it's not working. It looks like ImageReader is not capable of converting the input data |
|
|
|
|
* into JPEG: "RGBA override BLOB format buffer should have height == width" |
|
|
|
|
* 2. We have overlays to be drawn - we don't want to draw them on the preview surface, not even for a frame. |
|
|
|
|
*/ |
|
|
|
|
public class SnapshotGlPictureRecorder extends PictureRecorder { |
|
|
|
|
|
|
|
|
|
private static final String TAG = SnapshotGlPictureRecorder.class.getSimpleName(); |
|
|
|
|
private static final CameraLogger LOG = CameraLogger.create(TAG); |
|
|
|
|
|
|
|
|
|
private static final int METHOD_WINDOW_SURFACE = 0; |
|
|
|
|
private static final int METHOD_PBUFFER_SURFACE = 1; |
|
|
|
|
private static final int METHOD_IMAGEREADER_SURFACE = 2; |
|
|
|
|
|
|
|
|
|
// METHOD_PBUFFER_SURFACE: (409 + 420 + 394 + 424 + 437 + 482 + 420 + 463 + 451 + 378 + 416 + 429 + 459 + 361 + 466 + 517 + 492 + 435 + 585 + 427) / 20
|
|
|
|
|
// METHOD_WINDOW_SURFACE: (329 + 462 + 343 + 312 + 311 + 322 + 495 + 342 + 310 + 329 + 373 + 380 + 667 + 315 + 354 + 351 + 315 + 333 + 277 + 274) / 20
|
|
|
|
|
private static int METHOD = METHOD_WINDOW_SURFACE; |
|
|
|
|
|
|
|
|
|
private CameraEngine mEngine; |
|
|
|
|
private GlCameraPreview mPreview; |
|
|
|
|
private AspectRatio mOutputRatio; |
|
|
|
@ -97,6 +65,17 @@ public class SnapshotGlPictureRecorder extends PictureRecorder { |
|
|
|
|
private Overlay mOverlay; |
|
|
|
|
private boolean mHasOverlay; |
|
|
|
|
|
|
|
|
|
private int mTextureId; |
|
|
|
|
private SurfaceTexture mSurfaceTexture; |
|
|
|
|
private float[] mTransform; |
|
|
|
|
|
|
|
|
|
private int mOverlayTextureId = 0; |
|
|
|
|
private SurfaceTexture mOverlaySurfaceTexture; |
|
|
|
|
private Surface mOverlaySurface; |
|
|
|
|
private float[] mOverlayTransform; |
|
|
|
|
|
|
|
|
|
private EglViewport mViewport; |
|
|
|
|
|
|
|
|
|
public SnapshotGlPictureRecorder( |
|
|
|
|
@NonNull PictureResult.Stub stub, |
|
|
|
|
@NonNull CameraEngine engine, |
|
|
|
@ -116,19 +95,23 @@ public class SnapshotGlPictureRecorder extends PictureRecorder { |
|
|
|
|
public void take() { |
|
|
|
|
mPreview.addRendererFrameCallback(new RendererFrameCallback() { |
|
|
|
|
|
|
|
|
|
int mTextureId; |
|
|
|
|
SurfaceTexture mSurfaceTexture; |
|
|
|
|
float[] mTransform; |
|
|
|
|
|
|
|
|
|
int mOverlayTextureId = 0; |
|
|
|
|
SurfaceTexture mOverlaySurfaceTexture; |
|
|
|
|
Surface mOverlaySurface; |
|
|
|
|
float[] mOverlayTransform; |
|
|
|
|
@RendererThread |
|
|
|
|
public void onRendererTextureCreated(int textureId) { |
|
|
|
|
SnapshotGlPictureRecorder.this.onRendererTextureCreated(textureId); |
|
|
|
|
} |
|
|
|
|
|
|
|
|
|
EglViewport mViewport; |
|
|
|
|
@RendererThread |
|
|
|
|
@Override |
|
|
|
|
public void onRendererFrame(@NonNull SurfaceTexture surfaceTexture, final float scaleX, final float scaleY) { |
|
|
|
|
mPreview.removeRendererFrameCallback(this); |
|
|
|
|
SnapshotGlPictureRecorder.this.onRendererFrame(scaleX, scaleY); |
|
|
|
|
} |
|
|
|
|
}); |
|
|
|
|
} |
|
|
|
|
|
|
|
|
|
@RendererThread |
|
|
|
|
public void onRendererTextureCreated(int textureId) { |
|
|
|
|
@TargetApi(Build.VERSION_CODES.KITKAT) |
|
|
|
|
private void onRendererTextureCreated(int textureId) { |
|
|
|
|
mTextureId = textureId; |
|
|
|
|
mViewport = new EglViewport(); |
|
|
|
|
mSurfaceTexture = new SurfaceTexture(mTextureId, true); |
|
|
|
@ -147,11 +130,32 @@ public class SnapshotGlPictureRecorder extends PictureRecorder { |
|
|
|
|
} |
|
|
|
|
} |
|
|
|
|
|
|
|
|
|
/** |
|
|
|
|
* The tricky part here is the EGL surface creation. |
|
|
|
|
* |
|
|
|
|
* We don't have a real output window for the EGL surface - we will use glReadPixels() |
|
|
|
|
* and never call swapBuffers(), so what we draw is never published. |
|
|
|
|
* |
|
|
|
|
* 1. One option is to use a pbuffer EGL surface. This works, we just have to pass |
|
|
|
|
* the correct width and height. However, it is significantly slower than the current |
|
|
|
|
* solution. |
|
|
|
|
* |
|
|
|
|
* 2. Another option is to create the EGL surface out of a ImageReader.getSurface() |
|
|
|
|
* and use the reader to create a JPEG. In this case, we would have to publish |
|
|
|
|
* the frame with swapBuffers(). However, currently ImageReader does not support |
|
|
|
|
* all formats, it's risky. This is an example error that we get: |
|
|
|
|
* "RGBA override BLOB format buffer should have height == width" |
|
|
|
|
* |
|
|
|
|
* The third option, which we are using, is to create the EGL surface using whatever |
|
|
|
|
* {@link Surface} or {@link SurfaceTexture} we have at hand. Since we never call |
|
|
|
|
* swapBuffers(), the frame will not actually be rendered. This is the fastest. |
|
|
|
|
* |
|
|
|
|
* @param scaleX frame scale x in {@link Reference#VIEW} |
|
|
|
|
* @param scaleY frame scale y in {@link Reference#VIEW} |
|
|
|
|
*/ |
|
|
|
|
@RendererThread |
|
|
|
|
@Override |
|
|
|
|
public void onRendererFrame(@NonNull SurfaceTexture surfaceTexture, final float scaleX, final float scaleY) { |
|
|
|
|
mPreview.removeRendererFrameCallback(this); |
|
|
|
|
|
|
|
|
|
@TargetApi(Build.VERSION_CODES.KITKAT) |
|
|
|
|
private void onRendererFrame(final float scaleX, final float scaleY) { |
|
|
|
|
// Get egl context from the RendererThread, which is the one in which we have created
|
|
|
|
|
// the textureId and the overlayTextureId, managed by the GlSurfaceView.
|
|
|
|
|
// Next operations can then be performed on different threads using this handle.
|
|
|
|
@ -160,30 +164,15 @@ public class SnapshotGlPictureRecorder extends PictureRecorder { |
|
|
|
|
WorkerHandler.execute(new Runnable() { |
|
|
|
|
@Override |
|
|
|
|
public void run() { |
|
|
|
|
// 0. Create the output surface.
|
|
|
|
|
ImageReader reader = null; |
|
|
|
|
EglBaseSurface eglSurface; |
|
|
|
|
if (METHOD == METHOD_WINDOW_SURFACE) { |
|
|
|
|
eglSurface = new EglWindowSurface(core, mSurfaceTexture); |
|
|
|
|
eglSurface.makeCurrent(); |
|
|
|
|
} else if (METHOD == METHOD_PBUFFER_SURFACE) { |
|
|
|
|
eglSurface = new EglBaseSurface(core); |
|
|
|
|
eglSurface.createOffscreenSurface(mResult.size.getWidth(), mResult.size.getHeight()); |
|
|
|
|
eglSurface.makeCurrent(); |
|
|
|
|
} else if (METHOD == METHOD_IMAGEREADER_SURFACE) { |
|
|
|
|
reader = ImageReader.newInstance(mResult.size.getWidth(), mResult.size.getHeight(), ImageFormat.JPEG, 1); |
|
|
|
|
eglSurface = new EglWindowSurface(core, reader.getSurface(), true); |
|
|
|
|
// 0. Create an EGL surface
|
|
|
|
|
EglBaseSurface eglSurface = new EglWindowSurface(core, mSurfaceTexture); |
|
|
|
|
eglSurface.makeCurrent(); |
|
|
|
|
} else { |
|
|
|
|
throw new RuntimeException("Unknown method."); |
|
|
|
|
} |
|
|
|
|
|
|
|
|
|
// 1. Get latest texture
|
|
|
|
|
mSurfaceTexture.updateTexImage(); |
|
|
|
|
mSurfaceTexture.getTransformMatrix(mTransform); |
|
|
|
|
|
|
|
|
|
// 2. Apply scale and crop:
|
|
|
|
|
// scaleX and scaleY are in REF_VIEW, while our input appears to be in REF_SENSOR.
|
|
|
|
|
// 2. Apply scale and crop
|
|
|
|
|
boolean flip = mEngine.getAngles().flip(Reference.VIEW, Reference.SENSOR); |
|
|
|
|
float realScaleX = flip ? scaleY : scaleX; |
|
|
|
|
float realScaleY = flip ? scaleX : scaleY; |
|
|
|
@ -192,26 +181,24 @@ public class SnapshotGlPictureRecorder extends PictureRecorder { |
|
|
|
|
Matrix.translateM(mTransform, 0, scaleTranslX, scaleTranslY, 0); |
|
|
|
|
Matrix.scaleM(mTransform, 0, realScaleX, realScaleY, 1); |
|
|
|
|
|
|
|
|
|
// 3. Go back to 0,0 so that rotate and flip work well.
|
|
|
|
|
// 3. Go back to 0,0 so that rotate and flip work well
|
|
|
|
|
Matrix.translateM(mTransform, 0, 0.5F, 0.5F, 0); |
|
|
|
|
|
|
|
|
|
// 4. Apply rotation:
|
|
|
|
|
// Not sure why we need the minus here.
|
|
|
|
|
// 4. Apply rotation (not sure why we need the minus here)
|
|
|
|
|
Matrix.rotateM(mTransform, 0, -mResult.rotation, 0, 0, 1); |
|
|
|
|
mResult.rotation = 0; |
|
|
|
|
|
|
|
|
|
// 5. Flip horizontally for front camera:
|
|
|
|
|
// 5. Flip horizontally for front camera
|
|
|
|
|
if (mResult.facing == Facing.FRONT) { |
|
|
|
|
Matrix.scaleM(mTransform, 0, -1, 1, 1); |
|
|
|
|
} |
|
|
|
|
|
|
|
|
|
// 6. Go back to old position.
|
|
|
|
|
// 6. Go back to old position
|
|
|
|
|
Matrix.translateM(mTransform, 0, -0.5F, -0.5F, 0); |
|
|
|
|
|
|
|
|
|
// 7. Do pretty much the same for overlays, though with
|
|
|
|
|
// some differences.
|
|
|
|
|
// 7. Do pretty much the same for overlays
|
|
|
|
|
if (mHasOverlay) { |
|
|
|
|
// 1. First we must draw on the texture and get latest image.
|
|
|
|
|
// 1. First we must draw on the texture and get latest image
|
|
|
|
|
try { |
|
|
|
|
final Canvas surfaceCanvas = mOverlaySurface.lockCanvas(null); |
|
|
|
|
surfaceCanvas.drawColor(Color.TRANSPARENT, PorterDuff.Mode.CLEAR); |
|
|
|
@ -223,7 +210,7 @@ public class SnapshotGlPictureRecorder extends PictureRecorder { |
|
|
|
|
mOverlaySurfaceTexture.updateTexImage(); |
|
|
|
|
mOverlaySurfaceTexture.getTransformMatrix(mOverlayTransform); |
|
|
|
|
|
|
|
|
|
// 2. Then we can apply the transformations.
|
|
|
|
|
// 2. Then we can apply the transformations
|
|
|
|
|
int rotation = mEngine.getAngles().offset(Reference.VIEW, Reference.OUTPUT, Axis.ABSOLUTE); |
|
|
|
|
Matrix.translateM(mOverlayTransform, 0, 0.5F, 0.5F, 0); |
|
|
|
|
Matrix.rotateM(mOverlayTransform, 0, rotation, 0, 0, 1); |
|
|
|
@ -236,29 +223,7 @@ public class SnapshotGlPictureRecorder extends PictureRecorder { |
|
|
|
|
mViewport.drawFrame(mTextureId, mTransform); |
|
|
|
|
if (mHasOverlay) mViewport.drawFrame(mOverlayTextureId, mOverlayTransform); |
|
|
|
|
mResult.format = PictureResult.FORMAT_JPEG; |
|
|
|
|
if (METHOD == METHOD_WINDOW_SURFACE) { |
|
|
|
|
// We do not call swapBuffers so we do not really publish to the texture.
|
|
|
|
|
// We read pixels at the gl level.
|
|
|
|
|
mResult.data = eglSurface.saveFrameTo(Bitmap.CompressFormat.JPEG); |
|
|
|
|
} else if (METHOD == METHOD_PBUFFER_SURFACE) { |
|
|
|
|
// For pbuffer surfaces, swapBuffers is not needed. They already hold
|
|
|
|
|
// the data, so we can call glReadPixels safely.
|
|
|
|
|
mResult.data = eglSurface.saveFrameTo(Bitmap.CompressFormat.JPEG); |
|
|
|
|
} else if (METHOD == METHOD_IMAGEREADER_SURFACE) { |
|
|
|
|
// Publish into the output Surface.
|
|
|
|
|
eglSurface.swapBuffers(); |
|
|
|
|
Image image; |
|
|
|
|
//noinspection ConstantConditions,StatementWithEmptyBody
|
|
|
|
|
while ((image = reader.acquireNextImage()) == null) {} |
|
|
|
|
ByteBuffer buffer = image.getPlanes()[0].getBuffer(); |
|
|
|
|
byte[] bytes = new byte[buffer.remaining()]; |
|
|
|
|
buffer.get(bytes); |
|
|
|
|
mResult.data = bytes; |
|
|
|
|
image.close(); |
|
|
|
|
reader.close(); |
|
|
|
|
} else { |
|
|
|
|
throw new RuntimeException("Unknown method."); |
|
|
|
|
} |
|
|
|
|
|
|
|
|
|
// 9. Cleanup
|
|
|
|
|
mSurfaceTexture.releaseTexImage(); |
|
|
|
@ -275,8 +240,6 @@ public class SnapshotGlPictureRecorder extends PictureRecorder { |
|
|
|
|
} |
|
|
|
|
}); |
|
|
|
|
} |
|
|
|
|
}); |
|
|
|
|
} |
|
|
|
|
|
|
|
|
|
@Override |
|
|
|
|
protected void dispatchResult() { |
|
|
|
|