From 04697e917de01f6cf402440198c44b9f298715d6 Mon Sep 17 00:00:00 2001 From: Mattia Iavarone Date: Sun, 1 Sep 2019 01:41:41 +0200 Subject: [PATCH] Flash support for metered snapshots --- .../picture/Snapshot2PictureRecorder.java | 192 +++++++++--------- .../cameraview/preview/GlCameraPreview.java | 7 +- 2 files changed, 103 insertions(+), 96 deletions(-) diff --git a/cameraview/src/main/java/com/otaliastudios/cameraview/picture/Snapshot2PictureRecorder.java b/cameraview/src/main/java/com/otaliastudios/cameraview/picture/Snapshot2PictureRecorder.java index d67ad0b3..364a8025 100644 --- a/cameraview/src/main/java/com/otaliastudios/cameraview/picture/Snapshot2PictureRecorder.java +++ b/cameraview/src/main/java/com/otaliastudios/cameraview/picture/Snapshot2PictureRecorder.java @@ -6,23 +6,34 @@ import android.hardware.camera2.CameraCaptureSession; import android.hardware.camera2.CaptureRequest; import android.hardware.camera2.CaptureResult; import android.hardware.camera2.TotalCaptureResult; -import android.opengl.EGL14; -import android.opengl.EGLContext; import android.os.Build; -import android.util.Log; import androidx.annotation.NonNull; -import androidx.annotation.Nullable; import androidx.annotation.RequiresApi; import com.otaliastudios.cameraview.CameraLogger; import com.otaliastudios.cameraview.PictureResult; import com.otaliastudios.cameraview.engine.Camera2Engine; -import com.otaliastudios.cameraview.filter.Filter; import com.otaliastudios.cameraview.preview.GlCameraPreview; -import com.otaliastudios.cameraview.preview.RendererFrameCallback; import com.otaliastudios.cameraview.size.AspectRatio; +/** + * Wraps {@link SnapshotGlPictureRecorder} for Camera2. + * + * Camera2 engine supports metering for snapshots and we expect for them to correctly fire flash as well. + * The first idea, and in theory, the most correct one, was to set {@link CaptureRequest#CONTROL_CAPTURE_INTENT} + * to {@link CaptureRequest#CONTROL_CAPTURE_INTENT_STILL_CAPTURE}. + * + * According to documentation, this will automatically trigger the flash if parameters says so. + * In fact this is what happens, but it is a very fast flash that only lasts for 1 or 2 frames. + * It's not easy to call super.take() at the exact time so that we capture the frame that was lit. + * I have tried by comparing {@link SurfaceTexture#getTimestamp()} and {@link CaptureResult#SENSOR_TIMESTAMP} + * to identify the correct frame. These timestamps match, but the frame is not the correct one. + * + * So what we do here is ignore the {@link CaptureRequest#CONTROL_CAPTURE_INTENT} and instead open the + * torch, if requested to do so. Then wait for exposure to settle again and finally take a snapshot. + * I'd still love to use the capture intent instead of this, but was not able yet. + */ @RequiresApi(Build.VERSION_CODES.LOLLIPOP) public class Snapshot2PictureRecorder extends SnapshotGlPictureRecorder { @@ -30,24 +41,18 @@ public class Snapshot2PictureRecorder extends SnapshotGlPictureRecorder { private final static CameraLogger LOG = CameraLogger.create(TAG); private final static int STATE_IDLE = 0; - private final static int STATE_WAITING_CAPTURE = 1; - private final static int STATE_WAITING_CORRECT_FRAME = 2; - private final static int STATE_WAITING_IMAGE = 3; + private final static int STATE_WAITING_FIRST_FRAME = 1; + private final static int STATE_WAITING_TORCH = 2; + private final static int STATE_WAITING_TORCH_EXPOSURE = 3; + private final static int STATE_WAITING_IMAGE = 4; private final Camera2Engine mEngine; - private final GlCameraPreview mPreview; private final CameraCaptureSession mSession; private final CameraCaptureSession.CaptureCallback mCallback; private final CaptureRequest.Builder mBuilder; private int mState = STATE_IDLE; - private Integer mOldCaptureIntent; - private int mSequenceId; - - private SurfaceTexture mLastFrameSurfaceTexture; - private float mLastFrameScaleX; - private float mLastFrameScaleY; - private EGLContext mLastFrameScaleEGLContext; - private Long mDesiredTimestamp = null; + private Integer mOriginalAeMode; + private Integer mOriginalFlashMode; public Snapshot2PictureRecorder(@NonNull PictureResult.Stub stub, @NonNull Camera2Engine engine, @@ -58,95 +63,94 @@ public class Snapshot2PictureRecorder extends SnapshotGlPictureRecorder { @NonNull CaptureRequest.Builder builder) { super(stub, engine, preview, outputRatio); mEngine = engine; - mPreview = preview; mSession = session; mCallback = callback; mBuilder = builder; } - private final RendererFrameCallback mRendererCallback = new RendererFrameCallback() { - @Override - public void onRendererTextureCreated(int textureId) { - Snapshot2PictureRecorder.super.onRendererTextureCreated(textureId); - } - - @Override - public void onRendererFilterChanged(@NonNull Filter filter) { - Snapshot2PictureRecorder.super.onRendererFilterChanged(filter); - } - - @Override - public void onRendererFrame(@NonNull SurfaceTexture surfaceTexture, float scaleX, float scaleY) { - mLastFrameSurfaceTexture = surfaceTexture; - mLastFrameScaleX = scaleX; - mLastFrameScaleY = scaleY; - mLastFrameScaleEGLContext = EGL14.eglGetCurrentContext(); - maybeTakeFrame(); - } - }; - @Override public void take() { if (!mEngine.getPictureSnapshotMetering()) { + LOG.i("take:", "Engine does no metering, taking fast snapshot."); super.take(); - return; - } - - LOG.i("take:", "Engine does metering, adding our CONTROL_CAPTURE_INTENT."); - mPreview.addRendererFrameCallback(mRendererCallback); - mState = STATE_WAITING_CAPTURE; - try { - mOldCaptureIntent = mBuilder.get(CaptureRequest.CONTROL_CAPTURE_INTENT); - mBuilder.set(CaptureRequest.CONTROL_CAPTURE_INTENT, CaptureRequest.CONTROL_CAPTURE_INTENT_STILL_CAPTURE); - mSequenceId = mSession.setRepeatingRequest(mBuilder.build(), mCallback, null); - } catch (CameraAccessException e) { - LOG.w("take:", "Camera error while applying our CONTROL_CAPTURE_INTENT", e); - mResult = null; - mError = e; - dispatchResult(); + } else { + LOG.i("take:", "Engine does metering, caring about flash."); + mState = STATE_WAITING_FIRST_FRAME; + mOriginalAeMode = mBuilder.get(CaptureRequest.CONTROL_AE_MODE); + mOriginalFlashMode = mBuilder.get(CaptureRequest.FLASH_MODE); } } + /** + * This works by inspecting {@link CaptureResult#CONTROL_AE_STATE}, and specifically + * the {@link CaptureRequest#CONTROL_AE_STATE_FLASH_REQUIRED} value. + * The AE state will have this value iff: + * 1. flash is required + * 2. device has a flash + * 3. we are not in Flash.OFF + * So when we see this value we just have to open the torch and wait a bit more, + * just to have a correct exposure. + * + * @param result a total result + */ public void onCaptureCompleted(@NonNull TotalCaptureResult result) { - LOG.i("onCaptureCompleted:", - "aeState:", result.get(CaptureResult.CONTROL_AE_STATE), - "flashState:", result.get(CaptureResult.FLASH_STATE)); - if (mState == STATE_WAITING_CAPTURE) { - if (result.getSequenceId() == mSequenceId) { - // This is the request we launched and we're waiting for the right frame. - LOG.w("onCaptureCompleted:", - "aeState:", result.get(CaptureResult.CONTROL_AE_STATE), - "flashState:", result.get(CaptureResult.FLASH_STATE)); - mState = STATE_WAITING_CORRECT_FRAME; - mDesiredTimestamp = result.get(CaptureResult.SENSOR_TIMESTAMP); - if (mDesiredTimestamp == null) mDesiredTimestamp = 0L; - LOG.i("onCaptureCompleted:", "Got timestamp:", mDesiredTimestamp); - maybeTakeFrame(); + if (mState == STATE_WAITING_FIRST_FRAME) { + Integer aeState = result.get(CaptureResult.CONTROL_AE_STATE); + if (aeState == null) { + LOG.w("onCaptureCompleted:", "aeState is null! This should never happen. Taking snapshot."); + mState = STATE_WAITING_IMAGE; + super.take(); + } else if (aeState == CaptureRequest.CONTROL_AE_STATE_CONVERGED) { + LOG.i("onCaptureCompleted:", "aeState is optimal. Taking snapshot."); + mState = STATE_WAITING_IMAGE; + super.take(); + } else if (aeState != CaptureRequest.CONTROL_AE_STATE_FLASH_REQUIRED) { + LOG.w("onCaptureCompleted:", "aeState is not CONVERGED yet not FLASH_REQUIRED.", + "This seems to say that metering did not complete before. Let's take the snapshot."); + // TODO the metering system might set AE to LOCKED. what to do in this case? + mState = STATE_WAITING_IMAGE; + super.take(); + } else { + // Open the torch. AE_MODE_ON is guaranteed to be supported. + // Remove any AE lock that was set by the engine. + LOG.w("onCaptureCompleted:", "aeState is FLASH_REQUIRED. Opening torch."); + mState = STATE_WAITING_TORCH; + mBuilder.set(CaptureRequest.FLASH_MODE, CaptureRequest.FLASH_MODE_TORCH); + mBuilder.set(CaptureRequest.CONTROL_AE_MODE, CaptureRequest.CONTROL_AE_MODE_ON); + mBuilder.set(CaptureRequest.CONTROL_AE_LOCK, false); + try { + mSession.setRepeatingRequest(mBuilder.build(), mCallback, null); + } catch (CameraAccessException e) { + mResult = null; + mError = e; + dispatchResult(); + } } - } - } - private void maybeTakeFrame() { - if (mState != STATE_WAITING_CORRECT_FRAME) { - LOG.w("maybeTakeFrame:", "we're not waiting for a frame. Ignoring.", mState); - return; - } - if (mDesiredTimestamp == null || mLastFrameSurfaceTexture == null) { - LOG.w("maybeTakeFrame:", "either timestamp or surfaceTexture are null.", mDesiredTimestamp); - return; - } + } else if (mState == STATE_WAITING_TORCH) { + Integer flashState = result.get(CaptureResult.FLASH_STATE); + if (flashState == null) { + LOG.w("onCaptureCompleted:", "Waiting flash, but flashState is null! Taking snapshot."); + mState = STATE_WAITING_IMAGE; + super.take(); + } else if (flashState == CaptureResult.FLASH_STATE_FIRED) { + LOG.i("onCaptureCompleted:", "Waiting flash and we have FIRED state! Waiting exposure."); + mState = STATE_WAITING_TORCH_EXPOSURE; + } else { + LOG.i("onCaptureCompleted:", "aeState is FLASH_REQUIRED but flashState is", + flashState, ". Waiting..."); + } - long currentTimestamp = mLastFrameSurfaceTexture.getTimestamp(); - if (currentTimestamp == mDesiredTimestamp) { - LOG.i("maybeTakeFrame:", "taking frame with exact timestamp:", currentTimestamp); - mState = STATE_WAITING_IMAGE; - takeFrame(mLastFrameSurfaceTexture, mLastFrameScaleX, mLastFrameScaleY, mLastFrameScaleEGLContext); - } else if (currentTimestamp > mDesiredTimestamp) { - LOG.w("maybeTakeFrame:", "taking frame with some delay. Flash might not be respected."); - mState = STATE_WAITING_IMAGE; - takeFrame(mLastFrameSurfaceTexture, mLastFrameScaleX, mLastFrameScaleY, mLastFrameScaleEGLContext); - } else { - LOG.i("maybeTakeFrame:", "Waiting...", mDesiredTimestamp - currentTimestamp); + } else if (mState == STATE_WAITING_TORCH_EXPOSURE) { + Integer aeState = result.get(CaptureResult.CONTROL_AE_STATE); + if (aeState != null && (aeState == CaptureResult.CONTROL_AE_STATE_CONVERGED + || aeState == CaptureResult.CONTROL_AE_STATE_FLASH_REQUIRED)) { + LOG.i("onCaptureCompleted:", "We have torch and good torch exposure.", aeState); + mState = STATE_WAITING_IMAGE; + super.take(); + } else { + LOG.i("onCaptureCompleted:", "Waiting for torch exposure...:", aeState); + } } } @@ -156,11 +160,13 @@ public class Snapshot2PictureRecorder extends SnapshotGlPictureRecorder { // Revert our changes. LOG.i("dispatchResult:", "Reverting the capture intent changes."); try { - mBuilder.set(CaptureRequest.CONTROL_CAPTURE_INTENT, mOldCaptureIntent); + // See Camera2Engine.setFlash(). It's better to change these one by one. + mBuilder.set(CaptureRequest.CONTROL_AE_MODE, mOriginalAeMode); + mSession.capture(mBuilder.build(), mCallback, null); + mBuilder.set(CaptureRequest.FLASH_MODE, mOriginalFlashMode); mSession.setRepeatingRequest(mBuilder.build(), mCallback, null); } catch (CameraAccessException ignore) {} } - mPreview.removeRendererFrameCallback(mRendererCallback); mState = STATE_IDLE; super.dispatchResult(); } diff --git a/cameraview/src/main/java/com/otaliastudios/cameraview/preview/GlCameraPreview.java b/cameraview/src/main/java/com/otaliastudios/cameraview/preview/GlCameraPreview.java index f09c8977..10754fa3 100644 --- a/cameraview/src/main/java/com/otaliastudios/cameraview/preview/GlCameraPreview.java +++ b/cameraview/src/main/java/com/otaliastudios/cameraview/preview/GlCameraPreview.java @@ -187,13 +187,14 @@ public class GlCameraPreview extends FilterCameraPreview