Flash support for metered snapshots

pull/580/head
Mattia Iavarone 6 years ago
parent e31c3179b5
commit 04697e917d
  1. 192
      cameraview/src/main/java/com/otaliastudios/cameraview/picture/Snapshot2PictureRecorder.java
  2. 7
      cameraview/src/main/java/com/otaliastudios/cameraview/preview/GlCameraPreview.java

@ -6,23 +6,34 @@ import android.hardware.camera2.CameraCaptureSession;
import android.hardware.camera2.CaptureRequest; import android.hardware.camera2.CaptureRequest;
import android.hardware.camera2.CaptureResult; import android.hardware.camera2.CaptureResult;
import android.hardware.camera2.TotalCaptureResult; import android.hardware.camera2.TotalCaptureResult;
import android.opengl.EGL14;
import android.opengl.EGLContext;
import android.os.Build; import android.os.Build;
import android.util.Log;
import androidx.annotation.NonNull; import androidx.annotation.NonNull;
import androidx.annotation.Nullable;
import androidx.annotation.RequiresApi; import androidx.annotation.RequiresApi;
import com.otaliastudios.cameraview.CameraLogger; import com.otaliastudios.cameraview.CameraLogger;
import com.otaliastudios.cameraview.PictureResult; import com.otaliastudios.cameraview.PictureResult;
import com.otaliastudios.cameraview.engine.Camera2Engine; import com.otaliastudios.cameraview.engine.Camera2Engine;
import com.otaliastudios.cameraview.filter.Filter;
import com.otaliastudios.cameraview.preview.GlCameraPreview; import com.otaliastudios.cameraview.preview.GlCameraPreview;
import com.otaliastudios.cameraview.preview.RendererFrameCallback;
import com.otaliastudios.cameraview.size.AspectRatio; import com.otaliastudios.cameraview.size.AspectRatio;
/**
* Wraps {@link SnapshotGlPictureRecorder} for Camera2.
*
* Camera2 engine supports metering for snapshots and we expect for them to correctly fire flash as well.
* The first idea, and in theory, the most correct one, was to set {@link CaptureRequest#CONTROL_CAPTURE_INTENT}
* to {@link CaptureRequest#CONTROL_CAPTURE_INTENT_STILL_CAPTURE}.
*
* According to documentation, this will automatically trigger the flash if parameters says so.
* In fact this is what happens, but it is a very fast flash that only lasts for 1 or 2 frames.
* It's not easy to call super.take() at the exact time so that we capture the frame that was lit.
* I have tried by comparing {@link SurfaceTexture#getTimestamp()} and {@link CaptureResult#SENSOR_TIMESTAMP}
* to identify the correct frame. These timestamps match, but the frame is not the correct one.
*
* So what we do here is ignore the {@link CaptureRequest#CONTROL_CAPTURE_INTENT} and instead open the
* torch, if requested to do so. Then wait for exposure to settle again and finally take a snapshot.
* I'd still love to use the capture intent instead of this, but was not able yet.
*/
@RequiresApi(Build.VERSION_CODES.LOLLIPOP) @RequiresApi(Build.VERSION_CODES.LOLLIPOP)
public class Snapshot2PictureRecorder extends SnapshotGlPictureRecorder { public class Snapshot2PictureRecorder extends SnapshotGlPictureRecorder {
@ -30,24 +41,18 @@ public class Snapshot2PictureRecorder extends SnapshotGlPictureRecorder {
private final static CameraLogger LOG = CameraLogger.create(TAG); private final static CameraLogger LOG = CameraLogger.create(TAG);
private final static int STATE_IDLE = 0; private final static int STATE_IDLE = 0;
private final static int STATE_WAITING_CAPTURE = 1; private final static int STATE_WAITING_FIRST_FRAME = 1;
private final static int STATE_WAITING_CORRECT_FRAME = 2; private final static int STATE_WAITING_TORCH = 2;
private final static int STATE_WAITING_IMAGE = 3; private final static int STATE_WAITING_TORCH_EXPOSURE = 3;
private final static int STATE_WAITING_IMAGE = 4;
private final Camera2Engine mEngine; private final Camera2Engine mEngine;
private final GlCameraPreview mPreview;
private final CameraCaptureSession mSession; private final CameraCaptureSession mSession;
private final CameraCaptureSession.CaptureCallback mCallback; private final CameraCaptureSession.CaptureCallback mCallback;
private final CaptureRequest.Builder mBuilder; private final CaptureRequest.Builder mBuilder;
private int mState = STATE_IDLE; private int mState = STATE_IDLE;
private Integer mOldCaptureIntent; private Integer mOriginalAeMode;
private int mSequenceId; private Integer mOriginalFlashMode;
private SurfaceTexture mLastFrameSurfaceTexture;
private float mLastFrameScaleX;
private float mLastFrameScaleY;
private EGLContext mLastFrameScaleEGLContext;
private Long mDesiredTimestamp = null;
public Snapshot2PictureRecorder(@NonNull PictureResult.Stub stub, public Snapshot2PictureRecorder(@NonNull PictureResult.Stub stub,
@NonNull Camera2Engine engine, @NonNull Camera2Engine engine,
@ -58,95 +63,94 @@ public class Snapshot2PictureRecorder extends SnapshotGlPictureRecorder {
@NonNull CaptureRequest.Builder builder) { @NonNull CaptureRequest.Builder builder) {
super(stub, engine, preview, outputRatio); super(stub, engine, preview, outputRatio);
mEngine = engine; mEngine = engine;
mPreview = preview;
mSession = session; mSession = session;
mCallback = callback; mCallback = callback;
mBuilder = builder; mBuilder = builder;
} }
private final RendererFrameCallback mRendererCallback = new RendererFrameCallback() {
@Override
public void onRendererTextureCreated(int textureId) {
Snapshot2PictureRecorder.super.onRendererTextureCreated(textureId);
}
@Override
public void onRendererFilterChanged(@NonNull Filter filter) {
Snapshot2PictureRecorder.super.onRendererFilterChanged(filter);
}
@Override
public void onRendererFrame(@NonNull SurfaceTexture surfaceTexture, float scaleX, float scaleY) {
mLastFrameSurfaceTexture = surfaceTexture;
mLastFrameScaleX = scaleX;
mLastFrameScaleY = scaleY;
mLastFrameScaleEGLContext = EGL14.eglGetCurrentContext();
maybeTakeFrame();
}
};
@Override @Override
public void take() { public void take() {
if (!mEngine.getPictureSnapshotMetering()) { if (!mEngine.getPictureSnapshotMetering()) {
LOG.i("take:", "Engine does no metering, taking fast snapshot.");
super.take(); super.take();
return; } else {
} LOG.i("take:", "Engine does metering, caring about flash.");
mState = STATE_WAITING_FIRST_FRAME;
LOG.i("take:", "Engine does metering, adding our CONTROL_CAPTURE_INTENT."); mOriginalAeMode = mBuilder.get(CaptureRequest.CONTROL_AE_MODE);
mPreview.addRendererFrameCallback(mRendererCallback); mOriginalFlashMode = mBuilder.get(CaptureRequest.FLASH_MODE);
mState = STATE_WAITING_CAPTURE;
try {
mOldCaptureIntent = mBuilder.get(CaptureRequest.CONTROL_CAPTURE_INTENT);
mBuilder.set(CaptureRequest.CONTROL_CAPTURE_INTENT, CaptureRequest.CONTROL_CAPTURE_INTENT_STILL_CAPTURE);
mSequenceId = mSession.setRepeatingRequest(mBuilder.build(), mCallback, null);
} catch (CameraAccessException e) {
LOG.w("take:", "Camera error while applying our CONTROL_CAPTURE_INTENT", e);
mResult = null;
mError = e;
dispatchResult();
} }
} }
/**
* This works by inspecting {@link CaptureResult#CONTROL_AE_STATE}, and specifically
* the {@link CaptureRequest#CONTROL_AE_STATE_FLASH_REQUIRED} value.
* The AE state will have this value iff:
* 1. flash is required
* 2. device has a flash
* 3. we are not in Flash.OFF
* So when we see this value we just have to open the torch and wait a bit more,
* just to have a correct exposure.
*
* @param result a total result
*/
public void onCaptureCompleted(@NonNull TotalCaptureResult result) { public void onCaptureCompleted(@NonNull TotalCaptureResult result) {
LOG.i("onCaptureCompleted:", if (mState == STATE_WAITING_FIRST_FRAME) {
"aeState:", result.get(CaptureResult.CONTROL_AE_STATE), Integer aeState = result.get(CaptureResult.CONTROL_AE_STATE);
"flashState:", result.get(CaptureResult.FLASH_STATE)); if (aeState == null) {
if (mState == STATE_WAITING_CAPTURE) { LOG.w("onCaptureCompleted:", "aeState is null! This should never happen. Taking snapshot.");
if (result.getSequenceId() == mSequenceId) { mState = STATE_WAITING_IMAGE;
// This is the request we launched and we're waiting for the right frame. super.take();
LOG.w("onCaptureCompleted:", } else if (aeState == CaptureRequest.CONTROL_AE_STATE_CONVERGED) {
"aeState:", result.get(CaptureResult.CONTROL_AE_STATE), LOG.i("onCaptureCompleted:", "aeState is optimal. Taking snapshot.");
"flashState:", result.get(CaptureResult.FLASH_STATE)); mState = STATE_WAITING_IMAGE;
mState = STATE_WAITING_CORRECT_FRAME; super.take();
mDesiredTimestamp = result.get(CaptureResult.SENSOR_TIMESTAMP); } else if (aeState != CaptureRequest.CONTROL_AE_STATE_FLASH_REQUIRED) {
if (mDesiredTimestamp == null) mDesiredTimestamp = 0L; LOG.w("onCaptureCompleted:", "aeState is not CONVERGED yet not FLASH_REQUIRED.",
LOG.i("onCaptureCompleted:", "Got timestamp:", mDesiredTimestamp); "This seems to say that metering did not complete before. Let's take the snapshot.");
maybeTakeFrame(); // TODO the metering system might set AE to LOCKED. what to do in this case?
mState = STATE_WAITING_IMAGE;
super.take();
} else {
// Open the torch. AE_MODE_ON is guaranteed to be supported.
// Remove any AE lock that was set by the engine.
LOG.w("onCaptureCompleted:", "aeState is FLASH_REQUIRED. Opening torch.");
mState = STATE_WAITING_TORCH;
mBuilder.set(CaptureRequest.FLASH_MODE, CaptureRequest.FLASH_MODE_TORCH);
mBuilder.set(CaptureRequest.CONTROL_AE_MODE, CaptureRequest.CONTROL_AE_MODE_ON);
mBuilder.set(CaptureRequest.CONTROL_AE_LOCK, false);
try {
mSession.setRepeatingRequest(mBuilder.build(), mCallback, null);
} catch (CameraAccessException e) {
mResult = null;
mError = e;
dispatchResult();
}
} }
}
}
private void maybeTakeFrame() { } else if (mState == STATE_WAITING_TORCH) {
if (mState != STATE_WAITING_CORRECT_FRAME) { Integer flashState = result.get(CaptureResult.FLASH_STATE);
LOG.w("maybeTakeFrame:", "we're not waiting for a frame. Ignoring.", mState); if (flashState == null) {
return; LOG.w("onCaptureCompleted:", "Waiting flash, but flashState is null! Taking snapshot.");
} mState = STATE_WAITING_IMAGE;
if (mDesiredTimestamp == null || mLastFrameSurfaceTexture == null) { super.take();
LOG.w("maybeTakeFrame:", "either timestamp or surfaceTexture are null.", mDesiredTimestamp); } else if (flashState == CaptureResult.FLASH_STATE_FIRED) {
return; LOG.i("onCaptureCompleted:", "Waiting flash and we have FIRED state! Waiting exposure.");
} mState = STATE_WAITING_TORCH_EXPOSURE;
} else {
LOG.i("onCaptureCompleted:", "aeState is FLASH_REQUIRED but flashState is",
flashState, ". Waiting...");
}
long currentTimestamp = mLastFrameSurfaceTexture.getTimestamp(); } else if (mState == STATE_WAITING_TORCH_EXPOSURE) {
if (currentTimestamp == mDesiredTimestamp) { Integer aeState = result.get(CaptureResult.CONTROL_AE_STATE);
LOG.i("maybeTakeFrame:", "taking frame with exact timestamp:", currentTimestamp); if (aeState != null && (aeState == CaptureResult.CONTROL_AE_STATE_CONVERGED
mState = STATE_WAITING_IMAGE; || aeState == CaptureResult.CONTROL_AE_STATE_FLASH_REQUIRED)) {
takeFrame(mLastFrameSurfaceTexture, mLastFrameScaleX, mLastFrameScaleY, mLastFrameScaleEGLContext); LOG.i("onCaptureCompleted:", "We have torch and good torch exposure.", aeState);
} else if (currentTimestamp > mDesiredTimestamp) { mState = STATE_WAITING_IMAGE;
LOG.w("maybeTakeFrame:", "taking frame with some delay. Flash might not be respected."); super.take();
mState = STATE_WAITING_IMAGE; } else {
takeFrame(mLastFrameSurfaceTexture, mLastFrameScaleX, mLastFrameScaleY, mLastFrameScaleEGLContext); LOG.i("onCaptureCompleted:", "Waiting for torch exposure...:", aeState);
} else { }
LOG.i("maybeTakeFrame:", "Waiting...", mDesiredTimestamp - currentTimestamp);
} }
} }
@ -156,11 +160,13 @@ public class Snapshot2PictureRecorder extends SnapshotGlPictureRecorder {
// Revert our changes. // Revert our changes.
LOG.i("dispatchResult:", "Reverting the capture intent changes."); LOG.i("dispatchResult:", "Reverting the capture intent changes.");
try { try {
mBuilder.set(CaptureRequest.CONTROL_CAPTURE_INTENT, mOldCaptureIntent); // See Camera2Engine.setFlash(). It's better to change these one by one.
mBuilder.set(CaptureRequest.CONTROL_AE_MODE, mOriginalAeMode);
mSession.capture(mBuilder.build(), mCallback, null);
mBuilder.set(CaptureRequest.FLASH_MODE, mOriginalFlashMode);
mSession.setRepeatingRequest(mBuilder.build(), mCallback, null); mSession.setRepeatingRequest(mBuilder.build(), mCallback, null);
} catch (CameraAccessException ignore) {} } catch (CameraAccessException ignore) {}
} }
mPreview.removeRendererFrameCallback(mRendererCallback);
mState = STATE_IDLE; mState = STATE_IDLE;
super.dispatchResult(); super.dispatchResult();
} }

@ -187,13 +187,14 @@ public class GlCameraPreview extends FilterCameraPreview<GLSurfaceView, SurfaceT
@Override @Override
public void onDrawFrame(GL10 gl) { public void onDrawFrame(GL10 gl) {
if (mInputSurfaceTexture == null) return; if (mInputSurfaceTexture == null) return;
// Latch the latest frame. If there isn't anything new,
// we'll just re-use whatever was there before.
mInputSurfaceTexture.updateTexImage();
if (mInputStreamWidth <= 0 || mInputStreamHeight <= 0) { if (mInputStreamWidth <= 0 || mInputStreamHeight <= 0) {
// Skip drawing. Camera was not opened. // Skip drawing. Camera was not opened.
return; return;
} }
// Latch the latest frame. If there isn't anything new,
// we'll just re-use whatever was there before.
mInputSurfaceTexture.updateTexImage();
mInputSurfaceTexture.getTransformMatrix(mTransformMatrix); mInputSurfaceTexture.getTransformMatrix(mTransformMatrix);
LOG.v("onDrawFrame:", "timestamp:", mInputSurfaceTexture.getTimestamp()); LOG.v("onDrawFrame:", "timestamp:", mInputSurfaceTexture.getTimestamp());

Loading…
Cancel
Save