Flash support for metered snapshots

pull/580/head
Mattia Iavarone 6 years ago
parent e31c3179b5
commit 04697e917d
  1. 192
      cameraview/src/main/java/com/otaliastudios/cameraview/picture/Snapshot2PictureRecorder.java
  2. 7
      cameraview/src/main/java/com/otaliastudios/cameraview/preview/GlCameraPreview.java

@ -6,23 +6,34 @@ import android.hardware.camera2.CameraCaptureSession;
import android.hardware.camera2.CaptureRequest;
import android.hardware.camera2.CaptureResult;
import android.hardware.camera2.TotalCaptureResult;
import android.opengl.EGL14;
import android.opengl.EGLContext;
import android.os.Build;
import android.util.Log;
import androidx.annotation.NonNull;
import androidx.annotation.Nullable;
import androidx.annotation.RequiresApi;
import com.otaliastudios.cameraview.CameraLogger;
import com.otaliastudios.cameraview.PictureResult;
import com.otaliastudios.cameraview.engine.Camera2Engine;
import com.otaliastudios.cameraview.filter.Filter;
import com.otaliastudios.cameraview.preview.GlCameraPreview;
import com.otaliastudios.cameraview.preview.RendererFrameCallback;
import com.otaliastudios.cameraview.size.AspectRatio;
/**
* Wraps {@link SnapshotGlPictureRecorder} for Camera2.
*
* Camera2 engine supports metering for snapshots and we expect for them to correctly fire flash as well.
* The first idea, and in theory, the most correct one, was to set {@link CaptureRequest#CONTROL_CAPTURE_INTENT}
* to {@link CaptureRequest#CONTROL_CAPTURE_INTENT_STILL_CAPTURE}.
*
* According to documentation, this will automatically trigger the flash if parameters says so.
* In fact this is what happens, but it is a very fast flash that only lasts for 1 or 2 frames.
* It's not easy to call super.take() at the exact time so that we capture the frame that was lit.
* I have tried by comparing {@link SurfaceTexture#getTimestamp()} and {@link CaptureResult#SENSOR_TIMESTAMP}
* to identify the correct frame. These timestamps match, but the frame is not the correct one.
*
* So what we do here is ignore the {@link CaptureRequest#CONTROL_CAPTURE_INTENT} and instead open the
* torch, if requested to do so. Then wait for exposure to settle again and finally take a snapshot.
* I'd still love to use the capture intent instead of this, but was not able yet.
*/
@RequiresApi(Build.VERSION_CODES.LOLLIPOP)
public class Snapshot2PictureRecorder extends SnapshotGlPictureRecorder {
@ -30,24 +41,18 @@ public class Snapshot2PictureRecorder extends SnapshotGlPictureRecorder {
private final static CameraLogger LOG = CameraLogger.create(TAG);
private final static int STATE_IDLE = 0;
private final static int STATE_WAITING_CAPTURE = 1;
private final static int STATE_WAITING_CORRECT_FRAME = 2;
private final static int STATE_WAITING_IMAGE = 3;
private final static int STATE_WAITING_FIRST_FRAME = 1;
private final static int STATE_WAITING_TORCH = 2;
private final static int STATE_WAITING_TORCH_EXPOSURE = 3;
private final static int STATE_WAITING_IMAGE = 4;
private final Camera2Engine mEngine;
private final GlCameraPreview mPreview;
private final CameraCaptureSession mSession;
private final CameraCaptureSession.CaptureCallback mCallback;
private final CaptureRequest.Builder mBuilder;
private int mState = STATE_IDLE;
private Integer mOldCaptureIntent;
private int mSequenceId;
private SurfaceTexture mLastFrameSurfaceTexture;
private float mLastFrameScaleX;
private float mLastFrameScaleY;
private EGLContext mLastFrameScaleEGLContext;
private Long mDesiredTimestamp = null;
private Integer mOriginalAeMode;
private Integer mOriginalFlashMode;
public Snapshot2PictureRecorder(@NonNull PictureResult.Stub stub,
@NonNull Camera2Engine engine,
@ -58,95 +63,94 @@ public class Snapshot2PictureRecorder extends SnapshotGlPictureRecorder {
@NonNull CaptureRequest.Builder builder) {
super(stub, engine, preview, outputRatio);
mEngine = engine;
mPreview = preview;
mSession = session;
mCallback = callback;
mBuilder = builder;
}
private final RendererFrameCallback mRendererCallback = new RendererFrameCallback() {
@Override
public void onRendererTextureCreated(int textureId) {
Snapshot2PictureRecorder.super.onRendererTextureCreated(textureId);
}
@Override
public void onRendererFilterChanged(@NonNull Filter filter) {
Snapshot2PictureRecorder.super.onRendererFilterChanged(filter);
}
@Override
public void onRendererFrame(@NonNull SurfaceTexture surfaceTexture, float scaleX, float scaleY) {
mLastFrameSurfaceTexture = surfaceTexture;
mLastFrameScaleX = scaleX;
mLastFrameScaleY = scaleY;
mLastFrameScaleEGLContext = EGL14.eglGetCurrentContext();
maybeTakeFrame();
}
};
@Override
public void take() {
if (!mEngine.getPictureSnapshotMetering()) {
LOG.i("take:", "Engine does no metering, taking fast snapshot.");
super.take();
return;
}
LOG.i("take:", "Engine does metering, adding our CONTROL_CAPTURE_INTENT.");
mPreview.addRendererFrameCallback(mRendererCallback);
mState = STATE_WAITING_CAPTURE;
try {
mOldCaptureIntent = mBuilder.get(CaptureRequest.CONTROL_CAPTURE_INTENT);
mBuilder.set(CaptureRequest.CONTROL_CAPTURE_INTENT, CaptureRequest.CONTROL_CAPTURE_INTENT_STILL_CAPTURE);
mSequenceId = mSession.setRepeatingRequest(mBuilder.build(), mCallback, null);
} catch (CameraAccessException e) {
LOG.w("take:", "Camera error while applying our CONTROL_CAPTURE_INTENT", e);
mResult = null;
mError = e;
dispatchResult();
} else {
LOG.i("take:", "Engine does metering, caring about flash.");
mState = STATE_WAITING_FIRST_FRAME;
mOriginalAeMode = mBuilder.get(CaptureRequest.CONTROL_AE_MODE);
mOriginalFlashMode = mBuilder.get(CaptureRequest.FLASH_MODE);
}
}
/**
* This works by inspecting {@link CaptureResult#CONTROL_AE_STATE}, and specifically
* the {@link CaptureRequest#CONTROL_AE_STATE_FLASH_REQUIRED} value.
* The AE state will have this value iff:
* 1. flash is required
* 2. device has a flash
* 3. we are not in Flash.OFF
* So when we see this value we just have to open the torch and wait a bit more,
* just to have a correct exposure.
*
* @param result a total result
*/
public void onCaptureCompleted(@NonNull TotalCaptureResult result) {
LOG.i("onCaptureCompleted:",
"aeState:", result.get(CaptureResult.CONTROL_AE_STATE),
"flashState:", result.get(CaptureResult.FLASH_STATE));
if (mState == STATE_WAITING_CAPTURE) {
if (result.getSequenceId() == mSequenceId) {
// This is the request we launched and we're waiting for the right frame.
LOG.w("onCaptureCompleted:",
"aeState:", result.get(CaptureResult.CONTROL_AE_STATE),
"flashState:", result.get(CaptureResult.FLASH_STATE));
mState = STATE_WAITING_CORRECT_FRAME;
mDesiredTimestamp = result.get(CaptureResult.SENSOR_TIMESTAMP);
if (mDesiredTimestamp == null) mDesiredTimestamp = 0L;
LOG.i("onCaptureCompleted:", "Got timestamp:", mDesiredTimestamp);
maybeTakeFrame();
if (mState == STATE_WAITING_FIRST_FRAME) {
Integer aeState = result.get(CaptureResult.CONTROL_AE_STATE);
if (aeState == null) {
LOG.w("onCaptureCompleted:", "aeState is null! This should never happen. Taking snapshot.");
mState = STATE_WAITING_IMAGE;
super.take();
} else if (aeState == CaptureRequest.CONTROL_AE_STATE_CONVERGED) {
LOG.i("onCaptureCompleted:", "aeState is optimal. Taking snapshot.");
mState = STATE_WAITING_IMAGE;
super.take();
} else if (aeState != CaptureRequest.CONTROL_AE_STATE_FLASH_REQUIRED) {
LOG.w("onCaptureCompleted:", "aeState is not CONVERGED yet not FLASH_REQUIRED.",
"This seems to say that metering did not complete before. Let's take the snapshot.");
// TODO the metering system might set AE to LOCKED. what to do in this case?
mState = STATE_WAITING_IMAGE;
super.take();
} else {
// Open the torch. AE_MODE_ON is guaranteed to be supported.
// Remove any AE lock that was set by the engine.
LOG.w("onCaptureCompleted:", "aeState is FLASH_REQUIRED. Opening torch.");
mState = STATE_WAITING_TORCH;
mBuilder.set(CaptureRequest.FLASH_MODE, CaptureRequest.FLASH_MODE_TORCH);
mBuilder.set(CaptureRequest.CONTROL_AE_MODE, CaptureRequest.CONTROL_AE_MODE_ON);
mBuilder.set(CaptureRequest.CONTROL_AE_LOCK, false);
try {
mSession.setRepeatingRequest(mBuilder.build(), mCallback, null);
} catch (CameraAccessException e) {
mResult = null;
mError = e;
dispatchResult();
}
}
}
}
private void maybeTakeFrame() {
if (mState != STATE_WAITING_CORRECT_FRAME) {
LOG.w("maybeTakeFrame:", "we're not waiting for a frame. Ignoring.", mState);
return;
}
if (mDesiredTimestamp == null || mLastFrameSurfaceTexture == null) {
LOG.w("maybeTakeFrame:", "either timestamp or surfaceTexture are null.", mDesiredTimestamp);
return;
}
} else if (mState == STATE_WAITING_TORCH) {
Integer flashState = result.get(CaptureResult.FLASH_STATE);
if (flashState == null) {
LOG.w("onCaptureCompleted:", "Waiting flash, but flashState is null! Taking snapshot.");
mState = STATE_WAITING_IMAGE;
super.take();
} else if (flashState == CaptureResult.FLASH_STATE_FIRED) {
LOG.i("onCaptureCompleted:", "Waiting flash and we have FIRED state! Waiting exposure.");
mState = STATE_WAITING_TORCH_EXPOSURE;
} else {
LOG.i("onCaptureCompleted:", "aeState is FLASH_REQUIRED but flashState is",
flashState, ". Waiting...");
}
long currentTimestamp = mLastFrameSurfaceTexture.getTimestamp();
if (currentTimestamp == mDesiredTimestamp) {
LOG.i("maybeTakeFrame:", "taking frame with exact timestamp:", currentTimestamp);
mState = STATE_WAITING_IMAGE;
takeFrame(mLastFrameSurfaceTexture, mLastFrameScaleX, mLastFrameScaleY, mLastFrameScaleEGLContext);
} else if (currentTimestamp > mDesiredTimestamp) {
LOG.w("maybeTakeFrame:", "taking frame with some delay. Flash might not be respected.");
mState = STATE_WAITING_IMAGE;
takeFrame(mLastFrameSurfaceTexture, mLastFrameScaleX, mLastFrameScaleY, mLastFrameScaleEGLContext);
} else {
LOG.i("maybeTakeFrame:", "Waiting...", mDesiredTimestamp - currentTimestamp);
} else if (mState == STATE_WAITING_TORCH_EXPOSURE) {
Integer aeState = result.get(CaptureResult.CONTROL_AE_STATE);
if (aeState != null && (aeState == CaptureResult.CONTROL_AE_STATE_CONVERGED
|| aeState == CaptureResult.CONTROL_AE_STATE_FLASH_REQUIRED)) {
LOG.i("onCaptureCompleted:", "We have torch and good torch exposure.", aeState);
mState = STATE_WAITING_IMAGE;
super.take();
} else {
LOG.i("onCaptureCompleted:", "Waiting for torch exposure...:", aeState);
}
}
}
@ -156,11 +160,13 @@ public class Snapshot2PictureRecorder extends SnapshotGlPictureRecorder {
// Revert our changes.
LOG.i("dispatchResult:", "Reverting the capture intent changes.");
try {
mBuilder.set(CaptureRequest.CONTROL_CAPTURE_INTENT, mOldCaptureIntent);
// See Camera2Engine.setFlash(). It's better to change these one by one.
mBuilder.set(CaptureRequest.CONTROL_AE_MODE, mOriginalAeMode);
mSession.capture(mBuilder.build(), mCallback, null);
mBuilder.set(CaptureRequest.FLASH_MODE, mOriginalFlashMode);
mSession.setRepeatingRequest(mBuilder.build(), mCallback, null);
} catch (CameraAccessException ignore) {}
}
mPreview.removeRendererFrameCallback(mRendererCallback);
mState = STATE_IDLE;
super.dispatchResult();
}

@ -187,13 +187,14 @@ public class GlCameraPreview extends FilterCameraPreview<GLSurfaceView, SurfaceT
@Override
public void onDrawFrame(GL10 gl) {
if (mInputSurfaceTexture == null) return;
// Latch the latest frame. If there isn't anything new,
// we'll just re-use whatever was there before.
mInputSurfaceTexture.updateTexImage();
if (mInputStreamWidth <= 0 || mInputStreamHeight <= 0) {
// Skip drawing. Camera was not opened.
return;
}
// Latch the latest frame. If there isn't anything new,
// we'll just re-use whatever was there before.
mInputSurfaceTexture.updateTexImage();
mInputSurfaceTexture.getTransformMatrix(mTransformMatrix);
LOG.v("onDrawFrame:", "timestamp:", mInputSurfaceTexture.getTimestamp());

Loading…
Cancel
Save