From bba69e7438eb2d113b3acf90a1cb10a8f9a49074 Mon Sep 17 00:00:00 2001 From: Mattia Iavarone Date: Thu, 29 Aug 2019 21:24:41 +0200 Subject: [PATCH] Rearrange code into MeteringParameters objects --- .../cameraview/engine/Camera2Engine.java | 353 ++++-------------- .../cameraview/engine/Meter.java | 315 ++++++++++++++++ .../engine/metering/AutoExposure.java | 95 +++++ .../cameraview/engine/metering/AutoFocus.java | 82 ++++ .../engine/metering/AutoWhiteBalance.java | 80 ++++ .../engine/metering/MeteringParameter.java | 52 +++ 6 files changed, 689 insertions(+), 288 deletions(-) create mode 100644 cameraview/src/main/java/com/otaliastudios/cameraview/engine/Meter.java create mode 100644 cameraview/src/main/java/com/otaliastudios/cameraview/engine/metering/AutoExposure.java create mode 100644 cameraview/src/main/java/com/otaliastudios/cameraview/engine/metering/AutoFocus.java create mode 100644 cameraview/src/main/java/com/otaliastudios/cameraview/engine/metering/AutoWhiteBalance.java create mode 100644 cameraview/src/main/java/com/otaliastudios/cameraview/engine/metering/MeteringParameter.java diff --git a/cameraview/src/main/java/com/otaliastudios/cameraview/engine/Camera2Engine.java b/cameraview/src/main/java/com/otaliastudios/cameraview/engine/Camera2Engine.java index 0cd1eb20..b9048f8e 100644 --- a/cameraview/src/main/java/com/otaliastudios/cameraview/engine/Camera2Engine.java +++ b/cameraview/src/main/java/com/otaliastudios/cameraview/engine/Camera2Engine.java @@ -14,7 +14,6 @@ import android.hardware.camera2.CameraManager; import android.hardware.camera2.CaptureRequest; import android.hardware.camera2.CaptureResult; import android.hardware.camera2.TotalCaptureResult; -import android.hardware.camera2.params.MeteringRectangle; import android.hardware.camera2.params.StreamConfigurationMap; import android.location.Location; import android.media.Image; @@ -61,14 +60,13 @@ import com.otaliastudios.cameraview.video.Full2VideoRecorder; import com.otaliastudios.cameraview.video.SnapshotVideoRecorder; import java.util.ArrayList; -import java.util.Arrays; import java.util.List; import java.util.concurrent.Callable; import java.util.concurrent.ExecutionException; import java.util.concurrent.atomic.AtomicBoolean; @RequiresApi(Build.VERSION_CODES.LOLLIPOP) -public class Camera2Engine extends CameraEngine implements ImageReader.OnImageAvailableListener { +public class Camera2Engine extends CameraEngine implements ImageReader.OnImageAvailableListener, Meter.Callback { private static final String TAG = Camera2Engine.class.getSimpleName(); private static final CameraLogger LOG = CameraLogger.create(TAG); @@ -101,14 +99,8 @@ public class Camera2Engine extends CameraEngine implements ImageReader.OnImageAv private ImageReader mPictureReader; private final boolean mPictureCaptureStopsPreview = false; // can make configurable at some point - // 3A Metering - private PointF mMeteringPoint; - private Gesture mMeteringGesture; - private boolean mMeteringAEDone; - private boolean mMeteringAEStarted; - private boolean mMeteringAFDone; - private boolean mMeteringAFSuccess; - private boolean mMeteringAWBDone; + // 3A metering + private Meter mMeter; public Camera2Engine(Callback callback) { super(callback); @@ -239,7 +231,9 @@ public class Camera2Engine extends CameraEngine implements ImageReader.OnImageAv if (mPictureRecorder instanceof Full2PictureRecorder) { ((Full2PictureRecorder) mPictureRecorder).onCaptureProgressed(partialResult); } - if (isMetering()) onMeteringCapture(partialResult); + if (mMeter != null && mMeter.isMetering()) { + mMeter.onCapture(partialResult); + } } @Override @@ -248,7 +242,9 @@ public class Camera2Engine extends CameraEngine implements ImageReader.OnImageAv if (mPictureRecorder instanceof Full2PictureRecorder) { ((Full2PictureRecorder) mPictureRecorder).onCaptureCompleted(result); } - if (isMetering()) onMeteringCapture(result); + if (mMeter != null && mMeter.isMetering()) { + mMeter.onCapture(result); + } } }; @@ -559,8 +555,7 @@ public class Camera2Engine extends CameraEngine implements ImageReader.OnImageAv } removeRepeatingRequestBuilderSurfaces(); mRepeatingRequest = null; - mMeteringPoint = null; - mMeteringGesture = null; + mMeter = null; LOG.i("onStopPreview:", "Returning."); return Tasks.forResult(null); } @@ -1138,302 +1133,84 @@ public class Camera2Engine extends CameraEngine implements ImageReader.OnImageAv LOG.i("startMetering", "executing. Preview state:", getPreviewState()); // This will only work when we have a preview, since it launches the preview in the end. // Even without this it would need the bind state at least, since we need the preview size. - if (!mCameraOptions.isAutoFocusSupported()) return; if (getPreviewState() < STATE_STARTED) return; - mMeteringPoint = point; - mMeteringGesture = gesture; - - // This is a good Q/A. https://stackoverflow.com/a/33181620/4288782 - // At first, the point is relative to the View system and does not account our own cropping. - // Will keep updating these two below. - PointF referencePoint = new PointF(point.x, point.y); - Size referenceSize /* = previewSurfaceSize */; - - // 1. Account for cropping. - Size previewStreamSize = getPreviewStreamSize(Reference.VIEW); - Size previewSurfaceSize = mPreview.getSurfaceSize(); - if (previewStreamSize == null) throw new IllegalStateException("getPreviewStreamSize should not be null at this point."); - AspectRatio previewStreamAspectRatio = AspectRatio.of(previewStreamSize); - AspectRatio previewSurfaceAspectRatio = AspectRatio.of(previewSurfaceSize); - if (mPreview.isCropping()) { - if (previewStreamAspectRatio.toFloat() > previewSurfaceAspectRatio.toFloat()) { - // Stream is larger. The x coordinate must be increased: a touch on the left side - // of the surface is not on the left size of stream (it's more to the right). - float scale = previewStreamAspectRatio.toFloat() / previewSurfaceAspectRatio.toFloat(); - referencePoint.x += previewSurfaceSize.getWidth() * (scale - 1F) / 2F; - - } else { - // Stream is taller. The y coordinate must be increased: a touch on the top side - // of the surface is not on the top size of stream (it's a bit lower). - float scale = previewSurfaceAspectRatio.toFloat() / previewStreamAspectRatio.toFloat(); - referencePoint.x += previewSurfaceSize.getHeight() * (scale - 1F) / 2F; - } - } - // 2. Scale to the stream coordinates (not the surface). - referencePoint.x *= (float) previewStreamSize.getWidth() / previewSurfaceSize.getWidth(); - referencePoint.y *= (float) previewStreamSize.getHeight() / previewSurfaceSize.getHeight(); - referenceSize = previewStreamSize; - - // 3. Rotate to the stream coordinate system. - // Not elegant, but the sin/cos way was failing. - int angle = getAngles().offset(Reference.SENSOR, Reference.VIEW, Axis.ABSOLUTE); - boolean flip = angle % 180 != 0; - float tempX = referencePoint.x; float tempY = referencePoint.y; - if (angle == 0) { - referencePoint.x = tempX; - referencePoint.y = tempY; - } else if (angle == 90) { - //noinspection SuspiciousNameCombination - referencePoint.x = tempY; - referencePoint.y = referenceSize.getWidth() - tempX; - } else if (angle == 180) { - referencePoint.x = referenceSize.getWidth() - tempX; - referencePoint.y = referenceSize.getHeight() - tempY; - } else if (angle == 270) { - referencePoint.x = referenceSize.getHeight() - tempY; - //noinspection SuspiciousNameCombination - referencePoint.y = tempX; - } else { - throw new IllegalStateException("Unexpected angle " + angle); - } - referenceSize = flip ? referenceSize.flip() : referenceSize; - - // These points are now referencing the stream rect on the sensor array. - // But we still have to figure out how the stream rect is laid on the sensor array. - // https://source.android.com/devices/camera/camera3_crop_reprocess.html - // For sanity, let's assume it is centered. - // For sanity, let's also assume that the crop region is equal to the stream region. - - // 4. Move to the active sensor array coordinate system. - Rect activeRect = readCharacteristic(CameraCharacteristics.SENSOR_INFO_ACTIVE_ARRAY_SIZE, - new Rect(0, 0, referenceSize.getWidth(), referenceSize.getHeight())); - referencePoint.x += (activeRect.width() - referenceSize.getWidth()) / 2F; - referencePoint.y += (activeRect.height() - referenceSize.getHeight()) / 2F; - referenceSize = new Size(activeRect.width(), activeRect.height()); - - // 5. Account for zoom! This only works for mZoomValue = 0. - // We must scale down with respect to the reference size center. If mZoomValue = 1, - // This must leave everything unchanged. - float maxZoom = readCharacteristic(CameraCharacteristics.SCALER_AVAILABLE_MAX_DIGITAL_ZOOM, - 1F /* no zoom */); - float currZoom = 1 + mZoomValue * (maxZoom - 1); // 1 ... maxZoom - float currReduction = 1 / currZoom; - float referenceCenterX = referenceSize.getWidth() / 2F; - float referenceCenterY = referenceSize.getHeight() / 2F; - referencePoint.x = referenceCenterX + currReduction * (referencePoint.x - referenceCenterX); - referencePoint.y = referenceCenterY + currReduction * (referencePoint.y - referenceCenterY); - - // 6. NOW we can compute the metering regions. - float visibleWidth = referenceSize.getWidth() * currReduction; - float visibleHeight = referenceSize.getHeight() * currReduction; - MeteringRectangle area1 = createMeteringRectangle(referencePoint, referenceSize, visibleWidth, visibleHeight, 0.05F, 1000); - MeteringRectangle area2 = createMeteringRectangle(referencePoint, referenceSize, visibleWidth, visibleHeight, 0.1F, 100); - - // 7. And finally dispatch them... - List areas = Arrays.asList(area1, area2); - int maxReagionsAf = readCharacteristic(CameraCharacteristics.CONTROL_MAX_REGIONS_AF, 0); - int maxReagionsAe = readCharacteristic(CameraCharacteristics.CONTROL_MAX_REGIONS_AE, 0); - int maxReagionsAwb = readCharacteristic(CameraCharacteristics.CONTROL_MAX_REGIONS_AWB, 0); - if (maxReagionsAf > 0) { - int max = Math.min(maxReagionsAf, areas.size()); - mRepeatingRequestBuilder.set(CaptureRequest.CONTROL_AF_REGIONS, - areas.subList(0, max).toArray(new MeteringRectangle[]{})); - } - if (maxReagionsAe > 0) { - int max = Math.min(maxReagionsAe, areas.size()); - mRepeatingRequestBuilder.set(CaptureRequest.CONTROL_AE_REGIONS, - areas.subList(0, max).toArray(new MeteringRectangle[]{})); - } - if (maxReagionsAwb > 0) { - int max = Math.min(maxReagionsAwb, areas.size()); - mRepeatingRequestBuilder.set(CaptureRequest.CONTROL_AWB_REGIONS, - areas.subList(0, max).toArray(new MeteringRectangle[]{})); + // Reset the old meter if present. + if (mMeter != null) { + mMeter.resetMetering(); } - // 8. Set AF mode to AUTO so it doesn't use the CONTINUOUS schedule. - // When this ends, we will reset everything. We know CONTROL_AF_MODE_AUTO is available - // because we have called cameraOptions.isAutoFocusSupported(). - mCallback.dispatchOnFocusStart(gesture, point); - // AF - mRepeatingRequestBuilder.set(CaptureRequest.CONTROL_AF_MODE, CaptureRequest.CONTROL_AF_MODE_AUTO); - mRepeatingRequestBuilder.set(CaptureRequest.CONTROL_AF_TRIGGER, CaptureRequest.CONTROL_AF_TRIGGER_START); - mMeteringAFDone = false; - // AE - boolean isNotLegacy = readCharacteristic(CameraCharacteristics.INFO_SUPPORTED_HARDWARE_LEVEL, -1) != - CameraCharacteristics.INFO_SUPPORTED_HARDWARE_LEVEL_LEGACY; - Integer aeMode = mRepeatingRequestBuilder.get(CaptureRequest.CONTROL_AE_MODE); - boolean isAEOn = aeMode != null && - (aeMode == CameraCharacteristics.CONTROL_AE_MODE_ON - || aeMode == CameraCharacteristics.CONTROL_AE_MODE_ON_ALWAYS_FLASH - || aeMode == CameraCharacteristics.CONTROL_AE_MODE_ON_AUTO_FLASH - || aeMode == CameraCharacteristics.CONTROL_AE_MODE_ON_AUTO_FLASH_REDEYE); - boolean supportsAE = isNotLegacy && isAEOn; - if (supportsAE) { - mRepeatingRequestBuilder.set(CaptureRequest.CONTROL_AE_PRECAPTURE_TRIGGER, - CaptureRequest.CONTROL_AE_PRECAPTURE_TRIGGER_START); - } - mMeteringAEStarted = false; - mMeteringAEDone = !supportsAE; // If supported, we're not done. - // AWB - Integer awbMode = mRepeatingRequestBuilder.get(CaptureRequest.CONTROL_AWB_MODE); - boolean supportsAWB = isNotLegacy && awbMode != null && awbMode == CaptureRequest.CONTROL_AWB_MODE_AUTO; - mMeteringAWBDone = !supportsAWB; // legacy devices do not have the awb state - if (supportsAWB) { - // Remove any lock. We're not setting any, but just in case. - mRepeatingRequestBuilder.set(CaptureRequest.CONTROL_AWB_LOCK, false); - } - - // 9. Apply everything. - applyRepeatingRequestBuilder(); + // The meter will check the current state to see if AF/AE/AWB should be run. + // - AE should be on CONTROL_AE_MODE_ON* (depends on setFlash()) + // - AWB should be on CONTROL_AWB_MODE_AUTO (depends on setWhiteBalance()) + // - AF should be on CONTROL_AF_MODE_AUTO + // The last one depends on us because the library has no focus API and we have + // just been asked to fo auto focus. So let's do this. This operation is reverted + // during onMeteringReset(). + if (!mCameraOptions.isAutoFocusSupported()) return; + mRepeatingRequestBuilder.set( + CaptureRequest.CONTROL_AF_MODE, + CaptureRequest.CONTROL_AF_MODE_AUTO); + + // Create the meter and start. + mMeter = new Meter(Camera2Engine.this, + mRepeatingRequestBuilder, + mCameraCharacteristics, + Camera2Engine.this); + mMeter.startMetering(point, gesture); } }); } /** - * Creates a metering rectangle around the center point. - * The rectangle will have a size that's a factor of the visible width and height. - * The rectangle will also be constrained to be inside the given boundaries, - * so we don't exceed them in case the center point is exactly on one side for example. - * @return a new rectangle + * Called by {@link Meter} when the metering process has started. + * We are currently exposing an auto focus API so that's what we dispatch. + * @param point point + * @param gesture gesture */ - @NonNull - private MeteringRectangle createMeteringRectangle( - @NonNull PointF center, @NonNull Size boundaries, - float visibleWidth, float visibleHeight, - float factor, int weight) { - float halfWidth = factor * visibleWidth / 2F; - float halfHeight = factor * visibleHeight / 2F; - return new MeteringRectangle( - (int) Math.max(0, center.x - halfWidth), - (int) Math.max(0, center.y - halfHeight), - (int) Math.min(boundaries.getWidth(), halfWidth * 2F), - (int) Math.min(boundaries.getHeight(), halfHeight * 2F), - weight - ); + @Override + public void onMeteringStarted(@NonNull PointF point, @Nullable Gesture gesture) { + LOG.w("onMeteringStarted - point:", point, "gesture:", gesture); + mCallback.dispatchOnFocusStart(gesture, point); + applyRepeatingRequestBuilder(); } /** - * Whether we are in a metering operation, which means, among other things, that - * {@link CaptureResult#CONTROL_AF_MODE} is set to {@link CaptureResult#CONTROL_AF_MODE_AUTO}. - * @return true if we're in a metering operation + * Called by {@link Meter} when the metering process has ended. + * We are currently exposing an auto focus API so that's what we dispatch. + * @param point point + * @param gesture gesture + * @param success success */ - private boolean isMetering() { - return mMeteringPoint != null; + @Override + public void onMeteringEnd(@NonNull PointF point, @Nullable Gesture gesture, boolean success) { + LOG.w("onMeteringEnd - point:", point, "gesture:", gesture, "success:", success); + mCallback.dispatchOnFocusEnd(gesture, success, point); } /** - * If this is called, we're in 3A metering. - * @param result the result + * When metering is reset, we're not sure that the engine is still alive. + * We should check this here. + * @param point point + * @param gesture gesture + * @return true if metering can be reset */ - private void onMeteringCapture(@NonNull CaptureResult result) { - checkMeteringAutoFocus(result); - checkMeteringAutoExposure(result); - checkMeteringAutoWhiteBalance(result); - if (mMeteringAFDone && mMeteringAEDone && mMeteringAWBDone) { - // Use the AF success for dispatching the callback, since the public - // callback is currently related to AF. - onMeteringEnd(mMeteringAFSuccess); - } - } - - private void checkMeteringAutoFocus(@NonNull CaptureResult result) { - if (mMeteringAFDone || !(result instanceof TotalCaptureResult)) return; - Integer afState = result.get(CaptureResult.CONTROL_AF_STATE); - LOG.i("checkMeteringAutoFocus:", "afState:", afState); - if (afState == null) return; - - switch (afState) { - case CaptureRequest.CONTROL_AF_STATE_FOCUSED_LOCKED: { - mMeteringAFDone = true; - mMeteringAFSuccess = true; - break; - } - case CaptureRequest.CONTROL_AF_STATE_NOT_FOCUSED_LOCKED: { - mMeteringAFDone = true; - mMeteringAFSuccess = false; - break; - } - case CaptureRequest.CONTROL_AF_STATE_INACTIVE: break; - case CaptureRequest.CONTROL_AF_STATE_ACTIVE_SCAN: break; - default: break; - } - } - - private void checkMeteringAutoExposure(@NonNull CaptureResult result) { - if (mMeteringAEDone || !(result instanceof TotalCaptureResult)) return; - Integer aeState = result.get(CaptureResult.CONTROL_AE_STATE); - LOG.i("checkMeteringAutoExposure:", "aeState:", aeState); - if (aeState == null) return; - - if (!mMeteringAEStarted) { - if (aeState == CaptureRequest.CONTROL_AE_STATE_PRECAPTURE) { - mMeteringAEStarted = true; - } - } else { - if (aeState == CaptureRequest.CONTROL_AE_STATE_CONVERGED - || aeState == CaptureRequest.CONTROL_AE_STATE_FLASH_REQUIRED) { - mMeteringAEDone = true; - } - } - } - - private void checkMeteringAutoWhiteBalance(@NonNull CaptureResult result) { - if (mMeteringAWBDone || !(result instanceof TotalCaptureResult)) return; - Integer awbState = result.get(CaptureResult.CONTROL_AWB_STATE); - LOG.i("checkMeteringAutoWhiteBalance:", "awbState:", awbState); - if (awbState == null) return; - - switch (awbState) { - case CaptureRequest.CONTROL_AWB_STATE_CONVERGED: { - mMeteringAWBDone = true; - break; - } - case CaptureRequest.CONTROL_AWB_STATE_LOCKED: break; - case CaptureRequest.CONTROL_AWB_STATE_INACTIVE: break; - case CaptureRequest.CONTROL_AWB_STATE_SEARCHING: break; - default: break; - } + @Override + public boolean canResetMetering(@NonNull PointF point, @Nullable Gesture gesture) { + return getEngineState() == STATE_STARTED; } /** - * Called by {@link #onMeteringCapture(CaptureResult)} when we detect that the - * auto focus operataion has ended. - * @param success true if success + * Called by {@link Meter} after resetting the metering parameters. + * We should apply them, and also go back to default focus. + * @param point point + * @param gesture gesture */ - private void onMeteringEnd(boolean success) { - LOG.w("onMeteringEnd - success:", success); - Gesture gesture = mMeteringGesture; - PointF point = mMeteringPoint; - mMeteringGesture = null; - mMeteringPoint = null; - if (point == null) return; - mCallback.dispatchOnFocusEnd(gesture, success, point); - mHandler.remove(mMeteringResetRunnable); - if (shouldResetAutoFocus()) { - mHandler.post(getAutoFocusResetDelay(), mMeteringResetRunnable); - } + @Override + public void onMeteringReset(@NonNull PointF point, @Nullable Gesture gesture) { + applyDefaultFocus(mRepeatingRequestBuilder); + applyRepeatingRequestBuilder(); // only if preview started already } - private Runnable mMeteringResetRunnable = new Runnable() { - @Override - public void run() { - if (getEngineState() < STATE_STARTED) return; - LOG.i("Running the 3A metering reset runnable."); - Rect whole = readCharacteristic(CameraCharacteristics.SENSOR_INFO_ACTIVE_ARRAY_SIZE, new Rect()); - MeteringRectangle[] rectangle = new MeteringRectangle[]{new MeteringRectangle(whole, MeteringRectangle.METERING_WEIGHT_DONT_CARE)}; - int maxRegionsAf = readCharacteristic(CameraCharacteristics.CONTROL_MAX_REGIONS_AF, 0); - int maxRegionsAe = readCharacteristic(CameraCharacteristics.CONTROL_MAX_REGIONS_AE, 0); - int maxRegionsAwb = readCharacteristic(CameraCharacteristics.CONTROL_MAX_REGIONS_AWB, 0); - if (maxRegionsAf > 0) mRepeatingRequestBuilder.set(CaptureRequest.CONTROL_AF_REGIONS, rectangle); - if (maxRegionsAe > 0) mRepeatingRequestBuilder.set(CaptureRequest.CONTROL_AE_REGIONS, rectangle); - if (maxRegionsAwb > 0) mRepeatingRequestBuilder.set(CaptureRequest.CONTROL_AWB_REGIONS, rectangle); - // mRepeatingRequestBuilder.set(CaptureRequest.CONTROL_AF_TRIGGER, CaptureRequest.CONTROL_AF_TRIGGER_CANCEL); - // mRepeatingRequestBuilder.set(CaptureRequest.CONTROL_AE_PRECAPTURE_TRIGGER, CaptureRequest.CONTROL_AE_PRECAPTURE_TRIGGER_CANCEL); - applyDefaultFocus(mRepeatingRequestBuilder); - applyRepeatingRequestBuilder(); // only if preview started already - } - }; - //endregion } \ No newline at end of file diff --git a/cameraview/src/main/java/com/otaliastudios/cameraview/engine/Meter.java b/cameraview/src/main/java/com/otaliastudios/cameraview/engine/Meter.java new file mode 100644 index 00000000..326ead54 --- /dev/null +++ b/cameraview/src/main/java/com/otaliastudios/cameraview/engine/Meter.java @@ -0,0 +1,315 @@ +package com.otaliastudios.cameraview.engine; + +import android.graphics.PointF; +import android.graphics.Rect; +import android.hardware.camera2.CameraCharacteristics; +import android.hardware.camera2.CaptureRequest; +import android.hardware.camera2.CaptureResult; +import android.hardware.camera2.TotalCaptureResult; +import android.hardware.camera2.params.MeteringRectangle; +import android.os.Build; + +import androidx.annotation.NonNull; +import androidx.annotation.Nullable; +import androidx.annotation.RequiresApi; + +import com.otaliastudios.cameraview.CameraLogger; +import com.otaliastudios.cameraview.engine.metering.AutoExposure; +import com.otaliastudios.cameraview.engine.metering.AutoFocus; +import com.otaliastudios.cameraview.engine.metering.AutoWhiteBalance; +import com.otaliastudios.cameraview.engine.metering.MeteringParameter; +import com.otaliastudios.cameraview.engine.offset.Axis; +import com.otaliastudios.cameraview.engine.offset.Reference; +import com.otaliastudios.cameraview.gesture.Gesture; +import com.otaliastudios.cameraview.size.AspectRatio; +import com.otaliastudios.cameraview.size.Size; + +import java.util.Arrays; +import java.util.List; + +/** + * Helps Camera2-based engines to perform 3A (auto focus, auto exposure and auto white balance) + * metering. Users are required to: + * + * - Call {@link #startMetering(PointF, Gesture)} to start + * - Call {@link #onCapture(CaptureResult)} when they have partial or total results, as long as the + * meter is still in a metering operation, which can be checked through {@link #isMetering()} + * - Call {@link #resetMetering()} to reset the metering parameters if needed. This is done automatically + * by the meter based on the reset delay configuration in the engine, but can be called explicitly + * for example when we have multiple meter requests and want to cancel the old one. + */ +@RequiresApi(Build.VERSION_CODES.LOLLIPOP) +public class Meter { + + /** + * The meter callback. + */ + public interface Callback { + + /** + * Notifies that metering has started. At this point implementors should apply + * the builder onto the preview. + * @param point point + * @param gesture gesture + */ + void onMeteringStarted(@NonNull PointF point, @Nullable Gesture gesture); + + /** + * Notifies that metering has ended. No action is required for implementors. + * From now on, {@link #isMetering()} will return false so the meter should not + * be passed capture results anymore. + * @param point point + * @param gesture gesture + * @param success success + */ + void onMeteringEnd(@NonNull PointF point, @Nullable Gesture gesture, boolean success); + + /** + * Notifies that metering has been reset. From now on, this meter instance + * is done, although in theory it could be reused by calling + * {@link #startMetering(PointF, Gesture)} again. + * @param point point + * @param gesture gesture + */ + void onMeteringReset(@NonNull PointF point, @Nullable Gesture gesture); + + /** + * Whether metering can be reset. Since it happens at a future time, this should + * return true if the engine is still in a legit state for this operation. + * @param point point + * @param gesture gesture + * @return true if can reset + */ + // TODO is this useful? engine could do its checks onMeteringReset() + boolean canResetMetering(@NonNull PointF point, @Nullable Gesture gesture); + } + + private static final String TAG = Meter.class.getSimpleName(); + private static final CameraLogger LOG = CameraLogger.create(TAG); + + private final CameraEngine mEngine; + private final CaptureRequest.Builder mBuilder; + private final CameraCharacteristics mCharacteristics; + private final Callback mCallback; + private PointF mPoint; + private Gesture mGesture; + + private boolean mIsMetering; + private MeteringParameter mAutoFocus = new AutoFocus(); + private MeteringParameter mAutoWhiteBalance = new AutoWhiteBalance(); + private MeteringParameter mAutoExposure = new AutoExposure(); + + /** + * Creates a new meter. + * @param engine the engine + * @param builder a capture builder + * @param characteristics the camera characteristics + * @param callback the callback + */ + @SuppressWarnings("WeakerAccess") + public Meter(@NonNull CameraEngine engine, + @NonNull CaptureRequest.Builder builder, + @NonNull CameraCharacteristics characteristics, + @NonNull Callback callback) { + mEngine = engine; + mBuilder = builder; + mCharacteristics = characteristics; + mCallback = callback; + } + + @NonNull + private T readCharacteristic(@NonNull CameraCharacteristics.Key key, @NonNull T fallback) { + T value = mCharacteristics.get(key); + return value == null ? fallback : value; + } + + /** + * Starts a metering sequence. + * @param point point + * @param gesture gesture + */ + @SuppressWarnings("WeakerAccess") + public void startMetering(@NonNull PointF point, @Nullable Gesture gesture) { + mPoint = point; + mGesture = gesture; + mIsMetering = true; + + // This is a good Q/A. https://stackoverflow.com/a/33181620/4288782 + // At first, the point is relative to the View system and does not account our own cropping. + // Will keep updating these two below. + PointF referencePoint = new PointF(mPoint.x, mPoint.y); + Size referenceSize /* = previewSurfaceSize */; + + // 1. Account for cropping. + Size previewStreamSize = mEngine.getPreviewStreamSize(Reference.VIEW); + Size previewSurfaceSize = mEngine.mPreview.getSurfaceSize(); + if (previewStreamSize == null) throw new IllegalStateException("getPreviewStreamSize should not be null at this point."); + AspectRatio previewStreamAspectRatio = AspectRatio.of(previewStreamSize); + AspectRatio previewSurfaceAspectRatio = AspectRatio.of(previewSurfaceSize); + if (mEngine.mPreview.isCropping()) { + if (previewStreamAspectRatio.toFloat() > previewSurfaceAspectRatio.toFloat()) { + // Stream is larger. The x coordinate must be increased: a touch on the left side + // of the surface is not on the left size of stream (it's more to the right). + float scale = previewStreamAspectRatio.toFloat() / previewSurfaceAspectRatio.toFloat(); + referencePoint.x += previewSurfaceSize.getWidth() * (scale - 1F) / 2F; + + } else { + // Stream is taller. The y coordinate must be increased: a touch on the top side + // of the surface is not on the top size of stream (it's a bit lower). + float scale = previewSurfaceAspectRatio.toFloat() / previewStreamAspectRatio.toFloat(); + referencePoint.x += previewSurfaceSize.getHeight() * (scale - 1F) / 2F; + } + } + + // 2. Scale to the stream coordinates (not the surface). + referencePoint.x *= (float) previewStreamSize.getWidth() / previewSurfaceSize.getWidth(); + referencePoint.y *= (float) previewStreamSize.getHeight() / previewSurfaceSize.getHeight(); + referenceSize = previewStreamSize; + + // 3. Rotate to the stream coordinate system. + // Not elegant, but the sin/cos way was failing. + int angle = mEngine.getAngles().offset(Reference.SENSOR, Reference.VIEW, Axis.ABSOLUTE); + boolean flip = angle % 180 != 0; + float tempX = referencePoint.x; float tempY = referencePoint.y; + if (angle == 0) { + referencePoint.x = tempX; + referencePoint.y = tempY; + } else if (angle == 90) { + //noinspection SuspiciousNameCombination + referencePoint.x = tempY; + referencePoint.y = referenceSize.getWidth() - tempX; + } else if (angle == 180) { + referencePoint.x = referenceSize.getWidth() - tempX; + referencePoint.y = referenceSize.getHeight() - tempY; + } else if (angle == 270) { + referencePoint.x = referenceSize.getHeight() - tempY; + //noinspection SuspiciousNameCombination + referencePoint.y = tempX; + } else { + throw new IllegalStateException("Unexpected angle " + angle); + } + referenceSize = flip ? referenceSize.flip() : referenceSize; + + // These points are now referencing the stream rect on the sensor array. + // But we still have to figure out how the stream rect is laid on the sensor array. + // https://source.android.com/devices/camera/camera3_crop_reprocess.html + // For sanity, let's assume it is centered. + // For sanity, let's also assume that the crop region is equal to the stream region. + + // 4. Move to the active sensor array coordinate system. + Rect activeRect = readCharacteristic(CameraCharacteristics.SENSOR_INFO_ACTIVE_ARRAY_SIZE, + new Rect(0, 0, referenceSize.getWidth(), referenceSize.getHeight())); + referencePoint.x += (activeRect.width() - referenceSize.getWidth()) / 2F; + referencePoint.y += (activeRect.height() - referenceSize.getHeight()) / 2F; + referenceSize = new Size(activeRect.width(), activeRect.height()); + + // 5. Account for zoom! This only works for mZoomValue = 0. + // We must scale down with respect to the reference size center. If mZoomValue = 1, + // This must leave everything unchanged. + float maxZoom = readCharacteristic(CameraCharacteristics.SCALER_AVAILABLE_MAX_DIGITAL_ZOOM, + 1F /* no zoom */); + float currZoom = 1 + mEngine.mZoomValue * (maxZoom - 1); // 1 ... maxZoom + float currReduction = 1 / currZoom; + float referenceCenterX = referenceSize.getWidth() / 2F; + float referenceCenterY = referenceSize.getHeight() / 2F; + referencePoint.x = referenceCenterX + currReduction * (referencePoint.x - referenceCenterX); + referencePoint.y = referenceCenterY + currReduction * (referencePoint.y - referenceCenterY); + + // 6. NOW we can compute the metering regions. + float visibleWidth = referenceSize.getWidth() * currReduction; + float visibleHeight = referenceSize.getHeight() * currReduction; + MeteringRectangle area1 = createMeteringRectangle(referencePoint, referenceSize, visibleWidth, visibleHeight, 0.05F, 1000); + MeteringRectangle area2 = createMeteringRectangle(referencePoint, referenceSize, visibleWidth, visibleHeight, 0.1F, 100); + List areas = Arrays.asList(area1, area2); + + // 7. And finally dispatch everything + mAutoFocus.startMetering(mCharacteristics, mBuilder, areas); + mAutoWhiteBalance.startMetering(mCharacteristics, mBuilder, areas); + mAutoExposure.startMetering(mCharacteristics, mBuilder, areas); + + // Dispatch to callback + mCallback.onMeteringStarted(mPoint, mGesture); + } + + /** + * Creates a metering rectangle around the center point. + * The rectangle will have a size that's a factor of the visible width and height. + * The rectangle will also be constrained to be inside the given boundaries, + * so we don't exceed them in case the center point is exactly on one side for example. + * @return a new rectangle + */ + @NonNull + private MeteringRectangle createMeteringRectangle( + @NonNull PointF center, @NonNull Size boundaries, + float visibleWidth, float visibleHeight, + float factor, int weight) { + float halfWidth = factor * visibleWidth / 2F; + float halfHeight = factor * visibleHeight / 2F; + return new MeteringRectangle( + (int) Math.max(0, center.x - halfWidth), + (int) Math.max(0, center.y - halfHeight), + (int) Math.min(boundaries.getWidth(), halfWidth * 2F), + (int) Math.min(boundaries.getHeight(), halfHeight * 2F), + weight + ); + } + + /** + * True if we're metering. False if we're not, for example if we're waiting for + * a reset call, or if {@link #startMetering(PointF, Gesture)} was never called. + * @return true if metering + */ + @SuppressWarnings("WeakerAccess") + public boolean isMetering() { + return mIsMetering; + } + + /** + * Should be called when we have partial or total CaptureResults, + * but only while {@link #isMetering()} returns true. + * @param result result + */ + @SuppressWarnings("WeakerAccess") + public void onCapture(@NonNull CaptureResult result) { + if (!(result instanceof TotalCaptureResult)) return; // Let's ignore these, contents are missing/wrong + if (!mAutoFocus.isMetered()) mAutoFocus.onCapture(result); + if (!mAutoExposure.isMetered()) mAutoExposure.onCapture(result); + if (!mAutoWhiteBalance.isMetered()) mAutoWhiteBalance.onCapture(result); + if (mAutoFocus.isMetered() && mAutoExposure.isMetered() && mAutoWhiteBalance.isMetered()) { + // Use the AF success for dispatching the callback, since the public + // callback is currently related to AF. + mCallback.onMeteringEnd(mPoint, mGesture, mAutoFocus.isSuccessful()); + mIsMetering = false; + + mEngine.mHandler.remove(mResetRunnable); + if (mEngine.shouldResetAutoFocus()) { + mEngine.mHandler.post(mEngine.getAutoFocusResetDelay(), mResetRunnable); + } + } + } + + /** + * Can be called to perform the reset at a time different than the one + * specified by the {@link CameraEngine} reset delay. + */ + @SuppressWarnings("WeakerAccess") + public void resetMetering() { + mEngine.mHandler.remove(mResetRunnable); + if (mCallback.canResetMetering(mPoint, mGesture)) { + LOG.i("Resetting the meter parameters."); + Rect whole = readCharacteristic(CameraCharacteristics.SENSOR_INFO_ACTIVE_ARRAY_SIZE, new Rect()); + MeteringRectangle rectangle = new MeteringRectangle(whole, MeteringRectangle.METERING_WEIGHT_DONT_CARE); + mAutoFocus.resetMetering(mCharacteristics, mBuilder, rectangle); + mAutoWhiteBalance.resetMetering(mCharacteristics, mBuilder, rectangle); + mAutoExposure.resetMetering(mCharacteristics, mBuilder, rectangle); + mCallback.onMeteringReset(mPoint, mGesture); + } + } + + private Runnable mResetRunnable = new Runnable() { + @Override + public void run() { + resetMetering(); + } + }; +} diff --git a/cameraview/src/main/java/com/otaliastudios/cameraview/engine/metering/AutoExposure.java b/cameraview/src/main/java/com/otaliastudios/cameraview/engine/metering/AutoExposure.java new file mode 100644 index 00000000..ab28eb79 --- /dev/null +++ b/cameraview/src/main/java/com/otaliastudios/cameraview/engine/metering/AutoExposure.java @@ -0,0 +1,95 @@ +package com.otaliastudios.cameraview.engine.metering; + +import android.hardware.camera2.CameraCharacteristics; +import android.hardware.camera2.CaptureRequest; +import android.hardware.camera2.CaptureResult; +import android.hardware.camera2.params.MeteringRectangle; +import android.os.Build; + +import androidx.annotation.NonNull; +import androidx.annotation.RequiresApi; + +import com.otaliastudios.cameraview.CameraLogger; + +import java.util.List; + +@RequiresApi(Build.VERSION_CODES.LOLLIPOP) +public class AutoExposure extends MeteringParameter { + + private static final String TAG = AutoExposure.class.getSimpleName(); + private static final CameraLogger LOG = CameraLogger.create(TAG); + + private boolean isStarted; + + @Override + public void startMetering(@NonNull CameraCharacteristics characteristics, + @NonNull CaptureRequest.Builder builder, + @NonNull List areas) { + isSuccessful = false; + isMetered = false; + isStarted = false; + + boolean isNotLegacy = readCharacteristic(characteristics, + CameraCharacteristics.INFO_SUPPORTED_HARDWARE_LEVEL, -1) != + CameraCharacteristics.INFO_SUPPORTED_HARDWARE_LEVEL_LEGACY; + Integer aeMode = builder.get(CaptureRequest.CONTROL_AE_MODE); + boolean isAEOn = aeMode != null && + (aeMode == CameraCharacteristics.CONTROL_AE_MODE_ON + || aeMode == CameraCharacteristics.CONTROL_AE_MODE_ON_ALWAYS_FLASH + || aeMode == CameraCharacteristics.CONTROL_AE_MODE_ON_AUTO_FLASH + || aeMode == CameraCharacteristics.CONTROL_AE_MODE_ON_AUTO_FLASH_REDEYE); + isSupported = isNotLegacy && isAEOn; + + if (isSupported) { + builder.set(CaptureRequest.CONTROL_AE_PRECAPTURE_TRIGGER, + CaptureRequest.CONTROL_AE_PRECAPTURE_TRIGGER_START); + } + + // Even if precapture is not supported, check the regions anyway. + int maxRegions = readCharacteristic(characteristics, + CameraCharacteristics.CONTROL_MAX_REGIONS_AE, 0); + if (maxRegions > 0) { + int max = Math.min(maxRegions, areas.size()); + builder.set(CaptureRequest.CONTROL_AE_REGIONS, + areas.subList(0, max).toArray(new MeteringRectangle[]{})); + } + } + + @Override + public void onCapture(@NonNull CaptureResult result) { + if (isMetered || !isSupported) return; + Integer aeState = result.get(CaptureResult.CONTROL_AE_STATE); + LOG.i("onCapture:", "aeState:", aeState); + if (aeState == null) return; + + if (!isStarted) { + if (aeState == CaptureRequest.CONTROL_AE_STATE_PRECAPTURE) { + isStarted = true; + } + } else { + if (aeState == CaptureRequest.CONTROL_AE_STATE_CONVERGED + || aeState == CaptureRequest.CONTROL_AE_STATE_FLASH_REQUIRED) { + isMetered = true; + isSuccessful = true; + } + } + } + + @Override + public void resetMetering(@NonNull CameraCharacteristics characteristics, + @NonNull CaptureRequest.Builder builder, + @NonNull MeteringRectangle area) { + int maxRegions = readCharacteristic(characteristics, + CameraCharacteristics.CONTROL_MAX_REGIONS_AE, 0); + if (maxRegions > 0) { + builder.set(CaptureRequest.CONTROL_AE_REGIONS, new MeteringRectangle[]{area}); + } + if (isSupported) { + // Cleanup any precapture sequence. + if (Build.VERSION.SDK_INT >= 23) { + builder.set(CaptureRequest.CONTROL_AE_PRECAPTURE_TRIGGER, + CaptureRequest.CONTROL_AE_PRECAPTURE_TRIGGER_CANCEL); + } + } + } +} diff --git a/cameraview/src/main/java/com/otaliastudios/cameraview/engine/metering/AutoFocus.java b/cameraview/src/main/java/com/otaliastudios/cameraview/engine/metering/AutoFocus.java new file mode 100644 index 00000000..7a552a74 --- /dev/null +++ b/cameraview/src/main/java/com/otaliastudios/cameraview/engine/metering/AutoFocus.java @@ -0,0 +1,82 @@ +package com.otaliastudios.cameraview.engine.metering; + +import android.hardware.camera2.CameraCharacteristics; +import android.hardware.camera2.CaptureRequest; +import android.hardware.camera2.CaptureResult; +import android.hardware.camera2.params.MeteringRectangle; +import android.os.Build; + +import androidx.annotation.NonNull; +import androidx.annotation.RequiresApi; + +import com.otaliastudios.cameraview.CameraLogger; + +import java.util.List; + +@RequiresApi(Build.VERSION_CODES.LOLLIPOP) +public class AutoFocus extends MeteringParameter { + + private static final String TAG = AutoFocus.class.getSimpleName(); + private static final CameraLogger LOG = CameraLogger.create(TAG); + + @Override + public void startMetering(@NonNull CameraCharacteristics characteristics, + @NonNull CaptureRequest.Builder builder, + @NonNull List areas) { + isSuccessful = false; + isMetered = false; + + Integer afMode = builder.get(CaptureRequest.CONTROL_AF_MODE); + isSupported = afMode != null && afMode == CaptureRequest.CONTROL_AF_MODE_AUTO; + if (isSupported) { + builder.set(CaptureRequest.CONTROL_AF_TRIGGER, CaptureRequest.CONTROL_AF_TRIGGER_START); + } + + // Even if auto is not supported, change the regions anyway. + int maxRegions = readCharacteristic(characteristics, CameraCharacteristics.CONTROL_MAX_REGIONS_AF, 0); + if (maxRegions > 0) { + int max = Math.min(maxRegions, areas.size()); + builder.set(CaptureRequest.CONTROL_AF_REGIONS, + areas.subList(0, max).toArray(new MeteringRectangle[]{})); + } + + } + + @Override + public void onCapture(@NonNull CaptureResult result) { + if (isMetered || !isSupported) return; + Integer afState = result.get(CaptureResult.CONTROL_AF_STATE); + LOG.i("onCapture:", "afState:", afState); + if (afState == null) return; + switch (afState) { + case CaptureRequest.CONTROL_AF_STATE_FOCUSED_LOCKED: { + isMetered = true; + isSuccessful = true; + break; + } + case CaptureRequest.CONTROL_AF_STATE_NOT_FOCUSED_LOCKED: { + isMetered = true; + isSuccessful = false; + break; + } + case CaptureRequest.CONTROL_AF_STATE_INACTIVE: break; + case CaptureRequest.CONTROL_AF_STATE_ACTIVE_SCAN: break; + default: break; + } + } + + @Override + public void resetMetering(@NonNull CameraCharacteristics characteristics, + @NonNull CaptureRequest.Builder builder, + @NonNull MeteringRectangle area) { + int maxRegions = readCharacteristic(characteristics, + CameraCharacteristics.CONTROL_MAX_REGIONS_AF, 0); + if (maxRegions > 0) { + builder.set(CaptureRequest.CONTROL_AF_REGIONS, new MeteringRectangle[]{area}); + } + + if (isSupported) { // Cleanup any trigger. + builder.set(CaptureRequest.CONTROL_AF_TRIGGER, CaptureRequest.CONTROL_AF_TRIGGER_CANCEL); + } + } +} diff --git a/cameraview/src/main/java/com/otaliastudios/cameraview/engine/metering/AutoWhiteBalance.java b/cameraview/src/main/java/com/otaliastudios/cameraview/engine/metering/AutoWhiteBalance.java new file mode 100644 index 00000000..6fd5f3ea --- /dev/null +++ b/cameraview/src/main/java/com/otaliastudios/cameraview/engine/metering/AutoWhiteBalance.java @@ -0,0 +1,80 @@ +package com.otaliastudios.cameraview.engine.metering; + +import android.hardware.camera2.CameraCharacteristics; +import android.hardware.camera2.CaptureRequest; +import android.hardware.camera2.CaptureResult; +import android.hardware.camera2.params.MeteringRectangle; +import android.os.Build; + +import androidx.annotation.NonNull; +import androidx.annotation.RequiresApi; + +import com.otaliastudios.cameraview.CameraLogger; + +import java.util.List; + +@RequiresApi(Build.VERSION_CODES.LOLLIPOP) +public class AutoWhiteBalance extends MeteringParameter { + + private static final String TAG = AutoWhiteBalance.class.getSimpleName(); + private static final CameraLogger LOG = CameraLogger.create(TAG); + + @Override + public void startMetering(@NonNull CameraCharacteristics characteristics, + @NonNull CaptureRequest.Builder builder, + @NonNull List areas) { + isSuccessful = false; + isMetered = false; + + boolean isNotLegacy = readCharacteristic(characteristics, + CameraCharacteristics.INFO_SUPPORTED_HARDWARE_LEVEL, -1) != + CameraCharacteristics.INFO_SUPPORTED_HARDWARE_LEVEL_LEGACY; + Integer awbMode = builder.get(CaptureRequest.CONTROL_AWB_MODE); + isSupported = isNotLegacy && awbMode != null && awbMode == CaptureRequest.CONTROL_AWB_MODE_AUTO; + + if (isSupported) { + // Remove any lock. We're not setting any, but just in case. + builder.set(CaptureRequest.CONTROL_AWB_LOCK, false); + } + + // Even if auto is not supported, change the regions anyway. + int maxRegions = readCharacteristic(characteristics, CameraCharacteristics.CONTROL_MAX_REGIONS_AWB, 0); + if (maxRegions > 0) { + int max = Math.min(maxRegions, areas.size()); + builder.set(CaptureRequest.CONTROL_AWB_REGIONS, + areas.subList(0, max).toArray(new MeteringRectangle[]{})); + } + + } + + @Override + public void onCapture(@NonNull CaptureResult result) { + if (isMetered || !isSupported) return; + Integer awbState = result.get(CaptureResult.CONTROL_AWB_STATE); + LOG.i("onCapture:", "awbState:", awbState); + if (awbState == null) return; + + switch (awbState) { + case CaptureRequest.CONTROL_AWB_STATE_CONVERGED: { + isMetered = true; + isSuccessful = true; + break; + } + case CaptureRequest.CONTROL_AWB_STATE_LOCKED: break; + case CaptureRequest.CONTROL_AWB_STATE_INACTIVE: break; + case CaptureRequest.CONTROL_AWB_STATE_SEARCHING: break; + default: break; + } + } + + @Override + public void resetMetering(@NonNull CameraCharacteristics characteristics, + @NonNull CaptureRequest.Builder builder, + @NonNull MeteringRectangle area) { + int maxRegions = readCharacteristic(characteristics, + CameraCharacteristics.CONTROL_MAX_REGIONS_AWB, 0); + if (maxRegions > 0) { + builder.set(CaptureRequest.CONTROL_AWB_REGIONS, new MeteringRectangle[]{area}); + } + } +} diff --git a/cameraview/src/main/java/com/otaliastudios/cameraview/engine/metering/MeteringParameter.java b/cameraview/src/main/java/com/otaliastudios/cameraview/engine/metering/MeteringParameter.java new file mode 100644 index 00000000..9c34bb04 --- /dev/null +++ b/cameraview/src/main/java/com/otaliastudios/cameraview/engine/metering/MeteringParameter.java @@ -0,0 +1,52 @@ +package com.otaliastudios.cameraview.engine.metering; + +import android.hardware.camera2.CameraCharacteristics; +import android.hardware.camera2.CaptureRequest; +import android.hardware.camera2.CaptureResult; +import android.hardware.camera2.params.MeteringRectangle; +import android.os.Build; + +import androidx.annotation.NonNull; +import androidx.annotation.RequiresApi; + +import java.util.List; + +@RequiresApi(Build.VERSION_CODES.LOLLIPOP) +public abstract class MeteringParameter { + + @SuppressWarnings("WeakerAccess") + protected boolean isSupported; + + @SuppressWarnings("WeakerAccess") + protected boolean isSuccessful; + + @SuppressWarnings("WeakerAccess") + protected boolean isMetered; + + @SuppressWarnings("WeakerAccess") + @NonNull + protected T readCharacteristic(@NonNull CameraCharacteristics characteristics, + @NonNull CameraCharacteristics.Key key, + @NonNull T fallback) { + T value = characteristics.get(key); + return value == null ? fallback : value; + } + + public final boolean isMetered() { + return isMetered || !isSupported; + } + + public final boolean isSuccessful() { + return isSuccessful && isSupported; + } + + public abstract void startMetering(@NonNull CameraCharacteristics characteristics, + @NonNull CaptureRequest.Builder builder, + @NonNull List areas); + + public abstract void resetMetering(@NonNull CameraCharacteristics characteristics, + @NonNull CaptureRequest.Builder builder, + @NonNull MeteringRectangle area); + + public abstract void onCapture(@NonNull CaptureResult result); +}