From e96c9f08462fbfa0c0627554c2bd040a084d5d07 Mon Sep 17 00:00:00 2001 From: Mattia Iavarone Date: Mon, 1 Jul 2019 18:22:57 -0500 Subject: [PATCH] Camera2 support for frame processing --- .../cameraview/engine/MockCameraEngine.java | 5 +- .../otaliastudios/cameraview/CameraView.java | 8 +- .../cameraview/engine/Camera1Engine.java | 12 +- .../cameraview/engine/Camera2Engine.java | 132 +++++++++++++--- .../cameraview/engine/CameraEngine.java | 28 ++-- .../otaliastudios/cameraview/frame/Frame.java | 8 + .../cameraview/frame/FrameManager.java | 143 +++++++++++++----- .../internal/utils/ImageHelper.java | 96 ++++++++++++ .../cameraview/demo/CameraActivity.java | 13 ++ demo/src/main/res/layout/activity_camera.xml | 2 +- 10 files changed, 373 insertions(+), 74 deletions(-) create mode 100644 cameraview/src/main/java/com/otaliastudios/cameraview/internal/utils/ImageHelper.java diff --git a/cameraview/src/androidTest/java/com/otaliastudios/cameraview/engine/MockCameraEngine.java b/cameraview/src/androidTest/java/com/otaliastudios/cameraview/engine/MockCameraEngine.java index 63e553a1..8a271da9 100644 --- a/cameraview/src/androidTest/java/com/otaliastudios/cameraview/engine/MockCameraEngine.java +++ b/cameraview/src/androidTest/java/com/otaliastudios/cameraview/engine/MockCameraEngine.java @@ -12,6 +12,7 @@ import com.otaliastudios.cameraview.VideoResult; import com.otaliastudios.cameraview.controls.Audio; import com.otaliastudios.cameraview.controls.Facing; import com.otaliastudios.cameraview.controls.Flash; +import com.otaliastudios.cameraview.frame.FrameManager; import com.otaliastudios.cameraview.gesture.Gesture; import com.otaliastudios.cameraview.controls.Hdr; import com.otaliastudios.cameraview.controls.Mode; @@ -176,8 +177,10 @@ public class MockCameraEngine extends CameraEngine { mFocusStarted = true; } + @NonNull @Override - public void onBufferAvailable(@NonNull byte[] buffer) { + protected FrameManager instantiateFrameManager() { + return new FrameManager(2, null); } @Override diff --git a/cameraview/src/main/java/com/otaliastudios/cameraview/CameraView.java b/cameraview/src/main/java/com/otaliastudios/cameraview/CameraView.java index b79059de..fdcbdb35 100644 --- a/cameraview/src/main/java/com/otaliastudios/cameraview/CameraView.java +++ b/cameraview/src/main/java/com/otaliastudios/cameraview/CameraView.java @@ -1911,7 +1911,13 @@ public class CameraView extends FrameLayout implements LifecycleObserver { @Override public void run() { for (FrameProcessor processor : mFrameProcessors) { - processor.process(frame); + try { + processor.process(frame); + } catch (Exception e) { + mLogger.w("dispatchFrame:", "Error during processor implementation.", + "Can happen when camera is closed while processors are running.", + e); + } } frame.release(); } diff --git a/cameraview/src/main/java/com/otaliastudios/cameraview/engine/Camera1Engine.java b/cameraview/src/main/java/com/otaliastudios/cameraview/engine/Camera1Engine.java index bbd799a1..9523ccd2 100644 --- a/cameraview/src/main/java/com/otaliastudios/cameraview/engine/Camera1Engine.java +++ b/cameraview/src/main/java/com/otaliastudios/cameraview/engine/Camera1Engine.java @@ -26,6 +26,7 @@ import com.otaliastudios.cameraview.PictureResult; import com.otaliastudios.cameraview.VideoResult; import com.otaliastudios.cameraview.controls.Facing; import com.otaliastudios.cameraview.controls.Flash; +import com.otaliastudios.cameraview.frame.FrameManager; import com.otaliastudios.cameraview.gesture.Gesture; import com.otaliastudios.cameraview.controls.Hdr; import com.otaliastudios.cameraview.controls.Mode; @@ -47,7 +48,10 @@ import java.util.List; @SuppressWarnings("deprecation") -public class Camera1Engine extends CameraEngine implements Camera.PreviewCallback, Camera.ErrorCallback { +public class Camera1Engine extends CameraEngine implements + Camera.PreviewCallback, + Camera.ErrorCallback, + FrameManager.BufferCallback { private static final String TAG = Camera1Engine.class.getSimpleName(); private static final CameraLogger LOG = CameraLogger.create(TAG); @@ -284,6 +288,12 @@ public class Camera1Engine extends CameraEngine implements Camera.PreviewCallbac mHasFrameProcessors = hasFrameProcessors; } + @NonNull + @Override + protected FrameManager instantiateFrameManager() { + return new FrameManager(2, this); + } + @Override public void onBufferAvailable(@NonNull byte[] buffer) { // TODO: sync with handler? diff --git a/cameraview/src/main/java/com/otaliastudios/cameraview/engine/Camera2Engine.java b/cameraview/src/main/java/com/otaliastudios/cameraview/engine/Camera2Engine.java index 9139bdbb..4732f802 100644 --- a/cameraview/src/main/java/com/otaliastudios/cameraview/engine/Camera2Engine.java +++ b/cameraview/src/main/java/com/otaliastudios/cameraview/engine/Camera2Engine.java @@ -3,6 +3,8 @@ package com.otaliastudios.cameraview.engine; import android.annotation.SuppressLint; import android.content.Context; import android.graphics.Camera; +import android.graphics.ImageFormat; +import android.graphics.PixelFormat; import android.graphics.PointF; import android.graphics.Rect; import android.graphics.SurfaceTexture; @@ -14,6 +16,8 @@ import android.hardware.camera2.CameraManager; import android.hardware.camera2.CaptureRequest; import android.hardware.camera2.params.StreamConfigurationMap; import android.location.Location; +import android.media.Image; +import android.media.ImageReader; import android.media.MediaCodec; import android.os.Build; import android.view.Surface; @@ -33,14 +37,20 @@ import com.otaliastudios.cameraview.controls.Flash; import com.otaliastudios.cameraview.controls.Hdr; import com.otaliastudios.cameraview.controls.Mode; import com.otaliastudios.cameraview.controls.WhiteBalance; +import com.otaliastudios.cameraview.frame.Frame; +import com.otaliastudios.cameraview.frame.FrameManager; import com.otaliastudios.cameraview.gesture.Gesture; import com.otaliastudios.cameraview.internal.utils.CropHelper; +import com.otaliastudios.cameraview.internal.utils.ImageHelper; +import com.otaliastudios.cameraview.internal.utils.WorkerHandler; import com.otaliastudios.cameraview.picture.SnapshotGlPictureRecorder; import com.otaliastudios.cameraview.preview.GlCameraPreview; import com.otaliastudios.cameraview.size.AspectRatio; import com.otaliastudios.cameraview.size.Size; +import com.otaliastudios.cameraview.size.SizeSelectors; import com.otaliastudios.cameraview.video.Full2VideoRecorder; import com.otaliastudios.cameraview.video.SnapshotVideoRecorder; +import com.otaliastudios.cameraview.video.VideoRecorder; import java.util.ArrayList; import java.util.List; @@ -57,13 +67,15 @@ import androidx.annotation.WorkerThread; // TODO exposure correction // TODO autofocus // TODO pictures -// TODO frame processor @RequiresApi(Build.VERSION_CODES.LOLLIPOP) -public class Camera2Engine extends CameraEngine { +public class Camera2Engine extends CameraEngine implements ImageReader.OnImageAvailableListener { private static final String TAG = Camera2Engine.class.getSimpleName(); private static final CameraLogger LOG = CameraLogger.create(TAG); + private static final int FRAME_PROCESSING_FORMAT = ImageFormat.NV21; + private static final int FRAME_PROCESSING_INPUT_FORMAT = ImageFormat.YUV_420_888; + private final CameraManager mManager; private String mCameraId; private CameraDevice mCamera; @@ -72,6 +84,11 @@ public class Camera2Engine extends CameraEngine { private CaptureRequest.Builder mRepeatingRequestBuilder; private CaptureRequest mRepeatingRequest; + // Frame processing + private Size mFrameProcessingSize; + private ImageReader mFrameProcessingReader; // need this or the reader surface is collected + private final WorkerHandler mFrameConversionHandler; + private Surface mPreviewStreamSurface; private Surface mFrameProcessingSurface; @@ -85,6 +102,7 @@ public class Camera2Engine extends CameraEngine { super(callback); mMapper = Mapper.get(Engine.CAMERA2); mManager = (CameraManager) mCallback.getContext().getSystemService(Context.CAMERA_SERVICE); + mFrameConversionHandler = WorkerHandler.get("CameraViewFrameConversion"); } //region Utilities @@ -374,9 +392,29 @@ public class Camera2Engine extends CameraEngine { // 4. FRAME PROCESSING if (mHasFrameProcessors) { - // TODO + // Choose the size. + StreamConfigurationMap streamMap = mCameraCharacteristics.get(CameraCharacteristics.SCALER_STREAM_CONFIGURATION_MAP); + if (streamMap == null) throw new RuntimeException("StreamConfigurationMap is null. Should not happen."); + android.util.Size[] aSizes = streamMap.getOutputSizes(FRAME_PROCESSING_INPUT_FORMAT); + List sizes = new ArrayList<>(); + for (android.util.Size aSize : aSizes) { + sizes.add(new Size(aSize.getWidth(), aSize.getHeight())); + } + mFrameProcessingSize = SizeSelectors.and( + SizeSelectors.maxWidth(Math.min(700, mPreviewStreamSize.getWidth())), + SizeSelectors.maxHeight(Math.min(700, mPreviewStreamSize.getHeight())), + SizeSelectors.biggest()).select(sizes).get(0); + mFrameProcessingReader = ImageReader.newInstance( + mFrameProcessingSize.getWidth(), + mFrameProcessingSize.getHeight(), + FRAME_PROCESSING_INPUT_FORMAT, + 2); + mFrameProcessingReader.setOnImageAvailableListener(this, mFrameConversionHandler.getHandler()); + mFrameProcessingSurface = mFrameProcessingReader.getSurface(); outputSurfaces.add(mFrameProcessingSurface); } else { + mFrameProcessingReader = null; + mFrameProcessingSize = null; mFrameProcessingSurface = null; } @@ -414,23 +452,28 @@ public class Camera2Engine extends CameraEngine { throw new IllegalStateException("previewStreamSize should not be null at this point."); } mPreview.setStreamSize(previewSizeForView.getWidth(), previewSizeForView.getHeight()); - - // Set the preview rotation. mPreview.setDrawRotation(mDisplayOffset); + if (mHasFrameProcessors) { + getFrameManager().setUp(ImageFormat.getBitsPerPixel(FRAME_PROCESSING_FORMAT), mFrameProcessingSize); + } - // TODO mCamera.setPreviewCallbackWithBuffer(null); // Release anything left - // TODO mCamera.setPreviewCallbackWithBuffer(this); // Add ourselves - // TODO mFrameManager.setUp(ImageFormat.getBitsPerPixel(mPreviewStreamFormat), mPreviewStreamSize); - - LOG.i("onStartPreview", "Starting preview with startPreview()."); + LOG.i("onStartPreview", "Starting preview."); addRepeatingRequestBuilderSurfaces(); applyRepeatingRequestBuilder(false, CameraException.REASON_FAILED_TO_START_PREVIEW, null); LOG.i("onStartPreview", "Started preview."); // Start delayed video if needed. if (mFullVideoPendingStub != null) { - // Do not call takeVideo. It will reset some stub parameters that the recorder sets. - onTakeVideo(mFullVideoPendingStub); + // Do not call takeVideo/onTakeVideo. It will reset some stub parameters that the recorder sets. + // Also we are posting this so that doTakeVideo sees a started preview. + final VideoResult.Stub stub = mFullVideoPendingStub; + mFullVideoPendingStub = null; + mHandler.post(new Runnable() { + @Override + public void run() { + doTakeVideo(stub); + } + }); } return Tasks.forResult(null); } @@ -449,8 +492,9 @@ public class Camera2Engine extends CameraEngine { mVideoRecorder = null; } mPictureRecorder = null; - getFrameManager().release(); - // TODO mCamera.setPreviewCallbackWithBuffer(null); // Release anything left + if (mHasFrameProcessors) { + getFrameManager().release(); + } try { // NOTE: should we wait for onReady() like docs say? // Leaving this synchronous for now. @@ -478,6 +522,11 @@ public class Camera2Engine extends CameraEngine { mPreviewStreamSurface = null; mPreviewStreamSize = null; mCaptureSize = null; + mFrameProcessingSize = null; + if (mFrameProcessingReader != null) { + mFrameProcessingReader.close(); + mFrameProcessingReader = null; + } mSession.close(); mSession = null; return Tasks.forResult(null); @@ -537,15 +586,9 @@ public class Camera2Engine extends CameraEngine { stub.size = flip(REF_SENSOR, REF_OUTPUT) ? mCaptureSize.flip() : mCaptureSize; if (!Full2VideoRecorder.SUPPORTS_PERSISTENT_SURFACE) { // On API 21 and 22, we must restart the session at each time. - if (stub == mFullVideoPendingStub) { - // We have restarted and are ready to take the video. - doTakeVideo(mFullVideoPendingStub); - mFullVideoPendingStub = null; - } else { - // Save the pending data and restart the session. - mFullVideoPendingStub = stub; - restartBind(); - } + // Save the pending data and restart the session. + mFullVideoPendingStub = stub; + restartBind(); } else { doTakeVideo(stub); } @@ -820,9 +863,48 @@ public class Camera2Engine extends CameraEngine { //region FrameProcessing + @NonNull @Override - public void onBufferAvailable(@NonNull byte[] buffer) { - // TODO + protected FrameManager instantiateFrameManager() { + return new FrameManager(2, null); + } + + @Override + public void onImageAvailable(ImageReader reader) { + byte[] data = getFrameManager().getBuffer(); + if (data == null) { + LOG.w("onImageAvailable", "no byte buffer!"); + return; + } + Image image = null; + try { + image = reader.acquireLatestImage(); + } catch (IllegalStateException ignore) { } + if (image == null) { + LOG.w("onImageAvailable", "we have a byte buffer but no Image!"); + getFrameManager().onBufferUnused(data); + return; + } + LOG.i("onImageAvailable", "we have both a byte buffer and an Image."); + try { + ImageHelper.convertToNV21(image, data); + } catch (Exception e) { + LOG.w("onImageAvailable", "error while converting."); + getFrameManager().onBufferUnused(data); + image.close(); + return; + } + image.close(); + if (getEngineState() == STATE_STARTED) { + Frame frame = getFrameManager().getFrame(data, + System.currentTimeMillis(), + offset(REF_SENSOR, REF_OUTPUT), + mFrameProcessingSize, + FRAME_PROCESSING_FORMAT); + mCallback.dispatchFrame(frame); + } else { + getFrameManager().onBufferUnused(data); + } } @Override diff --git a/cameraview/src/main/java/com/otaliastudios/cameraview/engine/CameraEngine.java b/cameraview/src/main/java/com/otaliastudios/cameraview/engine/CameraEngine.java index d9723ed2..c550e1e1 100644 --- a/cameraview/src/main/java/com/otaliastudios/cameraview/engine/CameraEngine.java +++ b/cameraview/src/main/java/com/otaliastudios/cameraview/engine/CameraEngine.java @@ -115,7 +115,6 @@ import java.util.concurrent.Executor; */ public abstract class CameraEngine implements CameraPreview.SurfaceCallback, - FrameManager.BufferCallback, PictureRecorder.PictureResultListener, VideoRecorder.VideoResultListener { @@ -224,7 +223,7 @@ public abstract class CameraEngine implements mCrashHandler = new Handler(Looper.getMainLooper()); mHandler = WorkerHandler.get("CameraViewEngine"); mHandler.getThread().setUncaughtExceptionHandler(new CrashExceptionHandler()); - mFrameManager = new FrameManager(2, this); + mFrameManager = instantiateFrameManager(); } public void setPreview(@NonNull CameraPreview cameraPreview) { @@ -474,11 +473,11 @@ public abstract class CameraEngine implements @SuppressWarnings("WeakerAccess") protected void restartBind() { - LOG.i("restartPreviewAndBind", "posting."); + LOG.i("restartBind", "posting."); mHandler.run(new Runnable() { @Override public void run() { - LOG.i("restartPreviewAndBind", "executing."); + LOG.i("restartBind", "executing."); stopPreview(false).continueWithTask(mHandler.getExecutor(), new Continuation>() { @Override public Task then(@NonNull Task task) { @@ -605,10 +604,13 @@ public abstract class CameraEngine implements // Compute a new camera preview size and apply. Size newSize = computePreviewStreamSize(); - if (newSize.equals(mPreviewStreamSize)) return; - LOG.i("onSurfaceChanged:", "Computed a new preview size. Going on."); - mPreviewStreamSize = newSize; - onPreviewStreamSizeChanged(); + if (newSize.equals(mPreviewStreamSize)) { + LOG.i("onSurfaceChanged:", "The computed preview size is identical. No op."); + } else { + LOG.i("onSurfaceChanged:", "Computed a new preview size. Calling onPreviewStreamSizeChanged()."); + mPreviewStreamSize = newSize; + onPreviewStreamSizeChanged(); + } } }); } @@ -895,6 +897,14 @@ public abstract class CameraEngine implements */ protected abstract boolean collectCameraInfo(@NonNull Facing facing); + /** + * Called at construction time to get a frame manager that can later be + * accessed through {@link #getFrameManager()}. + * @return a frame manager + */ + @NonNull + protected abstract FrameManager instantiateFrameManager(); + // If closed, no-op. If opened, check supported and apply. public abstract void setZoom(float zoom, @Nullable PointF[] points, boolean notify); @@ -1353,7 +1363,7 @@ public abstract class CameraEngine implements if (targetMinSize == null) throw new IllegalStateException("targetMinSize should not be null here."); AspectRatio targetRatio = AspectRatio.of(mCaptureSize.getWidth(), mCaptureSize.getHeight()); if (flip) targetRatio = targetRatio.flip(); - LOG.i("size:", "computePreviewStreamSize:", "targetRatio:", targetRatio, "targetMinSize:", targetMinSize); + LOG.i("computePreviewStreamSize:", "targetRatio:", targetRatio, "targetMinSize:", targetMinSize); SizeSelector matchRatio = SizeSelectors.and( // Match this aspect ratio and sort by biggest SizeSelectors.aspectRatio(targetRatio, 0), SizeSelectors.biggest()); diff --git a/cameraview/src/main/java/com/otaliastudios/cameraview/frame/Frame.java b/cameraview/src/main/java/com/otaliastudios/cameraview/frame/Frame.java index 38f709fd..f8607b86 100644 --- a/cameraview/src/main/java/com/otaliastudios/cameraview/frame/Frame.java +++ b/cameraview/src/main/java/com/otaliastudios/cameraview/frame/Frame.java @@ -1,5 +1,6 @@ package com.otaliastudios.cameraview.frame; +import com.otaliastudios.cameraview.CameraLogger; import com.otaliastudios.cameraview.size.Size; import androidx.annotation.NonNull; @@ -10,10 +11,14 @@ import androidx.annotation.VisibleForTesting; */ public class Frame { + private final static String TAG = Frame.class.getSimpleName(); + private final static CameraLogger LOG = CameraLogger.create(TAG); + @VisibleForTesting FrameManager mManager; private byte[] mData = null; private long mTime = -1; + private long mLastTime = -1; private int mRotation = 0; private Size mSize = null; private int mFormat = -1; @@ -29,6 +34,7 @@ public class Frame { private void ensureAlive() { if (!isAlive()) { + LOG.e("Frame is dead! time:", mTime, "lastTime:", mLastTime); throw new RuntimeException("You should not access a released frame. " + "If this frame was passed to a FrameProcessor, you can only use its contents synchronously," + "for the duration of the process() method."); @@ -38,6 +44,7 @@ public class Frame { void set(@NonNull byte[] data, long time, int rotation, @NonNull Size size, int format) { this.mData = data; this.mTime = time; + this.mLastTime = time; this.mRotation = rotation; this.mSize = size; this.mFormat = format; @@ -73,6 +80,7 @@ public class Frame { */ public void release() { if (!isAlive()) return; + LOG.v("Frame with time", mTime, "is being released. Has manager:", mManager != null); if (mManager != null) { // If needed, the manager will call releaseManager on us. diff --git a/cameraview/src/main/java/com/otaliastudios/cameraview/frame/FrameManager.java b/cameraview/src/main/java/com/otaliastudios/cameraview/frame/FrameManager.java index 6c18068e..2f7d0046 100644 --- a/cameraview/src/main/java/com/otaliastudios/cameraview/frame/FrameManager.java +++ b/cameraview/src/main/java/com/otaliastudios/cameraview/frame/FrameManager.java @@ -1,6 +1,7 @@ package com.otaliastudios.cameraview.frame; +import com.otaliastudios.cameraview.CameraLogger; import com.otaliastudios.cameraview.size.Size; import androidx.annotation.NonNull; @@ -25,6 +26,9 @@ import java.util.concurrent.LinkedBlockingQueue; */ public class FrameManager { + private static final String TAG = FrameManager.class.getSimpleName(); + private static final CameraLogger LOG = CameraLogger.create(TAG); + /** * Receives callbacks on buffer availability * (when a Frame is released, we reuse its buffer). @@ -33,10 +37,24 @@ public class FrameManager { void onBufferAvailable(@NonNull byte[] buffer); } - private int mPoolSize; - private int mBufferSize; - private BufferCallback mCallback; - private LinkedBlockingQueue mQueue; + private final int mPoolSize; + private int mBufferSize = -1; + private LinkedBlockingQueue mFrameQueue; + private LinkedBlockingQueue mBufferQueue; + private BufferCallback mBufferCallback; + private final int mBufferMode; + + /** + * In this mode, we have a {@link #mBufferCallback} and dispatch + * new buffers to the callback. + */ + private final static int BUFFER_MODE_DISPATCH = 0; + + /** + * In this mode, we have a {@link #mBufferQueue} where we store + * buffers and only dispatch when requested. + */ + private final static int BUFFER_MODE_ENQUEUE = 1; /** * Construct a new frame manager. @@ -48,9 +66,14 @@ public class FrameManager { */ public FrameManager(int poolSize, @Nullable BufferCallback callback) { mPoolSize = poolSize; - mCallback = callback; - mQueue = new LinkedBlockingQueue<>(mPoolSize); - mBufferSize = -1; + mFrameQueue = new LinkedBlockingQueue<>(mPoolSize); + if (callback != null) { + mBufferCallback = callback; + mBufferMode = BUFFER_MODE_DISPATCH; + } else { + mBufferQueue = new LinkedBlockingQueue<>(mPoolSize); + mBufferMode = BUFFER_MODE_ENQUEUE; + } } /** @@ -65,41 +88,46 @@ public class FrameManager { */ public int setUp(int bitsPerPixel, @NonNull Size previewSize) { // TODO throw if called twice without release? - mBufferSize = getBufferSize(bitsPerPixel, previewSize); + long sizeInBits = previewSize.getHeight() * previewSize.getWidth() * bitsPerPixel; + mBufferSize = (int) Math.ceil(sizeInBits / 8.0d); for (int i = 0; i < mPoolSize; i++) { - mCallback.onBufferAvailable(new byte[mBufferSize]); + if (mBufferMode == BUFFER_MODE_DISPATCH) { + mBufferCallback.onBufferAvailable(new byte[mBufferSize]); + } else { + mBufferQueue.offer(new byte[mBufferSize]); + } } return mBufferSize; } /** - * Releases all frames controlled by this manager and - * clears the pool. + * Returns a new byte buffer than can be filled. + * This can only be called in {@link #BUFFER_MODE_ENQUEUE} mode! Where the frame + * manager also holds a queue of the byte buffers. + * + * If not null, the buffer returned by this method can be filled and used to get + * a new frame through {@link #getFrame(byte[], long, int, Size, int)}. + * + * @return a buffer, or null */ - public void release() { - for (Frame frame : mQueue) { - frame.releaseManager(); - frame.release(); + @Nullable + public byte[] getBuffer() { + if (mBufferMode != BUFFER_MODE_ENQUEUE) { + throw new IllegalStateException("Can't call getBuffer() when not in BUFFER_MODE_ENQUEUE."); } - mQueue.clear(); - mBufferSize = -1; + return mBufferQueue.poll(); } - void onFrameReleased(Frame frame) { - byte[] buffer = frame.getData(); - boolean willRecycle = mQueue.offer(frame); - if (!willRecycle) { - // If frame queue is full, let's drop everything. - frame.releaseManager(); - } else { - // If frame will be recycled, let's recycle the buffer as well. - int currSize = buffer.length; - int reqSize = mBufferSize; - if (currSize == reqSize && mCallback != null) { - mCallback.onBufferAvailable(buffer); - } + /** + * Can be called if the buffer obtained by {@link #getBuffer()} + * was not used to construct a frame, so it can be put back into the queue. + * @param buffer a buffer + */ + public void onBufferUnused(@NonNull byte[] buffer) { + if (mBufferMode != BUFFER_MODE_ENQUEUE) { + throw new IllegalStateException("Can't call onBufferUnused() when not in BUFFER_MODE_ENQUEUE."); } - + mBufferQueue.offer(buffer); } /** @@ -113,15 +141,58 @@ public class FrameManager { * * @return a new frame */ + @NonNull public Frame getFrame(@NonNull byte[] data, long time, int rotation, @NonNull Size previewSize, int previewFormat) { - Frame frame = mQueue.poll(); - if (frame == null) frame = new Frame(this); + Frame frame = mFrameQueue.poll(); + if (frame != null) { + LOG.v("getFrame for time:", time, "RECYCLING.", "Data:", data != null); + } else { + LOG.v("getFrame for time:", time, "CREATING.", "Data:", data != null); + frame = new Frame(this); + } frame.set(data, time, rotation, previewSize, previewFormat); return frame; } - private int getBufferSize(int bitsPerPixel, @NonNull Size previewSize) { - long sizeInBits = previewSize.getHeight() * previewSize.getWidth() * bitsPerPixel; - return (int) Math.ceil(sizeInBits / 8.0d); + /** + * Releases all frames controlled by this manager and + * clears the pool. + * In BUFFER_MODE_ENQUEUE, releases also all the buffers. + */ + public void release() { + LOG.w("Releasing all frames!"); + for (Frame frame : mFrameQueue) { + frame.releaseManager(); + frame.release(); + } + mFrameQueue.clear(); + if (mBufferMode == BUFFER_MODE_ENQUEUE) { + mBufferQueue.clear(); + } + mBufferSize = -1; + } + + /** + * Called by child frames when they are released. + * @param frame the released frame + */ + void onFrameReleased(@NonNull Frame frame) { + byte[] buffer = frame.getData(); + boolean willRecycle = mFrameQueue.offer(frame); + if (!willRecycle) { + // If frame queue is full, let's drop everything. + frame.releaseManager(); + } else { + // If frame will be recycled, let's recycle the buffer as well. + int currSize = buffer.length; + int reqSize = mBufferSize; + if (currSize == reqSize) { + if (mBufferMode == BUFFER_MODE_DISPATCH) { + mBufferCallback.onBufferAvailable(buffer); + } else { + mBufferQueue.offer(buffer); + } + } + } } } diff --git a/cameraview/src/main/java/com/otaliastudios/cameraview/internal/utils/ImageHelper.java b/cameraview/src/main/java/com/otaliastudios/cameraview/internal/utils/ImageHelper.java new file mode 100644 index 00000000..c92078cd --- /dev/null +++ b/cameraview/src/main/java/com/otaliastudios/cameraview/internal/utils/ImageHelper.java @@ -0,0 +1,96 @@ +package com.otaliastudios.cameraview.internal.utils; + +import android.graphics.ImageFormat; +import android.graphics.Rect; +import android.media.Image; + +import com.otaliastudios.cameraview.size.AspectRatio; +import com.otaliastudios.cameraview.size.Size; + +import java.nio.ByteBuffer; + +import androidx.annotation.NonNull; +import androidx.annotation.RequiresApi; + +/** + * Conversions for {@link android.media.Image}s into byte arrays. + */ +@RequiresApi(19) +public class ImageHelper { + + // https://stackoverflow.com/a/52740776/4288782 + public static void convertToNV21(@NonNull Image image, @NonNull byte[] result) { + if (image.getFormat() != ImageFormat.YUV_420_888) { + throw new IllegalStateException("CAn only convert from YUV_420_888."); + } + int width = image.getWidth(); + int height = image.getHeight(); + int ySize = width * height; + int uvSize = width * height / 4; + + ByteBuffer yBuffer = image.getPlanes()[0].getBuffer(); // Y + ByteBuffer uBuffer = image.getPlanes()[1].getBuffer(); // U + ByteBuffer vBuffer = image.getPlanes()[2].getBuffer(); // V + + int rowStride = image.getPlanes()[0].getRowStride(); + + if (image.getPlanes()[0].getPixelStride() != 1) { + throw new AssertionError("Something wrong in convertToNV21"); + } + + int pos = 0; + + if (rowStride == width) { // likely + yBuffer.get(result, 0, ySize); + pos += ySize; + } + else { + int yBufferPos = width - rowStride; // not an actual position + for (; pos