From 28023c72ebc8624f6d29ff4456e9a854ad997ad7 Mon Sep 17 00:00:00 2001 From: Mattia Iavarone Date: Sun, 15 Dec 2019 21:47:39 +0100 Subject: [PATCH] Prepare Frame for Images, abstract FrameManager, create ByteBufferFrameManager --- .../cameraview/CameraViewTest.java | 14 -- .../cameraview/engine/MockCameraEngine.java | 3 +- .../frame/ByteBufferFrameManagerTest.java | 105 +++++++++++ .../cameraview/frame/FrameManagerTest.java | 87 +-------- .../cameraview/engine/Camera1Engine.java | 12 +- .../cameraview/engine/Camera2Engine.java | 26 ++- .../cameraview/engine/CameraBaseEngine.java | 2 +- .../frame/ByteBufferFrameManager.java | 154 ++++++++++++++++ .../otaliastudios/cameraview/frame/Frame.java | 51 +++++- .../cameraview/frame/FrameManager.java | 172 +++++------------- .../cameraview/demo/CameraActivity.java | 30 +-- 11 files changed, 402 insertions(+), 254 deletions(-) create mode 100644 cameraview/src/androidTest/java/com/otaliastudios/cameraview/frame/ByteBufferFrameManagerTest.java create mode 100644 cameraview/src/main/java/com/otaliastudios/cameraview/frame/ByteBufferFrameManager.java diff --git a/cameraview/src/androidTest/java/com/otaliastudios/cameraview/CameraViewTest.java b/cameraview/src/androidTest/java/com/otaliastudios/cameraview/CameraViewTest.java index d1470a4e..c1a251d8 100644 --- a/cameraview/src/androidTest/java/com/otaliastudios/cameraview/CameraViewTest.java +++ b/cameraview/src/androidTest/java/com/otaliastudios/cameraview/CameraViewTest.java @@ -996,22 +996,8 @@ public class CameraViewTest extends BaseTest { //region Filter - @Test(expected = RuntimeException.class) - public void testSetFilter_notExperimental() { - cameraView.setExperimental(false); - cameraView.setFilter(Filters.AUTO_FIX.newInstance()); - } - - @Test - public void testSetFilter_notExperimental_noFilter() { - cameraView.setExperimental(false); - cameraView.setFilter(Filters.NONE.newInstance()); - // no exception thrown - } - @Test public void testSetFilter() { - cameraView.setExperimental(true); Filter filter = Filters.AUTO_FIX.newInstance(); cameraView.setFilter(filter); verify(mockPreview, times(1)).setFilter(filter); diff --git a/cameraview/src/androidTest/java/com/otaliastudios/cameraview/engine/MockCameraEngine.java b/cameraview/src/androidTest/java/com/otaliastudios/cameraview/engine/MockCameraEngine.java index 166bff19..a5dfb7e6 100644 --- a/cameraview/src/androidTest/java/com/otaliastudios/cameraview/engine/MockCameraEngine.java +++ b/cameraview/src/androidTest/java/com/otaliastudios/cameraview/engine/MockCameraEngine.java @@ -13,6 +13,7 @@ import com.otaliastudios.cameraview.controls.Facing; import com.otaliastudios.cameraview.controls.Flash; import com.otaliastudios.cameraview.controls.PictureFormat; import com.otaliastudios.cameraview.engine.orchestrator.CameraState; +import com.otaliastudios.cameraview.frame.ByteBufferFrameManager; import com.otaliastudios.cameraview.frame.FrameManager; import com.otaliastudios.cameraview.gesture.Gesture; import com.otaliastudios.cameraview.controls.Hdr; @@ -185,7 +186,7 @@ public class MockCameraEngine extends CameraBaseEngine { @NonNull @Override protected FrameManager instantiateFrameManager() { - return new FrameManager(2, null); + return new ByteBufferFrameManager(2, null); } @Override diff --git a/cameraview/src/androidTest/java/com/otaliastudios/cameraview/frame/ByteBufferFrameManagerTest.java b/cameraview/src/androidTest/java/com/otaliastudios/cameraview/frame/ByteBufferFrameManagerTest.java new file mode 100644 index 00000000..d3f1242f --- /dev/null +++ b/cameraview/src/androidTest/java/com/otaliastudios/cameraview/frame/ByteBufferFrameManagerTest.java @@ -0,0 +1,105 @@ +package com.otaliastudios.cameraview.frame; + + +import android.graphics.ImageFormat; + +import androidx.test.ext.junit.runners.AndroidJUnit4; +import androidx.test.filters.SmallTest; + +import com.otaliastudios.cameraview.BaseTest; +import com.otaliastudios.cameraview.size.Size; + +import org.junit.After; +import org.junit.Before; +import org.junit.Test; +import org.junit.runner.RunWith; + +import static org.mockito.Matchers.any; +import static org.mockito.Mockito.mock; +import static org.mockito.Mockito.never; +import static org.mockito.Mockito.reset; +import static org.mockito.Mockito.times; +import static org.mockito.Mockito.verify; + +@RunWith(AndroidJUnit4.class) +@SmallTest +public class ByteBufferFrameManagerTest extends BaseTest { + + private ByteBufferFrameManager.BufferCallback callback; + + @Before + public void setUp() { + callback = mock(ByteBufferFrameManager.BufferCallback.class); + } + + @After + public void tearDown() { + callback = null; + } + + @Test + public void testAllocate() { + ByteBufferFrameManager manager = new ByteBufferFrameManager(1, callback); + manager.setUp(ImageFormat.NV21, new Size(50, 50)); + verify(callback, times(1)).onBufferAvailable(any(byte[].class)); + reset(callback); + + manager = new ByteBufferFrameManager(5, callback); + manager.setUp(ImageFormat.NV21, new Size(50, 50)); + verify(callback, times(5)).onBufferAvailable(any(byte[].class)); + } + + @Test + public void testOnFrameReleased_alreadyFull() { + ByteBufferFrameManager manager = new ByteBufferFrameManager(1, callback); + manager.setUp(ImageFormat.NV21, new Size(50, 50)); + int length = manager.getFrameBytes(); + + Frame frame1 = manager.getFrame(new byte[length], 0, 0); + // Since frame1 is already taken and poolSize = 1, a new Frame is created. + Frame frame2 = manager.getFrame(new byte[length], 0, 0); + // Release the first frame so it goes back into the pool. + manager.onFrameReleased(frame1, (byte[]) frame1.getData()); + reset(callback); + // Release the second. The pool is already full, so onBufferAvailable should not be called + // since this Frame instance will NOT be reused. + manager.onFrameReleased(frame2, (byte[]) frame2.getData()); + verify(callback, never()).onBufferAvailable((byte[]) frame2.getData()); + } + + @Test + public void testOnFrameReleased_sameLength() { + ByteBufferFrameManager manager = new ByteBufferFrameManager(1, callback); + manager.setUp(ImageFormat.NV21, new Size(50, 50)); + int length = manager.getFrameBytes(); + + // A camera preview frame comes. Request a frame. + byte[] picture = new byte[length]; + Frame frame = manager.getFrame(picture, 0, 0); + + // Release the frame and ensure that onBufferAvailable is called. + reset(callback); + manager.onFrameReleased(frame, (byte[]) frame.getData()); + verify(callback, times(1)).onBufferAvailable(picture); + } + + @Test + public void testOnFrameReleased_differentLength() { + ByteBufferFrameManager manager = new ByteBufferFrameManager(1, callback); + manager.setUp(ImageFormat.NV21, new Size(50, 50)); + int length = manager.getFrameBytes(); + + // A camera preview frame comes. Request a frame. + byte[] picture = new byte[length]; + Frame frame = manager.getFrame(picture, 0, 0); + + // Don't release the frame. Change the allocation size. + manager.setUp(ImageFormat.NV16, new Size(15, 15)); + + // Now release the old frame and ensure that onBufferAvailable is NOT called, + // because the released data has wrong length. + manager.onFrameReleased(frame, (byte[]) frame.getData()); + reset(callback); + verify(callback, never()).onBufferAvailable(picture); + } +} diff --git a/cameraview/src/androidTest/java/com/otaliastudios/cameraview/frame/FrameManagerTest.java b/cameraview/src/androidTest/java/com/otaliastudios/cameraview/frame/FrameManagerTest.java index 77375047..746036ea 100644 --- a/cameraview/src/androidTest/java/com/otaliastudios/cameraview/frame/FrameManagerTest.java +++ b/cameraview/src/androidTest/java/com/otaliastudios/cameraview/frame/FrameManagerTest.java @@ -3,6 +3,7 @@ package com.otaliastudios.cameraview.frame; import android.graphics.ImageFormat; +import androidx.annotation.NonNull; import androidx.test.ext.junit.runners.AndroidJUnit4; import androidx.test.filters.SmallTest; @@ -26,93 +27,19 @@ import static org.mockito.Mockito.verify; @SmallTest public class FrameManagerTest extends BaseTest { - private FrameManager.BufferCallback callback; - - @Before - public void setUp() { - callback = mock(FrameManager.BufferCallback.class); - } - - @After - public void tearDown() { - callback = null; - } - - @Test - public void testAllocate() { - FrameManager manager = new FrameManager(1, callback); - manager.setUp(ImageFormat.NV21, new Size(50, 50)); - verify(callback, times(1)).onBufferAvailable(any(byte[].class)); - reset(callback); - - manager = new FrameManager(5, callback); - manager.setUp(ImageFormat.NV21, new Size(50, 50)); - verify(callback, times(5)).onBufferAvailable(any(byte[].class)); - } - @Test public void testFrameRecycling() { // A 1-pool manager will always recycle the same frame. - FrameManager manager = new FrameManager(1, callback); + FrameManager manager = new FrameManager(1, String.class) { + @Override + protected void onFrameDataRecycled(@NonNull String data) { } + }; manager.setUp(ImageFormat.NV21, new Size(50, 50)); - Frame first = manager.getFrame(null, 0, 0); + Frame first = manager.getFrame("foo", 0, 0); first.release(); - - Frame second = manager.getFrame(null, 0, 0); + Frame second = manager.getFrame("bar", 0, 0); second.release(); - assertEquals(first, second); } - - @Test - public void testOnFrameReleased_alreadyFull() { - FrameManager manager = new FrameManager(1, callback); - int length = manager.setUp(ImageFormat.NV21, new Size(50, 50)); - - Frame frame1 = manager.getFrame(new byte[length], 0, 0); - // Since frame1 is already taken and poolSize = 1, a new Frame is created. - Frame frame2 = manager.getFrame(new byte[length], 0, 0); - // Release the first frame so it goes back into the pool. - manager.onFrameReleased(frame1, frame1.getData()); - reset(callback); - // Release the second. The pool is already full, so onBufferAvailable should not be called - // since this Frame instance will NOT be reused. - manager.onFrameReleased(frame2, frame2.getData()); - verify(callback, never()).onBufferAvailable(frame2.getData()); - } - - @Test - public void testOnFrameReleased_sameLength() { - FrameManager manager = new FrameManager(1, callback); - int length = manager.setUp(ImageFormat.NV21, new Size(50, 50)); - - // A camera preview frame comes. Request a frame. - byte[] picture = new byte[length]; - Frame frame = manager.getFrame(picture, 0, 0); - - // Release the frame and ensure that onBufferAvailable is called. - reset(callback); - manager.onFrameReleased(frame, frame.getData()); - verify(callback, times(1)).onBufferAvailable(picture); - } - - @Test - public void testOnFrameReleased_differentLength() { - FrameManager manager = new FrameManager(1, callback); - int length = manager.setUp(ImageFormat.NV21, new Size(50, 50)); - - // A camera preview frame comes. Request a frame. - byte[] picture = new byte[length]; - Frame frame = manager.getFrame(picture, 0, 0); - - // Don't release the frame. Change the allocation size. - manager.setUp(ImageFormat.NV16, new Size(15, 15)); - - // Now release the old frame and ensure that onBufferAvailable is NOT called, - // because the released data has wrong length. - manager.onFrameReleased(frame, frame.getData()); - reset(callback); - verify(callback, never()).onBufferAvailable(picture); - } } diff --git a/cameraview/src/main/java/com/otaliastudios/cameraview/engine/Camera1Engine.java b/cameraview/src/main/java/com/otaliastudios/cameraview/engine/Camera1Engine.java index fb9aa9e3..1646f24f 100644 --- a/cameraview/src/main/java/com/otaliastudios/cameraview/engine/Camera1Engine.java +++ b/cameraview/src/main/java/com/otaliastudios/cameraview/engine/Camera1Engine.java @@ -25,6 +25,7 @@ import com.otaliastudios.cameraview.engine.offset.Axis; import com.otaliastudios.cameraview.engine.offset.Reference; import com.otaliastudios.cameraview.engine.options.Camera1Options; import com.otaliastudios.cameraview.engine.orchestrator.CameraState; +import com.otaliastudios.cameraview.frame.ByteBufferFrameManager; import com.otaliastudios.cameraview.frame.Frame; import com.otaliastudios.cameraview.PictureResult; import com.otaliastudios.cameraview.VideoResult; @@ -47,7 +48,6 @@ import com.otaliastudios.cameraview.video.SnapshotVideoRecorder; import java.io.IOException; import java.util.ArrayList; -import java.util.Arrays; import java.util.Collections; import java.util.List; @@ -55,7 +55,7 @@ import java.util.List; public class Camera1Engine extends CameraBaseEngine implements Camera.PreviewCallback, Camera.ErrorCallback, - FrameManager.BufferCallback { + ByteBufferFrameManager.BufferCallback { private static final String JOB_FOCUS_RESET = "focus reset"; private static final String JOB_FOCUS_END = "focus end"; @@ -758,7 +758,13 @@ public class Camera1Engine extends CameraBaseEngine implements @NonNull @Override protected FrameManager instantiateFrameManager() { - return new FrameManager(2, this); + return new ByteBufferFrameManager(2, this); + } + + @NonNull + @Override + public ByteBufferFrameManager getFrameManager() { + return (ByteBufferFrameManager) super.getFrameManager(); } @Override diff --git a/cameraview/src/main/java/com/otaliastudios/cameraview/engine/Camera2Engine.java b/cameraview/src/main/java/com/otaliastudios/cameraview/engine/Camera2Engine.java index 22d5a435..ccb77355 100644 --- a/cameraview/src/main/java/com/otaliastudios/cameraview/engine/Camera2Engine.java +++ b/cameraview/src/main/java/com/otaliastudios/cameraview/engine/Camera2Engine.java @@ -56,6 +56,7 @@ import com.otaliastudios.cameraview.engine.offset.Axis; import com.otaliastudios.cameraview.engine.offset.Reference; import com.otaliastudios.cameraview.engine.options.Camera2Options; import com.otaliastudios.cameraview.engine.orchestrator.CameraState; +import com.otaliastudios.cameraview.frame.ByteBufferFrameManager; import com.otaliastudios.cameraview.frame.Frame; import com.otaliastudios.cameraview.frame.FrameManager; import com.otaliastudios.cameraview.gesture.Gesture; @@ -67,7 +68,6 @@ import com.otaliastudios.cameraview.picture.Snapshot2PictureRecorder; import com.otaliastudios.cameraview.preview.GlCameraPreview; import com.otaliastudios.cameraview.size.AspectRatio; import com.otaliastudios.cameraview.size.Size; -import com.otaliastudios.cameraview.size.SizeSelectors; import com.otaliastudios.cameraview.video.Full2VideoRecorder; import com.otaliastudios.cameraview.video.SnapshotVideoRecorder; @@ -82,7 +82,6 @@ public class Camera2Engine extends CameraBaseEngine implements ImageReader.OnImageAvailableListener, ActionHolder { private static final int FRAME_PROCESSING_FORMAT = ImageFormat.NV21; - private static final int FRAME_PROCESSING_INPUT_FORMAT = ImageFormat.YUV_420_888; @VisibleForTesting static final long METER_TIMEOUT = 2500; private final CameraManager mManager; @@ -336,7 +335,7 @@ public class Camera2Engine extends CameraBaseEngine implements if (streamMap == null) { throw new RuntimeException("StreamConfigurationMap is null. Should not happen."); } - android.util.Size[] sizes = streamMap.getOutputSizes(FRAME_PROCESSING_INPUT_FORMAT); + android.util.Size[] sizes = streamMap.getOutputSizes(getFrameProcessingImageFormat()); List candidates = new ArrayList<>(sizes.length); for (android.util.Size size : sizes) { Size add = new Size(size.getWidth(), size.getHeight()); @@ -546,7 +545,7 @@ public class Camera2Engine extends CameraBaseEngine implements mFrameProcessingReader = ImageReader.newInstance( mFrameProcessingSize.getWidth(), mFrameProcessingSize.getHeight(), - FRAME_PROCESSING_INPUT_FORMAT, + getFrameProcessingImageFormat(), 2); mFrameProcessingReader.setOnImageAvailableListener(this, mFrameConversionHandler.getHandler()); @@ -1408,7 +1407,24 @@ public class Camera2Engine extends CameraBaseEngine implements @NonNull @Override protected FrameManager instantiateFrameManager() { - return new FrameManager(2, null); + return new ByteBufferFrameManager(2, null); + } + + @NonNull + @Override + public ByteBufferFrameManager getFrameManager() { + return (ByteBufferFrameManager) super.getFrameManager(); + } + + /** + * Returns the frame processing image format. + * Defaults to {@link ImageFormat#YUV_420_888}. + * Override at your own risk! + * @return format + */ + @SuppressWarnings("WeakerAccess") + protected int getFrameProcessingImageFormat() { + return ImageFormat.YUV_420_888; } @Override diff --git a/cameraview/src/main/java/com/otaliastudios/cameraview/engine/CameraBaseEngine.java b/cameraview/src/main/java/com/otaliastudios/cameraview/engine/CameraBaseEngine.java index 90871509..eb01ba64 100644 --- a/cameraview/src/main/java/com/otaliastudios/cameraview/engine/CameraBaseEngine.java +++ b/cameraview/src/main/java/com/otaliastudios/cameraview/engine/CameraBaseEngine.java @@ -126,7 +126,7 @@ public abstract class CameraBaseEngine extends CameraEngine { @NonNull @Override - public final FrameManager getFrameManager() { + public FrameManager getFrameManager() { return mFrameManager; } diff --git a/cameraview/src/main/java/com/otaliastudios/cameraview/frame/ByteBufferFrameManager.java b/cameraview/src/main/java/com/otaliastudios/cameraview/frame/ByteBufferFrameManager.java new file mode 100644 index 00000000..424d1ac0 --- /dev/null +++ b/cameraview/src/main/java/com/otaliastudios/cameraview/frame/ByteBufferFrameManager.java @@ -0,0 +1,154 @@ +package com.otaliastudios.cameraview.frame; + + +import androidx.annotation.NonNull; +import androidx.annotation.Nullable; + +import com.otaliastudios.cameraview.size.Size; + +import java.util.concurrent.LinkedBlockingQueue; + +/** + * This class manages the allocation of byte buffers and {@link Frame} objects. + * We are interested in recycling both of them, especially byte[] buffers which can create a lot + * of overhead. + * + * The pool size applies to both the {@link Frame} pool and the byte[] pool - it makes sense to use + * the same number since they are consumed at the same time. + * + * We can work in two modes, depending on whether a + * {@link BufferCallback} is passed to the constructor. The modes changes the buffer behavior. + * + * 1. {@link #BUFFER_MODE_DISPATCH}: in this mode, as soon as we have a buffer, it is dispatched to + * the {@link BufferCallback}. The callback should then fill the buffer, and finally call + * {@link #getFrame(byte[], long, int)} to receive a frame. + * This is used for Camera1. + * + * 2. {@link #BUFFER_MODE_ENQUEUE}: in this mode, the manager internally keeps a queue of byte + * buffers, instead of handing them to the callback. The users can ask for buffers through + * {@link #getBuffer()}. + * This buffer can be filled with data and used to get a frame + * {@link #getFrame(byte[], long, int)}, or, in case it was not filled, returned to the queue + * using {@link #onBufferUnused(byte[])}. + * This is used for Camera2. + */ +public class ByteBufferFrameManager extends FrameManager { + + /** + * Receives callbacks on buffer availability + * (when a Frame is released, we reuse its buffer). + */ + public interface BufferCallback { + void onBufferAvailable(@NonNull byte[] buffer); + } + + /** + * In this mode, we have a {@link #mBufferCallback} and dispatch + * new buffers to the callback. + */ + private final static int BUFFER_MODE_DISPATCH = 0; + + /** + * In this mode, we have a {@link #mBufferQueue} where we store + * buffers and only dispatch when requested. + */ + private final static int BUFFER_MODE_ENQUEUE = 1; + + private LinkedBlockingQueue mBufferQueue; + private BufferCallback mBufferCallback; + private final int mBufferMode; + + /** + * Construct a new frame manager. + * The construction must be followed by an {@link #setUp(int, Size)} call + * as soon as the parameters are known. + * + * @param poolSize the size of the backing pool. + * @param callback a callback + */ + public ByteBufferFrameManager(int poolSize, @Nullable BufferCallback callback) { + super(poolSize, byte[].class); + if (callback != null) { + mBufferCallback = callback; + mBufferMode = BUFFER_MODE_DISPATCH; + } else { + mBufferQueue = new LinkedBlockingQueue<>(poolSize); + mBufferMode = BUFFER_MODE_ENQUEUE; + } + } + + + @Override + public void setUp(int format, @NonNull Size size) { + super.setUp(format, size); + int bytes = getFrameBytes(); + for (int i = 0; i < getPoolSize(); i++) { + if (mBufferMode == BUFFER_MODE_DISPATCH) { + mBufferCallback.onBufferAvailable(new byte[bytes]); + } else { + mBufferQueue.offer(new byte[bytes]); + } + } + } + + /** + * Returns a new byte buffer than can be filled. + * This can only be called in {@link #BUFFER_MODE_ENQUEUE} mode! Where the frame + * manager also holds a queue of the byte buffers. + * + * If not null, the buffer returned by this method can be filled and used to get + * a new frame through {@link #getFrame(byte[], long, int)}. + * + * @return a buffer, or null + */ + @Nullable + public byte[] getBuffer() { + if (mBufferMode != BUFFER_MODE_ENQUEUE) { + throw new IllegalStateException("Can't call getBuffer() " + + "when not in BUFFER_MODE_ENQUEUE."); + } + return mBufferQueue.poll(); + } + + /** + * Can be called if the buffer obtained by {@link #getBuffer()} + * was not used to construct a frame, so it can be put back into the queue. + * @param buffer a buffer + */ + public void onBufferUnused(@NonNull byte[] buffer) { + if (mBufferMode != BUFFER_MODE_ENQUEUE) { + throw new IllegalStateException("Can't call onBufferUnused() " + + "when not in BUFFER_MODE_ENQUEUE."); + } + + if (isSetUp()) { + mBufferQueue.offer(buffer); + } else { + LOG.w("onBufferUnused: buffer was returned but we're not set up anymore."); + } + } + + @Override + protected void onFrameDataRecycled(@NonNull byte[] data) { + if (data.length == getFrameBytes()) { + if (mBufferMode == BUFFER_MODE_DISPATCH) { + mBufferCallback.onBufferAvailable(data); + } else { + mBufferQueue.offer(data); + } + } + } + + /** + * Releases all frames controlled by this manager and + * clears the pool. + * In BUFFER_MODE_ENQUEUE, releases also all the buffers. + */ + @Override + public void release() { + super.release(); + if (mBufferMode == BUFFER_MODE_ENQUEUE) { + mBufferQueue.clear(); + } + } +} diff --git a/cameraview/src/main/java/com/otaliastudios/cameraview/frame/Frame.java b/cameraview/src/main/java/com/otaliastudios/cameraview/frame/Frame.java index 8ee2f97c..131f8fcc 100644 --- a/cameraview/src/main/java/com/otaliastudios/cameraview/frame/Frame.java +++ b/cameraview/src/main/java/com/otaliastudios/cameraview/frame/Frame.java @@ -1,6 +1,10 @@ package com.otaliastudios.cameraview.frame; +import android.annotation.SuppressLint; +import android.media.Image; + import com.otaliastudios.cameraview.CameraLogger; +import com.otaliastudios.cameraview.controls.Engine; import com.otaliastudios.cameraview.size.Size; import androidx.annotation.NonNull; @@ -14,19 +18,21 @@ public class Frame { private final static CameraLogger LOG = CameraLogger.create(TAG); private final FrameManager mManager; + private final Class mDataClass; - private byte[] mData = null; + private Object mData = null; private long mTime = -1; private long mLastTime = -1; private int mRotation = 0; private Size mSize = null; private int mFormat = -1; - Frame(@NonNull FrameManager manager) { + Frame(@NonNull FrameManager manager, @NonNull Class dataClass) { mManager = manager; + mDataClass = dataClass; } - void setContent(@NonNull byte[] data, long time, int rotation, @NonNull Size size, int format) { + void setContent(@NonNull Object data, long time, int rotation, @NonNull Size size, int format) { this.mData = data; this.mTime = time; this.mLastTime = time; @@ -63,13 +69,23 @@ public class Frame { * * @return a frozen Frame */ + @SuppressLint("NewApi") @NonNull public Frame freeze() { ensureHasContent(); - byte[] data = new byte[mData.length]; - System.arraycopy(mData, 0, data, 0, mData.length); - Frame other = new Frame(mManager); - other.setContent(data, mTime, mRotation, mSize, mFormat); + Frame other = new Frame(mManager, mDataClass); + if (getDataClass() == byte[].class) { + byte[] input = getData(); + byte[] output = new byte[input.length]; + System.arraycopy(input, 0, output, 0, input.length); + other.setContent(output, mTime, mRotation, mSize, mFormat); + } else if (getDataClass() == Image.class) { + throw new RuntimeException("Cannot freeze() a Image Frame. " + + "Please consider using the frame synchronously in your process() method, " + + "which also gives better performance."); + } else { + throw new RuntimeException("Unknown data class: " + getDataClass()); + } return other; } @@ -80,7 +96,7 @@ public class Frame { public void release() { if (!hasContent()) return; LOG.v("Frame with time", mTime, "is being released."); - byte[] data = mData; + Object data = mData; mData = null; mRotation = 0; mTime = -1; @@ -89,6 +105,7 @@ public class Frame { // After the manager is notified, this frame instance can be taken by // someone else, possibly from another thread. So this should be the // last call in this method. If we null data after, we can have issues. + //noinspection unchecked mManager.onFrameReleased(this, data); } @@ -96,10 +113,24 @@ public class Frame { * Returns the frame data. * @return the frame data */ + @SuppressWarnings("unchecked") @NonNull - public byte[] getData() { + public T getData() { ensureHasContent(); - return mData; + return (T) mData; + } + + /** + * Returns the class returned by {@link #getData()}. + * This class depends on the engine that produced this frame. + * - {@link Engine#CAMERA1} will produce byte[] arrays + * - {@link Engine#CAMERA2} will produce {@link android.media.Image}s + * @return the data class + */ + @SuppressWarnings("WeakerAccess") + @NonNull + public Class getDataClass() { + return mDataClass; } /** diff --git a/cameraview/src/main/java/com/otaliastudios/cameraview/frame/FrameManager.java b/cameraview/src/main/java/com/otaliastudios/cameraview/frame/FrameManager.java index 55f2a5e5..bb47fe30 100644 --- a/cameraview/src/main/java/com/otaliastudios/cameraview/frame/FrameManager.java +++ b/cameraview/src/main/java/com/otaliastudios/cameraview/frame/FrameManager.java @@ -12,71 +12,29 @@ import androidx.annotation.Nullable; import java.util.concurrent.LinkedBlockingQueue; /** - * This class manages the allocation of byte buffers and {@link Frame} objects. - * We are interested in recycling both of them, especially byte[] buffers which can create a lot - * of overhead. - * + * This class manages the allocation of {@link Frame} objects. * The FrameManager keeps a {@link #mPoolSize} integer that defines the number of instances to keep. - * The pool size applies to both the {@link Frame} pool and the byte[] pool - it makes sense to use - * the same number since they are consumed at the same time. * * Main methods are: * - {@link #setUp(int, Size)}: to set up with size and allocate buffers * - {@link #release()}: to release. After release, a manager can be setUp again. - * - {@link #getFrame(byte[], long, int)}: gets a new {@link Frame}. + * - {@link #getFrame(Object, long, int)}: gets a new {@link Frame}. * - * For both byte buffers and frames to get back to the FrameManager pool, all you have to do + * For frames to get back to the FrameManager pool, all you have to do * is call {@link Frame#release()} when done. - * - * Other than this, the FrameManager can work in two modes, depending on whether a - * {@link BufferCallback} is passed to the constructor. The modes changes the buffer behavior. - * - * 1. {@link #BUFFER_MODE_DISPATCH}: in this mode, as soon as we have a buffer, it is dispatched to - * the {@link BufferCallback}. The callback should then fill the buffer, and finally call - * {@link #getFrame(byte[], long, int)} to receive a frame. - * This is used for Camera1. - * - * 2. {@link #BUFFER_MODE_ENQUEUE}: in this mode, the manager internally keeps a queue of byte - * buffers, instead of handing them to the callback. The users can ask for buffers through - * {@link #getBuffer()}. - * This buffer can be filled with data and used to get a frame - * {@link #getFrame(byte[], long, int)}, or, in case it was not filled, returned to the queue - * using {@link #onBufferUnused(byte[])}. - * This is used for Camera2. */ -public class FrameManager { +public abstract class FrameManager { private static final String TAG = FrameManager.class.getSimpleName(); - private static final CameraLogger LOG = CameraLogger.create(TAG); - - /** - * Receives callbacks on buffer availability - * (when a Frame is released, we reuse its buffer). - */ - public interface BufferCallback { - void onBufferAvailable(@NonNull byte[] buffer); - } + protected static final CameraLogger LOG = CameraLogger.create(TAG); private final int mPoolSize; - private int mBufferSize = -1; + private int mFrameBytes = -1; private Size mFrameSize = null; private int mFrameFormat = -1; + private final Class mFrameDataClass; private LinkedBlockingQueue mFrameQueue; - private LinkedBlockingQueue mBufferQueue; - private BufferCallback mBufferCallback; - private final int mBufferMode; - - /** - * In this mode, we have a {@link #mBufferCallback} and dispatch - * new buffers to the callback. - */ - private final static int BUFFER_MODE_DISPATCH = 0; - /** - * In this mode, we have a {@link #mBufferQueue} where we store - * buffers and only dispatch when requested. - */ - private final static int BUFFER_MODE_ENQUEUE = 1; /** * Construct a new frame manager. @@ -84,18 +42,29 @@ public class FrameManager { * as soon as the parameters are known. * * @param poolSize the size of the backing pool. - * @param callback a callback */ - public FrameManager(int poolSize, @Nullable BufferCallback callback) { + protected FrameManager(int poolSize, @NonNull Class dataClass) { mPoolSize = poolSize; + mFrameDataClass = dataClass; mFrameQueue = new LinkedBlockingQueue<>(mPoolSize); - if (callback != null) { - mBufferCallback = callback; - mBufferMode = BUFFER_MODE_DISPATCH; - } else { - mBufferQueue = new LinkedBlockingQueue<>(mPoolSize); - mBufferMode = BUFFER_MODE_ENQUEUE; - } + } + + /** + * Returns the pool size. + * @return pool size + */ + @SuppressWarnings("WeakerAccess") + public final int getPoolSize() { + return mPoolSize; + } + + /** + * Returns the frame size in bytes. + * @return frame size in bytes + */ + @SuppressWarnings("WeakerAccess") + public final int getFrameBytes() { + return mFrameBytes; } /** @@ -106,9 +75,8 @@ public class FrameManager { * * @param format the image format * @param size the frame size - * @return the buffer size */ - public int setUp(int format, @NonNull Size size) { + public void setUp(int format, @NonNull Size size) { if (isSetUp()) { // TODO throw or just reconfigure? } @@ -116,15 +84,7 @@ public class FrameManager { mFrameFormat = format; int bitsPerPixel = ImageFormat.getBitsPerPixel(format); long sizeInBits = size.getHeight() * size.getWidth() * bitsPerPixel; - mBufferSize = (int) Math.ceil(sizeInBits / 8.0d); - for (int i = 0; i < mPoolSize; i++) { - if (mBufferMode == BUFFER_MODE_DISPATCH) { - mBufferCallback.onBufferAvailable(new byte[mBufferSize]); - } else { - mBufferQueue.offer(new byte[mBufferSize]); - } - } - return mBufferSize; + mFrameBytes = (int) Math.ceil(sizeInBits / 8.0d); } /** @@ -134,47 +94,10 @@ public class FrameManager { * * @return true if set up */ - private boolean isSetUp() { + protected boolean isSetUp() { return mFrameSize != null; } - /** - * Returns a new byte buffer than can be filled. - * This can only be called in {@link #BUFFER_MODE_ENQUEUE} mode! Where the frame - * manager also holds a queue of the byte buffers. - * - * If not null, the buffer returned by this method can be filled and used to get - * a new frame through {@link #getFrame(byte[], long, int)}. - * - * @return a buffer, or null - */ - @Nullable - public byte[] getBuffer() { - if (mBufferMode != BUFFER_MODE_ENQUEUE) { - throw new IllegalStateException("Can't call getBuffer() " + - "when not in BUFFER_MODE_ENQUEUE."); - } - return mBufferQueue.poll(); - } - - /** - * Can be called if the buffer obtained by {@link #getBuffer()} - * was not used to construct a frame, so it can be put back into the queue. - * @param buffer a buffer - */ - public void onBufferUnused(@NonNull byte[] buffer) { - if (mBufferMode != BUFFER_MODE_ENQUEUE) { - throw new IllegalStateException("Can't call onBufferUnused() " + - "when not in BUFFER_MODE_ENQUEUE."); - } - - if (isSetUp()) { - mBufferQueue.offer(buffer); - } else { - LOG.w("onBufferUnused: buffer was returned but we're not set up anymore."); - } - } - /** * Returns a new Frame for the given data. This must be called * - after {@link #setUp(int, Size)}, which sets the buffer size @@ -190,7 +113,7 @@ public class FrameManager { * @return a new frame */ @NonNull - public Frame getFrame(@NonNull byte[] data, long time, int rotation) { + public Frame getFrame(@NonNull T data, long time, int rotation) { if (!isSetUp()) { throw new IllegalStateException("Can't call getFrame() after releasing " + "or before setUp."); @@ -201,7 +124,7 @@ public class FrameManager { LOG.v("getFrame for time:", time, "RECYCLING."); } else { LOG.v("getFrame for time:", time, "CREATING."); - frame = new Frame(this); + frame = new Frame(this, mFrameDataClass); } frame.setContent(data, time, rotation, mFrameSize, mFrameFormat); return frame; @@ -209,33 +132,29 @@ public class FrameManager { /** * Called by child frames when they are released. - * This might be called from old Frames that belong to an old 'setUp' - * of this FrameManager instance. So the buffer size might be different, - * for instance. - * * @param frame the released frame */ - void onFrameReleased(@NonNull Frame frame, @NonNull byte[] buffer) { + void onFrameReleased(@NonNull Frame frame, @NonNull T data) { if (!isSetUp()) return; // If frame queue is full, let's drop everything. // If frame queue accepts this frame, let's recycle the buffer as well. if (mFrameQueue.offer(frame)) { - int currSize = buffer.length; - int reqSize = mBufferSize; - if (currSize == reqSize) { - if (mBufferMode == BUFFER_MODE_DISPATCH) { - mBufferCallback.onBufferAvailable(buffer); - } else { - mBufferQueue.offer(buffer); - } - } + onFrameDataRecycled(data); } } + /** + * Called when a Frame was recycled and its data is now available. + * This might be called from old Frames that belong to an old 'setUp' + * of this FrameManager instance. So the buffer size might be different, + * for instance. + * @param data data + */ + protected abstract void onFrameDataRecycled(@NonNull T data); + /** * Releases all frames controlled by this manager and * clears the pool. - * In BUFFER_MODE_ENQUEUE, releases also all the buffers. */ public void release() { if (!isSetUp()) { @@ -245,10 +164,7 @@ public class FrameManager { LOG.i("release: Clearing the frame and buffer queue."); mFrameQueue.clear(); - if (mBufferMode == BUFFER_MODE_ENQUEUE) { - mBufferQueue.clear(); - } - mBufferSize = -1; + mFrameBytes = -1; mFrameSize = null; mFrameFormat = -1; } diff --git a/demo/src/main/java/com/otaliastudios/cameraview/demo/CameraActivity.java b/demo/src/main/java/com/otaliastudios/cameraview/demo/CameraActivity.java index 97923046..25a311c0 100644 --- a/demo/src/main/java/com/otaliastudios/cameraview/demo/CameraActivity.java +++ b/demo/src/main/java/com/otaliastudios/cameraview/demo/CameraActivity.java @@ -78,18 +78,24 @@ public class CameraActivity extends AppCompatActivity implements View.OnClickLis lastTime = newTime; LOG.e("Frame delayMillis:", delay, "FPS:", 1000 / delay); if (DECODE_BITMAP) { - YuvImage yuvImage = new YuvImage(frame.getData(), ImageFormat.NV21, - frame.getSize().getWidth(), - frame.getSize().getHeight(), - null); - ByteArrayOutputStream jpegStream = new ByteArrayOutputStream(); - yuvImage.compressToJpeg(new Rect(0, 0, - frame.getSize().getWidth(), - frame.getSize().getHeight()), 100, jpegStream); - byte[] jpegByteArray = jpegStream.toByteArray(); - Bitmap bitmap = BitmapFactory.decodeByteArray(jpegByteArray, 0, jpegByteArray.length); - //noinspection ResultOfMethodCallIgnored - bitmap.toString(); + if (frame.getFormat() == ImageFormat.NV21 + && frame.getDataClass() == byte[].class) { + byte[] data = frame.getData(); + YuvImage yuvImage = new YuvImage(data, + frame.getFormat(), + frame.getSize().getWidth(), + frame.getSize().getHeight(), + null); + ByteArrayOutputStream jpegStream = new ByteArrayOutputStream(); + yuvImage.compressToJpeg(new Rect(0, 0, + frame.getSize().getWidth(), + frame.getSize().getHeight()), 100, jpegStream); + byte[] jpegByteArray = jpegStream.toByteArray(); + Bitmap bitmap = BitmapFactory.decodeByteArray(jpegByteArray, + 0, jpegByteArray.length); + //noinspection ResultOfMethodCallIgnored + bitmap.toString(); + } } } });