Prepare Frame for Images, abstract FrameManager, create ByteBufferFrameManager

pull/704/head
Mattia Iavarone 6 years ago
parent cf1055e776
commit 28023c72eb
  1. 14
      cameraview/src/androidTest/java/com/otaliastudios/cameraview/CameraViewTest.java
  2. 3
      cameraview/src/androidTest/java/com/otaliastudios/cameraview/engine/MockCameraEngine.java
  3. 105
      cameraview/src/androidTest/java/com/otaliastudios/cameraview/frame/ByteBufferFrameManagerTest.java
  4. 87
      cameraview/src/androidTest/java/com/otaliastudios/cameraview/frame/FrameManagerTest.java
  5. 12
      cameraview/src/main/java/com/otaliastudios/cameraview/engine/Camera1Engine.java
  6. 26
      cameraview/src/main/java/com/otaliastudios/cameraview/engine/Camera2Engine.java
  7. 2
      cameraview/src/main/java/com/otaliastudios/cameraview/engine/CameraBaseEngine.java
  8. 154
      cameraview/src/main/java/com/otaliastudios/cameraview/frame/ByteBufferFrameManager.java
  9. 51
      cameraview/src/main/java/com/otaliastudios/cameraview/frame/Frame.java
  10. 170
      cameraview/src/main/java/com/otaliastudios/cameraview/frame/FrameManager.java
  11. 10
      demo/src/main/java/com/otaliastudios/cameraview/demo/CameraActivity.java

@ -996,22 +996,8 @@ public class CameraViewTest extends BaseTest {
//region Filter //region Filter
@Test(expected = RuntimeException.class)
public void testSetFilter_notExperimental() {
cameraView.setExperimental(false);
cameraView.setFilter(Filters.AUTO_FIX.newInstance());
}
@Test
public void testSetFilter_notExperimental_noFilter() {
cameraView.setExperimental(false);
cameraView.setFilter(Filters.NONE.newInstance());
// no exception thrown
}
@Test @Test
public void testSetFilter() { public void testSetFilter() {
cameraView.setExperimental(true);
Filter filter = Filters.AUTO_FIX.newInstance(); Filter filter = Filters.AUTO_FIX.newInstance();
cameraView.setFilter(filter); cameraView.setFilter(filter);
verify(mockPreview, times(1)).setFilter(filter); verify(mockPreview, times(1)).setFilter(filter);

@ -13,6 +13,7 @@ import com.otaliastudios.cameraview.controls.Facing;
import com.otaliastudios.cameraview.controls.Flash; import com.otaliastudios.cameraview.controls.Flash;
import com.otaliastudios.cameraview.controls.PictureFormat; import com.otaliastudios.cameraview.controls.PictureFormat;
import com.otaliastudios.cameraview.engine.orchestrator.CameraState; import com.otaliastudios.cameraview.engine.orchestrator.CameraState;
import com.otaliastudios.cameraview.frame.ByteBufferFrameManager;
import com.otaliastudios.cameraview.frame.FrameManager; import com.otaliastudios.cameraview.frame.FrameManager;
import com.otaliastudios.cameraview.gesture.Gesture; import com.otaliastudios.cameraview.gesture.Gesture;
import com.otaliastudios.cameraview.controls.Hdr; import com.otaliastudios.cameraview.controls.Hdr;
@ -185,7 +186,7 @@ public class MockCameraEngine extends CameraBaseEngine {
@NonNull @NonNull
@Override @Override
protected FrameManager instantiateFrameManager() { protected FrameManager instantiateFrameManager() {
return new FrameManager(2, null); return new ByteBufferFrameManager(2, null);
} }
@Override @Override

@ -0,0 +1,105 @@
package com.otaliastudios.cameraview.frame;
import android.graphics.ImageFormat;
import androidx.test.ext.junit.runners.AndroidJUnit4;
import androidx.test.filters.SmallTest;
import com.otaliastudios.cameraview.BaseTest;
import com.otaliastudios.cameraview.size.Size;
import org.junit.After;
import org.junit.Before;
import org.junit.Test;
import org.junit.runner.RunWith;
import static org.mockito.Matchers.any;
import static org.mockito.Mockito.mock;
import static org.mockito.Mockito.never;
import static org.mockito.Mockito.reset;
import static org.mockito.Mockito.times;
import static org.mockito.Mockito.verify;
@RunWith(AndroidJUnit4.class)
@SmallTest
public class ByteBufferFrameManagerTest extends BaseTest {
private ByteBufferFrameManager.BufferCallback callback;
@Before
public void setUp() {
callback = mock(ByteBufferFrameManager.BufferCallback.class);
}
@After
public void tearDown() {
callback = null;
}
@Test
public void testAllocate() {
ByteBufferFrameManager manager = new ByteBufferFrameManager(1, callback);
manager.setUp(ImageFormat.NV21, new Size(50, 50));
verify(callback, times(1)).onBufferAvailable(any(byte[].class));
reset(callback);
manager = new ByteBufferFrameManager(5, callback);
manager.setUp(ImageFormat.NV21, new Size(50, 50));
verify(callback, times(5)).onBufferAvailable(any(byte[].class));
}
@Test
public void testOnFrameReleased_alreadyFull() {
ByteBufferFrameManager manager = new ByteBufferFrameManager(1, callback);
manager.setUp(ImageFormat.NV21, new Size(50, 50));
int length = manager.getFrameBytes();
Frame frame1 = manager.getFrame(new byte[length], 0, 0);
// Since frame1 is already taken and poolSize = 1, a new Frame is created.
Frame frame2 = manager.getFrame(new byte[length], 0, 0);
// Release the first frame so it goes back into the pool.
manager.onFrameReleased(frame1, (byte[]) frame1.getData());
reset(callback);
// Release the second. The pool is already full, so onBufferAvailable should not be called
// since this Frame instance will NOT be reused.
manager.onFrameReleased(frame2, (byte[]) frame2.getData());
verify(callback, never()).onBufferAvailable((byte[]) frame2.getData());
}
@Test
public void testOnFrameReleased_sameLength() {
ByteBufferFrameManager manager = new ByteBufferFrameManager(1, callback);
manager.setUp(ImageFormat.NV21, new Size(50, 50));
int length = manager.getFrameBytes();
// A camera preview frame comes. Request a frame.
byte[] picture = new byte[length];
Frame frame = manager.getFrame(picture, 0, 0);
// Release the frame and ensure that onBufferAvailable is called.
reset(callback);
manager.onFrameReleased(frame, (byte[]) frame.getData());
verify(callback, times(1)).onBufferAvailable(picture);
}
@Test
public void testOnFrameReleased_differentLength() {
ByteBufferFrameManager manager = new ByteBufferFrameManager(1, callback);
manager.setUp(ImageFormat.NV21, new Size(50, 50));
int length = manager.getFrameBytes();
// A camera preview frame comes. Request a frame.
byte[] picture = new byte[length];
Frame frame = manager.getFrame(picture, 0, 0);
// Don't release the frame. Change the allocation size.
manager.setUp(ImageFormat.NV16, new Size(15, 15));
// Now release the old frame and ensure that onBufferAvailable is NOT called,
// because the released data has wrong length.
manager.onFrameReleased(frame, (byte[]) frame.getData());
reset(callback);
verify(callback, never()).onBufferAvailable(picture);
}
}

@ -3,6 +3,7 @@ package com.otaliastudios.cameraview.frame;
import android.graphics.ImageFormat; import android.graphics.ImageFormat;
import androidx.annotation.NonNull;
import androidx.test.ext.junit.runners.AndroidJUnit4; import androidx.test.ext.junit.runners.AndroidJUnit4;
import androidx.test.filters.SmallTest; import androidx.test.filters.SmallTest;
@ -26,93 +27,19 @@ import static org.mockito.Mockito.verify;
@SmallTest @SmallTest
public class FrameManagerTest extends BaseTest { public class FrameManagerTest extends BaseTest {
private FrameManager.BufferCallback callback;
@Before
public void setUp() {
callback = mock(FrameManager.BufferCallback.class);
}
@After
public void tearDown() {
callback = null;
}
@Test
public void testAllocate() {
FrameManager manager = new FrameManager(1, callback);
manager.setUp(ImageFormat.NV21, new Size(50, 50));
verify(callback, times(1)).onBufferAvailable(any(byte[].class));
reset(callback);
manager = new FrameManager(5, callback);
manager.setUp(ImageFormat.NV21, new Size(50, 50));
verify(callback, times(5)).onBufferAvailable(any(byte[].class));
}
@Test @Test
public void testFrameRecycling() { public void testFrameRecycling() {
// A 1-pool manager will always recycle the same frame. // A 1-pool manager will always recycle the same frame.
FrameManager manager = new FrameManager(1, callback); FrameManager<String> manager = new FrameManager<String>(1, String.class) {
@Override
protected void onFrameDataRecycled(@NonNull String data) { }
};
manager.setUp(ImageFormat.NV21, new Size(50, 50)); manager.setUp(ImageFormat.NV21, new Size(50, 50));
Frame first = manager.getFrame(null, 0, 0); Frame first = manager.getFrame("foo", 0, 0);
first.release(); first.release();
Frame second = manager.getFrame("bar", 0, 0);
Frame second = manager.getFrame(null, 0, 0);
second.release(); second.release();
assertEquals(first, second); assertEquals(first, second);
} }
@Test
public void testOnFrameReleased_alreadyFull() {
FrameManager manager = new FrameManager(1, callback);
int length = manager.setUp(ImageFormat.NV21, new Size(50, 50));
Frame frame1 = manager.getFrame(new byte[length], 0, 0);
// Since frame1 is already taken and poolSize = 1, a new Frame is created.
Frame frame2 = manager.getFrame(new byte[length], 0, 0);
// Release the first frame so it goes back into the pool.
manager.onFrameReleased(frame1, frame1.getData());
reset(callback);
// Release the second. The pool is already full, so onBufferAvailable should not be called
// since this Frame instance will NOT be reused.
manager.onFrameReleased(frame2, frame2.getData());
verify(callback, never()).onBufferAvailable(frame2.getData());
}
@Test
public void testOnFrameReleased_sameLength() {
FrameManager manager = new FrameManager(1, callback);
int length = manager.setUp(ImageFormat.NV21, new Size(50, 50));
// A camera preview frame comes. Request a frame.
byte[] picture = new byte[length];
Frame frame = manager.getFrame(picture, 0, 0);
// Release the frame and ensure that onBufferAvailable is called.
reset(callback);
manager.onFrameReleased(frame, frame.getData());
verify(callback, times(1)).onBufferAvailable(picture);
}
@Test
public void testOnFrameReleased_differentLength() {
FrameManager manager = new FrameManager(1, callback);
int length = manager.setUp(ImageFormat.NV21, new Size(50, 50));
// A camera preview frame comes. Request a frame.
byte[] picture = new byte[length];
Frame frame = manager.getFrame(picture, 0, 0);
// Don't release the frame. Change the allocation size.
manager.setUp(ImageFormat.NV16, new Size(15, 15));
// Now release the old frame and ensure that onBufferAvailable is NOT called,
// because the released data has wrong length.
manager.onFrameReleased(frame, frame.getData());
reset(callback);
verify(callback, never()).onBufferAvailable(picture);
}
} }

@ -25,6 +25,7 @@ import com.otaliastudios.cameraview.engine.offset.Axis;
import com.otaliastudios.cameraview.engine.offset.Reference; import com.otaliastudios.cameraview.engine.offset.Reference;
import com.otaliastudios.cameraview.engine.options.Camera1Options; import com.otaliastudios.cameraview.engine.options.Camera1Options;
import com.otaliastudios.cameraview.engine.orchestrator.CameraState; import com.otaliastudios.cameraview.engine.orchestrator.CameraState;
import com.otaliastudios.cameraview.frame.ByteBufferFrameManager;
import com.otaliastudios.cameraview.frame.Frame; import com.otaliastudios.cameraview.frame.Frame;
import com.otaliastudios.cameraview.PictureResult; import com.otaliastudios.cameraview.PictureResult;
import com.otaliastudios.cameraview.VideoResult; import com.otaliastudios.cameraview.VideoResult;
@ -47,7 +48,6 @@ import com.otaliastudios.cameraview.video.SnapshotVideoRecorder;
import java.io.IOException; import java.io.IOException;
import java.util.ArrayList; import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collections; import java.util.Collections;
import java.util.List; import java.util.List;
@ -55,7 +55,7 @@ import java.util.List;
public class Camera1Engine extends CameraBaseEngine implements public class Camera1Engine extends CameraBaseEngine implements
Camera.PreviewCallback, Camera.PreviewCallback,
Camera.ErrorCallback, Camera.ErrorCallback,
FrameManager.BufferCallback { ByteBufferFrameManager.BufferCallback {
private static final String JOB_FOCUS_RESET = "focus reset"; private static final String JOB_FOCUS_RESET = "focus reset";
private static final String JOB_FOCUS_END = "focus end"; private static final String JOB_FOCUS_END = "focus end";
@ -758,7 +758,13 @@ public class Camera1Engine extends CameraBaseEngine implements
@NonNull @NonNull
@Override @Override
protected FrameManager instantiateFrameManager() { protected FrameManager instantiateFrameManager() {
return new FrameManager(2, this); return new ByteBufferFrameManager(2, this);
}
@NonNull
@Override
public ByteBufferFrameManager getFrameManager() {
return (ByteBufferFrameManager) super.getFrameManager();
} }
@Override @Override

@ -56,6 +56,7 @@ import com.otaliastudios.cameraview.engine.offset.Axis;
import com.otaliastudios.cameraview.engine.offset.Reference; import com.otaliastudios.cameraview.engine.offset.Reference;
import com.otaliastudios.cameraview.engine.options.Camera2Options; import com.otaliastudios.cameraview.engine.options.Camera2Options;
import com.otaliastudios.cameraview.engine.orchestrator.CameraState; import com.otaliastudios.cameraview.engine.orchestrator.CameraState;
import com.otaliastudios.cameraview.frame.ByteBufferFrameManager;
import com.otaliastudios.cameraview.frame.Frame; import com.otaliastudios.cameraview.frame.Frame;
import com.otaliastudios.cameraview.frame.FrameManager; import com.otaliastudios.cameraview.frame.FrameManager;
import com.otaliastudios.cameraview.gesture.Gesture; import com.otaliastudios.cameraview.gesture.Gesture;
@ -67,7 +68,6 @@ import com.otaliastudios.cameraview.picture.Snapshot2PictureRecorder;
import com.otaliastudios.cameraview.preview.GlCameraPreview; import com.otaliastudios.cameraview.preview.GlCameraPreview;
import com.otaliastudios.cameraview.size.AspectRatio; import com.otaliastudios.cameraview.size.AspectRatio;
import com.otaliastudios.cameraview.size.Size; import com.otaliastudios.cameraview.size.Size;
import com.otaliastudios.cameraview.size.SizeSelectors;
import com.otaliastudios.cameraview.video.Full2VideoRecorder; import com.otaliastudios.cameraview.video.Full2VideoRecorder;
import com.otaliastudios.cameraview.video.SnapshotVideoRecorder; import com.otaliastudios.cameraview.video.SnapshotVideoRecorder;
@ -82,7 +82,6 @@ public class Camera2Engine extends CameraBaseEngine implements
ImageReader.OnImageAvailableListener, ImageReader.OnImageAvailableListener,
ActionHolder { ActionHolder {
private static final int FRAME_PROCESSING_FORMAT = ImageFormat.NV21; private static final int FRAME_PROCESSING_FORMAT = ImageFormat.NV21;
private static final int FRAME_PROCESSING_INPUT_FORMAT = ImageFormat.YUV_420_888;
@VisibleForTesting static final long METER_TIMEOUT = 2500; @VisibleForTesting static final long METER_TIMEOUT = 2500;
private final CameraManager mManager; private final CameraManager mManager;
@ -336,7 +335,7 @@ public class Camera2Engine extends CameraBaseEngine implements
if (streamMap == null) { if (streamMap == null) {
throw new RuntimeException("StreamConfigurationMap is null. Should not happen."); throw new RuntimeException("StreamConfigurationMap is null. Should not happen.");
} }
android.util.Size[] sizes = streamMap.getOutputSizes(FRAME_PROCESSING_INPUT_FORMAT); android.util.Size[] sizes = streamMap.getOutputSizes(getFrameProcessingImageFormat());
List<Size> candidates = new ArrayList<>(sizes.length); List<Size> candidates = new ArrayList<>(sizes.length);
for (android.util.Size size : sizes) { for (android.util.Size size : sizes) {
Size add = new Size(size.getWidth(), size.getHeight()); Size add = new Size(size.getWidth(), size.getHeight());
@ -546,7 +545,7 @@ public class Camera2Engine extends CameraBaseEngine implements
mFrameProcessingReader = ImageReader.newInstance( mFrameProcessingReader = ImageReader.newInstance(
mFrameProcessingSize.getWidth(), mFrameProcessingSize.getWidth(),
mFrameProcessingSize.getHeight(), mFrameProcessingSize.getHeight(),
FRAME_PROCESSING_INPUT_FORMAT, getFrameProcessingImageFormat(),
2); 2);
mFrameProcessingReader.setOnImageAvailableListener(this, mFrameProcessingReader.setOnImageAvailableListener(this,
mFrameConversionHandler.getHandler()); mFrameConversionHandler.getHandler());
@ -1408,7 +1407,24 @@ public class Camera2Engine extends CameraBaseEngine implements
@NonNull @NonNull
@Override @Override
protected FrameManager instantiateFrameManager() { protected FrameManager instantiateFrameManager() {
return new FrameManager(2, null); return new ByteBufferFrameManager(2, null);
}
@NonNull
@Override
public ByteBufferFrameManager getFrameManager() {
return (ByteBufferFrameManager) super.getFrameManager();
}
/**
* Returns the frame processing image format.
* Defaults to {@link ImageFormat#YUV_420_888}.
* Override at your own risk!
* @return format
*/
@SuppressWarnings("WeakerAccess")
protected int getFrameProcessingImageFormat() {
return ImageFormat.YUV_420_888;
} }
@Override @Override

@ -126,7 +126,7 @@ public abstract class CameraBaseEngine extends CameraEngine {
@NonNull @NonNull
@Override @Override
public final FrameManager getFrameManager() { public FrameManager getFrameManager() {
return mFrameManager; return mFrameManager;
} }

@ -0,0 +1,154 @@
package com.otaliastudios.cameraview.frame;
import androidx.annotation.NonNull;
import androidx.annotation.Nullable;
import com.otaliastudios.cameraview.size.Size;
import java.util.concurrent.LinkedBlockingQueue;
/**
* This class manages the allocation of byte buffers and {@link Frame} objects.
* We are interested in recycling both of them, especially byte[] buffers which can create a lot
* of overhead.
*
* The pool size applies to both the {@link Frame} pool and the byte[] pool - it makes sense to use
* the same number since they are consumed at the same time.
*
* We can work in two modes, depending on whether a
* {@link BufferCallback} is passed to the constructor. The modes changes the buffer behavior.
*
* 1. {@link #BUFFER_MODE_DISPATCH}: in this mode, as soon as we have a buffer, it is dispatched to
* the {@link BufferCallback}. The callback should then fill the buffer, and finally call
* {@link #getFrame(byte[], long, int)} to receive a frame.
* This is used for Camera1.
*
* 2. {@link #BUFFER_MODE_ENQUEUE}: in this mode, the manager internally keeps a queue of byte
* buffers, instead of handing them to the callback. The users can ask for buffers through
* {@link #getBuffer()}.
* This buffer can be filled with data and used to get a frame
* {@link #getFrame(byte[], long, int)}, or, in case it was not filled, returned to the queue
* using {@link #onBufferUnused(byte[])}.
* This is used for Camera2.
*/
public class ByteBufferFrameManager extends FrameManager<byte[]> {
/**
* Receives callbacks on buffer availability
* (when a Frame is released, we reuse its buffer).
*/
public interface BufferCallback {
void onBufferAvailable(@NonNull byte[] buffer);
}
/**
* In this mode, we have a {@link #mBufferCallback} and dispatch
* new buffers to the callback.
*/
private final static int BUFFER_MODE_DISPATCH = 0;
/**
* In this mode, we have a {@link #mBufferQueue} where we store
* buffers and only dispatch when requested.
*/
private final static int BUFFER_MODE_ENQUEUE = 1;
private LinkedBlockingQueue<byte[]> mBufferQueue;
private BufferCallback mBufferCallback;
private final int mBufferMode;
/**
* Construct a new frame manager.
* The construction must be followed by an {@link #setUp(int, Size)} call
* as soon as the parameters are known.
*
* @param poolSize the size of the backing pool.
* @param callback a callback
*/
public ByteBufferFrameManager(int poolSize, @Nullable BufferCallback callback) {
super(poolSize, byte[].class);
if (callback != null) {
mBufferCallback = callback;
mBufferMode = BUFFER_MODE_DISPATCH;
} else {
mBufferQueue = new LinkedBlockingQueue<>(poolSize);
mBufferMode = BUFFER_MODE_ENQUEUE;
}
}
@Override
public void setUp(int format, @NonNull Size size) {
super.setUp(format, size);
int bytes = getFrameBytes();
for (int i = 0; i < getPoolSize(); i++) {
if (mBufferMode == BUFFER_MODE_DISPATCH) {
mBufferCallback.onBufferAvailable(new byte[bytes]);
} else {
mBufferQueue.offer(new byte[bytes]);
}
}
}
/**
* Returns a new byte buffer than can be filled.
* This can only be called in {@link #BUFFER_MODE_ENQUEUE} mode! Where the frame
* manager also holds a queue of the byte buffers.
*
* If not null, the buffer returned by this method can be filled and used to get
* a new frame through {@link #getFrame(byte[], long, int)}.
*
* @return a buffer, or null
*/
@Nullable
public byte[] getBuffer() {
if (mBufferMode != BUFFER_MODE_ENQUEUE) {
throw new IllegalStateException("Can't call getBuffer() " +
"when not in BUFFER_MODE_ENQUEUE.");
}
return mBufferQueue.poll();
}
/**
* Can be called if the buffer obtained by {@link #getBuffer()}
* was not used to construct a frame, so it can be put back into the queue.
* @param buffer a buffer
*/
public void onBufferUnused(@NonNull byte[] buffer) {
if (mBufferMode != BUFFER_MODE_ENQUEUE) {
throw new IllegalStateException("Can't call onBufferUnused() " +
"when not in BUFFER_MODE_ENQUEUE.");
}
if (isSetUp()) {
mBufferQueue.offer(buffer);
} else {
LOG.w("onBufferUnused: buffer was returned but we're not set up anymore.");
}
}
@Override
protected void onFrameDataRecycled(@NonNull byte[] data) {
if (data.length == getFrameBytes()) {
if (mBufferMode == BUFFER_MODE_DISPATCH) {
mBufferCallback.onBufferAvailable(data);
} else {
mBufferQueue.offer(data);
}
}
}
/**
* Releases all frames controlled by this manager and
* clears the pool.
* In BUFFER_MODE_ENQUEUE, releases also all the buffers.
*/
@Override
public void release() {
super.release();
if (mBufferMode == BUFFER_MODE_ENQUEUE) {
mBufferQueue.clear();
}
}
}

@ -1,6 +1,10 @@
package com.otaliastudios.cameraview.frame; package com.otaliastudios.cameraview.frame;
import android.annotation.SuppressLint;
import android.media.Image;
import com.otaliastudios.cameraview.CameraLogger; import com.otaliastudios.cameraview.CameraLogger;
import com.otaliastudios.cameraview.controls.Engine;
import com.otaliastudios.cameraview.size.Size; import com.otaliastudios.cameraview.size.Size;
import androidx.annotation.NonNull; import androidx.annotation.NonNull;
@ -14,19 +18,21 @@ public class Frame {
private final static CameraLogger LOG = CameraLogger.create(TAG); private final static CameraLogger LOG = CameraLogger.create(TAG);
private final FrameManager mManager; private final FrameManager mManager;
private final Class<?> mDataClass;
private byte[] mData = null; private Object mData = null;
private long mTime = -1; private long mTime = -1;
private long mLastTime = -1; private long mLastTime = -1;
private int mRotation = 0; private int mRotation = 0;
private Size mSize = null; private Size mSize = null;
private int mFormat = -1; private int mFormat = -1;
Frame(@NonNull FrameManager manager) { Frame(@NonNull FrameManager manager, @NonNull Class<?> dataClass) {
mManager = manager; mManager = manager;
mDataClass = dataClass;
} }
void setContent(@NonNull byte[] data, long time, int rotation, @NonNull Size size, int format) { void setContent(@NonNull Object data, long time, int rotation, @NonNull Size size, int format) {
this.mData = data; this.mData = data;
this.mTime = time; this.mTime = time;
this.mLastTime = time; this.mLastTime = time;
@ -63,13 +69,23 @@ public class Frame {
* *
* @return a frozen Frame * @return a frozen Frame
*/ */
@SuppressLint("NewApi")
@NonNull @NonNull
public Frame freeze() { public Frame freeze() {
ensureHasContent(); ensureHasContent();
byte[] data = new byte[mData.length]; Frame other = new Frame(mManager, mDataClass);
System.arraycopy(mData, 0, data, 0, mData.length); if (getDataClass() == byte[].class) {
Frame other = new Frame(mManager); byte[] input = getData();
other.setContent(data, mTime, mRotation, mSize, mFormat); byte[] output = new byte[input.length];
System.arraycopy(input, 0, output, 0, input.length);
other.setContent(output, mTime, mRotation, mSize, mFormat);
} else if (getDataClass() == Image.class) {
throw new RuntimeException("Cannot freeze() a Image Frame. " +
"Please consider using the frame synchronously in your process() method, " +
"which also gives better performance.");
} else {
throw new RuntimeException("Unknown data class: " + getDataClass());
}
return other; return other;
} }
@ -80,7 +96,7 @@ public class Frame {
public void release() { public void release() {
if (!hasContent()) return; if (!hasContent()) return;
LOG.v("Frame with time", mTime, "is being released."); LOG.v("Frame with time", mTime, "is being released.");
byte[] data = mData; Object data = mData;
mData = null; mData = null;
mRotation = 0; mRotation = 0;
mTime = -1; mTime = -1;
@ -89,6 +105,7 @@ public class Frame {
// After the manager is notified, this frame instance can be taken by // After the manager is notified, this frame instance can be taken by
// someone else, possibly from another thread. So this should be the // someone else, possibly from another thread. So this should be the
// last call in this method. If we null data after, we can have issues. // last call in this method. If we null data after, we can have issues.
//noinspection unchecked
mManager.onFrameReleased(this, data); mManager.onFrameReleased(this, data);
} }
@ -96,10 +113,24 @@ public class Frame {
* Returns the frame data. * Returns the frame data.
* @return the frame data * @return the frame data
*/ */
@SuppressWarnings("unchecked")
@NonNull @NonNull
public byte[] getData() { public <T> T getData() {
ensureHasContent(); ensureHasContent();
return mData; return (T) mData;
}
/**
* Returns the class returned by {@link #getData()}.
* This class depends on the engine that produced this frame.
* - {@link Engine#CAMERA1} will produce byte[] arrays
* - {@link Engine#CAMERA2} will produce {@link android.media.Image}s
* @return the data class
*/
@SuppressWarnings("WeakerAccess")
@NonNull
public Class<?> getDataClass() {
return mDataClass;
} }
/** /**

@ -12,71 +12,29 @@ import androidx.annotation.Nullable;
import java.util.concurrent.LinkedBlockingQueue; import java.util.concurrent.LinkedBlockingQueue;
/** /**
* This class manages the allocation of byte buffers and {@link Frame} objects. * This class manages the allocation of {@link Frame} objects.
* We are interested in recycling both of them, especially byte[] buffers which can create a lot
* of overhead.
*
* The FrameManager keeps a {@link #mPoolSize} integer that defines the number of instances to keep. * The FrameManager keeps a {@link #mPoolSize} integer that defines the number of instances to keep.
* The pool size applies to both the {@link Frame} pool and the byte[] pool - it makes sense to use
* the same number since they are consumed at the same time.
* *
* Main methods are: * Main methods are:
* - {@link #setUp(int, Size)}: to set up with size and allocate buffers * - {@link #setUp(int, Size)}: to set up with size and allocate buffers
* - {@link #release()}: to release. After release, a manager can be setUp again. * - {@link #release()}: to release. After release, a manager can be setUp again.
* - {@link #getFrame(byte[], long, int)}: gets a new {@link Frame}. * - {@link #getFrame(Object, long, int)}: gets a new {@link Frame}.
* *
* For both byte buffers and frames to get back to the FrameManager pool, all you have to do * For frames to get back to the FrameManager pool, all you have to do
* is call {@link Frame#release()} when done. * is call {@link Frame#release()} when done.
*
* Other than this, the FrameManager can work in two modes, depending on whether a
* {@link BufferCallback} is passed to the constructor. The modes changes the buffer behavior.
*
* 1. {@link #BUFFER_MODE_DISPATCH}: in this mode, as soon as we have a buffer, it is dispatched to
* the {@link BufferCallback}. The callback should then fill the buffer, and finally call
* {@link #getFrame(byte[], long, int)} to receive a frame.
* This is used for Camera1.
*
* 2. {@link #BUFFER_MODE_ENQUEUE}: in this mode, the manager internally keeps a queue of byte
* buffers, instead of handing them to the callback. The users can ask for buffers through
* {@link #getBuffer()}.
* This buffer can be filled with data and used to get a frame
* {@link #getFrame(byte[], long, int)}, or, in case it was not filled, returned to the queue
* using {@link #onBufferUnused(byte[])}.
* This is used for Camera2.
*/ */
public class FrameManager { public abstract class FrameManager<T> {
private static final String TAG = FrameManager.class.getSimpleName(); private static final String TAG = FrameManager.class.getSimpleName();
private static final CameraLogger LOG = CameraLogger.create(TAG); protected static final CameraLogger LOG = CameraLogger.create(TAG);
/**
* Receives callbacks on buffer availability
* (when a Frame is released, we reuse its buffer).
*/
public interface BufferCallback {
void onBufferAvailable(@NonNull byte[] buffer);
}
private final int mPoolSize; private final int mPoolSize;
private int mBufferSize = -1; private int mFrameBytes = -1;
private Size mFrameSize = null; private Size mFrameSize = null;
private int mFrameFormat = -1; private int mFrameFormat = -1;
private final Class<T> mFrameDataClass;
private LinkedBlockingQueue<Frame> mFrameQueue; private LinkedBlockingQueue<Frame> mFrameQueue;
private LinkedBlockingQueue<byte[]> mBufferQueue;
private BufferCallback mBufferCallback;
private final int mBufferMode;
/**
* In this mode, we have a {@link #mBufferCallback} and dispatch
* new buffers to the callback.
*/
private final static int BUFFER_MODE_DISPATCH = 0;
/**
* In this mode, we have a {@link #mBufferQueue} where we store
* buffers and only dispatch when requested.
*/
private final static int BUFFER_MODE_ENQUEUE = 1;
/** /**
* Construct a new frame manager. * Construct a new frame manager.
@ -84,18 +42,29 @@ public class FrameManager {
* as soon as the parameters are known. * as soon as the parameters are known.
* *
* @param poolSize the size of the backing pool. * @param poolSize the size of the backing pool.
* @param callback a callback
*/ */
public FrameManager(int poolSize, @Nullable BufferCallback callback) { protected FrameManager(int poolSize, @NonNull Class<T> dataClass) {
mPoolSize = poolSize; mPoolSize = poolSize;
mFrameDataClass = dataClass;
mFrameQueue = new LinkedBlockingQueue<>(mPoolSize); mFrameQueue = new LinkedBlockingQueue<>(mPoolSize);
if (callback != null) {
mBufferCallback = callback;
mBufferMode = BUFFER_MODE_DISPATCH;
} else {
mBufferQueue = new LinkedBlockingQueue<>(mPoolSize);
mBufferMode = BUFFER_MODE_ENQUEUE;
} }
/**
* Returns the pool size.
* @return pool size
*/
@SuppressWarnings("WeakerAccess")
public final int getPoolSize() {
return mPoolSize;
}
/**
* Returns the frame size in bytes.
* @return frame size in bytes
*/
@SuppressWarnings("WeakerAccess")
public final int getFrameBytes() {
return mFrameBytes;
} }
/** /**
@ -106,9 +75,8 @@ public class FrameManager {
* *
* @param format the image format * @param format the image format
* @param size the frame size * @param size the frame size
* @return the buffer size
*/ */
public int setUp(int format, @NonNull Size size) { public void setUp(int format, @NonNull Size size) {
if (isSetUp()) { if (isSetUp()) {
// TODO throw or just reconfigure? // TODO throw or just reconfigure?
} }
@ -116,15 +84,7 @@ public class FrameManager {
mFrameFormat = format; mFrameFormat = format;
int bitsPerPixel = ImageFormat.getBitsPerPixel(format); int bitsPerPixel = ImageFormat.getBitsPerPixel(format);
long sizeInBits = size.getHeight() * size.getWidth() * bitsPerPixel; long sizeInBits = size.getHeight() * size.getWidth() * bitsPerPixel;
mBufferSize = (int) Math.ceil(sizeInBits / 8.0d); mFrameBytes = (int) Math.ceil(sizeInBits / 8.0d);
for (int i = 0; i < mPoolSize; i++) {
if (mBufferMode == BUFFER_MODE_DISPATCH) {
mBufferCallback.onBufferAvailable(new byte[mBufferSize]);
} else {
mBufferQueue.offer(new byte[mBufferSize]);
}
}
return mBufferSize;
} }
/** /**
@ -134,47 +94,10 @@ public class FrameManager {
* *
* @return true if set up * @return true if set up
*/ */
private boolean isSetUp() { protected boolean isSetUp() {
return mFrameSize != null; return mFrameSize != null;
} }
/**
* Returns a new byte buffer than can be filled.
* This can only be called in {@link #BUFFER_MODE_ENQUEUE} mode! Where the frame
* manager also holds a queue of the byte buffers.
*
* If not null, the buffer returned by this method can be filled and used to get
* a new frame through {@link #getFrame(byte[], long, int)}.
*
* @return a buffer, or null
*/
@Nullable
public byte[] getBuffer() {
if (mBufferMode != BUFFER_MODE_ENQUEUE) {
throw new IllegalStateException("Can't call getBuffer() " +
"when not in BUFFER_MODE_ENQUEUE.");
}
return mBufferQueue.poll();
}
/**
* Can be called if the buffer obtained by {@link #getBuffer()}
* was not used to construct a frame, so it can be put back into the queue.
* @param buffer a buffer
*/
public void onBufferUnused(@NonNull byte[] buffer) {
if (mBufferMode != BUFFER_MODE_ENQUEUE) {
throw new IllegalStateException("Can't call onBufferUnused() " +
"when not in BUFFER_MODE_ENQUEUE.");
}
if (isSetUp()) {
mBufferQueue.offer(buffer);
} else {
LOG.w("onBufferUnused: buffer was returned but we're not set up anymore.");
}
}
/** /**
* Returns a new Frame for the given data. This must be called * Returns a new Frame for the given data. This must be called
* - after {@link #setUp(int, Size)}, which sets the buffer size * - after {@link #setUp(int, Size)}, which sets the buffer size
@ -190,7 +113,7 @@ public class FrameManager {
* @return a new frame * @return a new frame
*/ */
@NonNull @NonNull
public Frame getFrame(@NonNull byte[] data, long time, int rotation) { public Frame getFrame(@NonNull T data, long time, int rotation) {
if (!isSetUp()) { if (!isSetUp()) {
throw new IllegalStateException("Can't call getFrame() after releasing " + throw new IllegalStateException("Can't call getFrame() after releasing " +
"or before setUp."); "or before setUp.");
@ -201,7 +124,7 @@ public class FrameManager {
LOG.v("getFrame for time:", time, "RECYCLING."); LOG.v("getFrame for time:", time, "RECYCLING.");
} else { } else {
LOG.v("getFrame for time:", time, "CREATING."); LOG.v("getFrame for time:", time, "CREATING.");
frame = new Frame(this); frame = new Frame(this, mFrameDataClass);
} }
frame.setContent(data, time, rotation, mFrameSize, mFrameFormat); frame.setContent(data, time, rotation, mFrameSize, mFrameFormat);
return frame; return frame;
@ -209,33 +132,29 @@ public class FrameManager {
/** /**
* Called by child frames when they are released. * Called by child frames when they are released.
* This might be called from old Frames that belong to an old 'setUp'
* of this FrameManager instance. So the buffer size might be different,
* for instance.
*
* @param frame the released frame * @param frame the released frame
*/ */
void onFrameReleased(@NonNull Frame frame, @NonNull byte[] buffer) { void onFrameReleased(@NonNull Frame frame, @NonNull T data) {
if (!isSetUp()) return; if (!isSetUp()) return;
// If frame queue is full, let's drop everything. // If frame queue is full, let's drop everything.
// If frame queue accepts this frame, let's recycle the buffer as well. // If frame queue accepts this frame, let's recycle the buffer as well.
if (mFrameQueue.offer(frame)) { if (mFrameQueue.offer(frame)) {
int currSize = buffer.length; onFrameDataRecycled(data);
int reqSize = mBufferSize;
if (currSize == reqSize) {
if (mBufferMode == BUFFER_MODE_DISPATCH) {
mBufferCallback.onBufferAvailable(buffer);
} else {
mBufferQueue.offer(buffer);
}
}
} }
} }
/**
* Called when a Frame was recycled and its data is now available.
* This might be called from old Frames that belong to an old 'setUp'
* of this FrameManager instance. So the buffer size might be different,
* for instance.
* @param data data
*/
protected abstract void onFrameDataRecycled(@NonNull T data);
/** /**
* Releases all frames controlled by this manager and * Releases all frames controlled by this manager and
* clears the pool. * clears the pool.
* In BUFFER_MODE_ENQUEUE, releases also all the buffers.
*/ */
public void release() { public void release() {
if (!isSetUp()) { if (!isSetUp()) {
@ -245,10 +164,7 @@ public class FrameManager {
LOG.i("release: Clearing the frame and buffer queue."); LOG.i("release: Clearing the frame and buffer queue.");
mFrameQueue.clear(); mFrameQueue.clear();
if (mBufferMode == BUFFER_MODE_ENQUEUE) { mFrameBytes = -1;
mBufferQueue.clear();
}
mBufferSize = -1;
mFrameSize = null; mFrameSize = null;
mFrameFormat = -1; mFrameFormat = -1;
} }

@ -78,7 +78,11 @@ public class CameraActivity extends AppCompatActivity implements View.OnClickLis
lastTime = newTime; lastTime = newTime;
LOG.e("Frame delayMillis:", delay, "FPS:", 1000 / delay); LOG.e("Frame delayMillis:", delay, "FPS:", 1000 / delay);
if (DECODE_BITMAP) { if (DECODE_BITMAP) {
YuvImage yuvImage = new YuvImage(frame.getData(), ImageFormat.NV21, if (frame.getFormat() == ImageFormat.NV21
&& frame.getDataClass() == byte[].class) {
byte[] data = frame.getData();
YuvImage yuvImage = new YuvImage(data,
frame.getFormat(),
frame.getSize().getWidth(), frame.getSize().getWidth(),
frame.getSize().getHeight(), frame.getSize().getHeight(),
null); null);
@ -87,11 +91,13 @@ public class CameraActivity extends AppCompatActivity implements View.OnClickLis
frame.getSize().getWidth(), frame.getSize().getWidth(),
frame.getSize().getHeight()), 100, jpegStream); frame.getSize().getHeight()), 100, jpegStream);
byte[] jpegByteArray = jpegStream.toByteArray(); byte[] jpegByteArray = jpegStream.toByteArray();
Bitmap bitmap = BitmapFactory.decodeByteArray(jpegByteArray, 0, jpegByteArray.length); Bitmap bitmap = BitmapFactory.decodeByteArray(jpegByteArray,
0, jpegByteArray.length);
//noinspection ResultOfMethodCallIgnored //noinspection ResultOfMethodCallIgnored
bitmap.toString(); bitmap.toString();
} }
} }
}
}); });
} }

Loading…
Cancel
Save