Prepare Frame for Images, abstract FrameManager, create ByteBufferFrameManager

pull/704/head
Mattia Iavarone 6 years ago
parent cf1055e776
commit 28023c72eb
  1. 14
      cameraview/src/androidTest/java/com/otaliastudios/cameraview/CameraViewTest.java
  2. 3
      cameraview/src/androidTest/java/com/otaliastudios/cameraview/engine/MockCameraEngine.java
  3. 105
      cameraview/src/androidTest/java/com/otaliastudios/cameraview/frame/ByteBufferFrameManagerTest.java
  4. 87
      cameraview/src/androidTest/java/com/otaliastudios/cameraview/frame/FrameManagerTest.java
  5. 12
      cameraview/src/main/java/com/otaliastudios/cameraview/engine/Camera1Engine.java
  6. 26
      cameraview/src/main/java/com/otaliastudios/cameraview/engine/Camera2Engine.java
  7. 2
      cameraview/src/main/java/com/otaliastudios/cameraview/engine/CameraBaseEngine.java
  8. 154
      cameraview/src/main/java/com/otaliastudios/cameraview/frame/ByteBufferFrameManager.java
  9. 51
      cameraview/src/main/java/com/otaliastudios/cameraview/frame/Frame.java
  10. 172
      cameraview/src/main/java/com/otaliastudios/cameraview/frame/FrameManager.java
  11. 30
      demo/src/main/java/com/otaliastudios/cameraview/demo/CameraActivity.java

@ -996,22 +996,8 @@ public class CameraViewTest extends BaseTest {
//region Filter
@Test(expected = RuntimeException.class)
public void testSetFilter_notExperimental() {
cameraView.setExperimental(false);
cameraView.setFilter(Filters.AUTO_FIX.newInstance());
}
@Test
public void testSetFilter_notExperimental_noFilter() {
cameraView.setExperimental(false);
cameraView.setFilter(Filters.NONE.newInstance());
// no exception thrown
}
@Test
public void testSetFilter() {
cameraView.setExperimental(true);
Filter filter = Filters.AUTO_FIX.newInstance();
cameraView.setFilter(filter);
verify(mockPreview, times(1)).setFilter(filter);

@ -13,6 +13,7 @@ import com.otaliastudios.cameraview.controls.Facing;
import com.otaliastudios.cameraview.controls.Flash;
import com.otaliastudios.cameraview.controls.PictureFormat;
import com.otaliastudios.cameraview.engine.orchestrator.CameraState;
import com.otaliastudios.cameraview.frame.ByteBufferFrameManager;
import com.otaliastudios.cameraview.frame.FrameManager;
import com.otaliastudios.cameraview.gesture.Gesture;
import com.otaliastudios.cameraview.controls.Hdr;
@ -185,7 +186,7 @@ public class MockCameraEngine extends CameraBaseEngine {
@NonNull
@Override
protected FrameManager instantiateFrameManager() {
return new FrameManager(2, null);
return new ByteBufferFrameManager(2, null);
}
@Override

@ -0,0 +1,105 @@
package com.otaliastudios.cameraview.frame;
import android.graphics.ImageFormat;
import androidx.test.ext.junit.runners.AndroidJUnit4;
import androidx.test.filters.SmallTest;
import com.otaliastudios.cameraview.BaseTest;
import com.otaliastudios.cameraview.size.Size;
import org.junit.After;
import org.junit.Before;
import org.junit.Test;
import org.junit.runner.RunWith;
import static org.mockito.Matchers.any;
import static org.mockito.Mockito.mock;
import static org.mockito.Mockito.never;
import static org.mockito.Mockito.reset;
import static org.mockito.Mockito.times;
import static org.mockito.Mockito.verify;
@RunWith(AndroidJUnit4.class)
@SmallTest
public class ByteBufferFrameManagerTest extends BaseTest {
private ByteBufferFrameManager.BufferCallback callback;
@Before
public void setUp() {
callback = mock(ByteBufferFrameManager.BufferCallback.class);
}
@After
public void tearDown() {
callback = null;
}
@Test
public void testAllocate() {
ByteBufferFrameManager manager = new ByteBufferFrameManager(1, callback);
manager.setUp(ImageFormat.NV21, new Size(50, 50));
verify(callback, times(1)).onBufferAvailable(any(byte[].class));
reset(callback);
manager = new ByteBufferFrameManager(5, callback);
manager.setUp(ImageFormat.NV21, new Size(50, 50));
verify(callback, times(5)).onBufferAvailable(any(byte[].class));
}
@Test
public void testOnFrameReleased_alreadyFull() {
ByteBufferFrameManager manager = new ByteBufferFrameManager(1, callback);
manager.setUp(ImageFormat.NV21, new Size(50, 50));
int length = manager.getFrameBytes();
Frame frame1 = manager.getFrame(new byte[length], 0, 0);
// Since frame1 is already taken and poolSize = 1, a new Frame is created.
Frame frame2 = manager.getFrame(new byte[length], 0, 0);
// Release the first frame so it goes back into the pool.
manager.onFrameReleased(frame1, (byte[]) frame1.getData());
reset(callback);
// Release the second. The pool is already full, so onBufferAvailable should not be called
// since this Frame instance will NOT be reused.
manager.onFrameReleased(frame2, (byte[]) frame2.getData());
verify(callback, never()).onBufferAvailable((byte[]) frame2.getData());
}
@Test
public void testOnFrameReleased_sameLength() {
ByteBufferFrameManager manager = new ByteBufferFrameManager(1, callback);
manager.setUp(ImageFormat.NV21, new Size(50, 50));
int length = manager.getFrameBytes();
// A camera preview frame comes. Request a frame.
byte[] picture = new byte[length];
Frame frame = manager.getFrame(picture, 0, 0);
// Release the frame and ensure that onBufferAvailable is called.
reset(callback);
manager.onFrameReleased(frame, (byte[]) frame.getData());
verify(callback, times(1)).onBufferAvailable(picture);
}
@Test
public void testOnFrameReleased_differentLength() {
ByteBufferFrameManager manager = new ByteBufferFrameManager(1, callback);
manager.setUp(ImageFormat.NV21, new Size(50, 50));
int length = manager.getFrameBytes();
// A camera preview frame comes. Request a frame.
byte[] picture = new byte[length];
Frame frame = manager.getFrame(picture, 0, 0);
// Don't release the frame. Change the allocation size.
manager.setUp(ImageFormat.NV16, new Size(15, 15));
// Now release the old frame and ensure that onBufferAvailable is NOT called,
// because the released data has wrong length.
manager.onFrameReleased(frame, (byte[]) frame.getData());
reset(callback);
verify(callback, never()).onBufferAvailable(picture);
}
}

@ -3,6 +3,7 @@ package com.otaliastudios.cameraview.frame;
import android.graphics.ImageFormat;
import androidx.annotation.NonNull;
import androidx.test.ext.junit.runners.AndroidJUnit4;
import androidx.test.filters.SmallTest;
@ -26,93 +27,19 @@ import static org.mockito.Mockito.verify;
@SmallTest
public class FrameManagerTest extends BaseTest {
private FrameManager.BufferCallback callback;
@Before
public void setUp() {
callback = mock(FrameManager.BufferCallback.class);
}
@After
public void tearDown() {
callback = null;
}
@Test
public void testAllocate() {
FrameManager manager = new FrameManager(1, callback);
manager.setUp(ImageFormat.NV21, new Size(50, 50));
verify(callback, times(1)).onBufferAvailable(any(byte[].class));
reset(callback);
manager = new FrameManager(5, callback);
manager.setUp(ImageFormat.NV21, new Size(50, 50));
verify(callback, times(5)).onBufferAvailable(any(byte[].class));
}
@Test
public void testFrameRecycling() {
// A 1-pool manager will always recycle the same frame.
FrameManager manager = new FrameManager(1, callback);
FrameManager<String> manager = new FrameManager<String>(1, String.class) {
@Override
protected void onFrameDataRecycled(@NonNull String data) { }
};
manager.setUp(ImageFormat.NV21, new Size(50, 50));
Frame first = manager.getFrame(null, 0, 0);
Frame first = manager.getFrame("foo", 0, 0);
first.release();
Frame second = manager.getFrame(null, 0, 0);
Frame second = manager.getFrame("bar", 0, 0);
second.release();
assertEquals(first, second);
}
@Test
public void testOnFrameReleased_alreadyFull() {
FrameManager manager = new FrameManager(1, callback);
int length = manager.setUp(ImageFormat.NV21, new Size(50, 50));
Frame frame1 = manager.getFrame(new byte[length], 0, 0);
// Since frame1 is already taken and poolSize = 1, a new Frame is created.
Frame frame2 = manager.getFrame(new byte[length], 0, 0);
// Release the first frame so it goes back into the pool.
manager.onFrameReleased(frame1, frame1.getData());
reset(callback);
// Release the second. The pool is already full, so onBufferAvailable should not be called
// since this Frame instance will NOT be reused.
manager.onFrameReleased(frame2, frame2.getData());
verify(callback, never()).onBufferAvailable(frame2.getData());
}
@Test
public void testOnFrameReleased_sameLength() {
FrameManager manager = new FrameManager(1, callback);
int length = manager.setUp(ImageFormat.NV21, new Size(50, 50));
// A camera preview frame comes. Request a frame.
byte[] picture = new byte[length];
Frame frame = manager.getFrame(picture, 0, 0);
// Release the frame and ensure that onBufferAvailable is called.
reset(callback);
manager.onFrameReleased(frame, frame.getData());
verify(callback, times(1)).onBufferAvailable(picture);
}
@Test
public void testOnFrameReleased_differentLength() {
FrameManager manager = new FrameManager(1, callback);
int length = manager.setUp(ImageFormat.NV21, new Size(50, 50));
// A camera preview frame comes. Request a frame.
byte[] picture = new byte[length];
Frame frame = manager.getFrame(picture, 0, 0);
// Don't release the frame. Change the allocation size.
manager.setUp(ImageFormat.NV16, new Size(15, 15));
// Now release the old frame and ensure that onBufferAvailable is NOT called,
// because the released data has wrong length.
manager.onFrameReleased(frame, frame.getData());
reset(callback);
verify(callback, never()).onBufferAvailable(picture);
}
}

@ -25,6 +25,7 @@ import com.otaliastudios.cameraview.engine.offset.Axis;
import com.otaliastudios.cameraview.engine.offset.Reference;
import com.otaliastudios.cameraview.engine.options.Camera1Options;
import com.otaliastudios.cameraview.engine.orchestrator.CameraState;
import com.otaliastudios.cameraview.frame.ByteBufferFrameManager;
import com.otaliastudios.cameraview.frame.Frame;
import com.otaliastudios.cameraview.PictureResult;
import com.otaliastudios.cameraview.VideoResult;
@ -47,7 +48,6 @@ import com.otaliastudios.cameraview.video.SnapshotVideoRecorder;
import java.io.IOException;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collections;
import java.util.List;
@ -55,7 +55,7 @@ import java.util.List;
public class Camera1Engine extends CameraBaseEngine implements
Camera.PreviewCallback,
Camera.ErrorCallback,
FrameManager.BufferCallback {
ByteBufferFrameManager.BufferCallback {
private static final String JOB_FOCUS_RESET = "focus reset";
private static final String JOB_FOCUS_END = "focus end";
@ -758,7 +758,13 @@ public class Camera1Engine extends CameraBaseEngine implements
@NonNull
@Override
protected FrameManager instantiateFrameManager() {
return new FrameManager(2, this);
return new ByteBufferFrameManager(2, this);
}
@NonNull
@Override
public ByteBufferFrameManager getFrameManager() {
return (ByteBufferFrameManager) super.getFrameManager();
}
@Override

@ -56,6 +56,7 @@ import com.otaliastudios.cameraview.engine.offset.Axis;
import com.otaliastudios.cameraview.engine.offset.Reference;
import com.otaliastudios.cameraview.engine.options.Camera2Options;
import com.otaliastudios.cameraview.engine.orchestrator.CameraState;
import com.otaliastudios.cameraview.frame.ByteBufferFrameManager;
import com.otaliastudios.cameraview.frame.Frame;
import com.otaliastudios.cameraview.frame.FrameManager;
import com.otaliastudios.cameraview.gesture.Gesture;
@ -67,7 +68,6 @@ import com.otaliastudios.cameraview.picture.Snapshot2PictureRecorder;
import com.otaliastudios.cameraview.preview.GlCameraPreview;
import com.otaliastudios.cameraview.size.AspectRatio;
import com.otaliastudios.cameraview.size.Size;
import com.otaliastudios.cameraview.size.SizeSelectors;
import com.otaliastudios.cameraview.video.Full2VideoRecorder;
import com.otaliastudios.cameraview.video.SnapshotVideoRecorder;
@ -82,7 +82,6 @@ public class Camera2Engine extends CameraBaseEngine implements
ImageReader.OnImageAvailableListener,
ActionHolder {
private static final int FRAME_PROCESSING_FORMAT = ImageFormat.NV21;
private static final int FRAME_PROCESSING_INPUT_FORMAT = ImageFormat.YUV_420_888;
@VisibleForTesting static final long METER_TIMEOUT = 2500;
private final CameraManager mManager;
@ -336,7 +335,7 @@ public class Camera2Engine extends CameraBaseEngine implements
if (streamMap == null) {
throw new RuntimeException("StreamConfigurationMap is null. Should not happen.");
}
android.util.Size[] sizes = streamMap.getOutputSizes(FRAME_PROCESSING_INPUT_FORMAT);
android.util.Size[] sizes = streamMap.getOutputSizes(getFrameProcessingImageFormat());
List<Size> candidates = new ArrayList<>(sizes.length);
for (android.util.Size size : sizes) {
Size add = new Size(size.getWidth(), size.getHeight());
@ -546,7 +545,7 @@ public class Camera2Engine extends CameraBaseEngine implements
mFrameProcessingReader = ImageReader.newInstance(
mFrameProcessingSize.getWidth(),
mFrameProcessingSize.getHeight(),
FRAME_PROCESSING_INPUT_FORMAT,
getFrameProcessingImageFormat(),
2);
mFrameProcessingReader.setOnImageAvailableListener(this,
mFrameConversionHandler.getHandler());
@ -1408,7 +1407,24 @@ public class Camera2Engine extends CameraBaseEngine implements
@NonNull
@Override
protected FrameManager instantiateFrameManager() {
return new FrameManager(2, null);
return new ByteBufferFrameManager(2, null);
}
@NonNull
@Override
public ByteBufferFrameManager getFrameManager() {
return (ByteBufferFrameManager) super.getFrameManager();
}
/**
* Returns the frame processing image format.
* Defaults to {@link ImageFormat#YUV_420_888}.
* Override at your own risk!
* @return format
*/
@SuppressWarnings("WeakerAccess")
protected int getFrameProcessingImageFormat() {
return ImageFormat.YUV_420_888;
}
@Override

@ -126,7 +126,7 @@ public abstract class CameraBaseEngine extends CameraEngine {
@NonNull
@Override
public final FrameManager getFrameManager() {
public FrameManager getFrameManager() {
return mFrameManager;
}

@ -0,0 +1,154 @@
package com.otaliastudios.cameraview.frame;
import androidx.annotation.NonNull;
import androidx.annotation.Nullable;
import com.otaliastudios.cameraview.size.Size;
import java.util.concurrent.LinkedBlockingQueue;
/**
* This class manages the allocation of byte buffers and {@link Frame} objects.
* We are interested in recycling both of them, especially byte[] buffers which can create a lot
* of overhead.
*
* The pool size applies to both the {@link Frame} pool and the byte[] pool - it makes sense to use
* the same number since they are consumed at the same time.
*
* We can work in two modes, depending on whether a
* {@link BufferCallback} is passed to the constructor. The modes changes the buffer behavior.
*
* 1. {@link #BUFFER_MODE_DISPATCH}: in this mode, as soon as we have a buffer, it is dispatched to
* the {@link BufferCallback}. The callback should then fill the buffer, and finally call
* {@link #getFrame(byte[], long, int)} to receive a frame.
* This is used for Camera1.
*
* 2. {@link #BUFFER_MODE_ENQUEUE}: in this mode, the manager internally keeps a queue of byte
* buffers, instead of handing them to the callback. The users can ask for buffers through
* {@link #getBuffer()}.
* This buffer can be filled with data and used to get a frame
* {@link #getFrame(byte[], long, int)}, or, in case it was not filled, returned to the queue
* using {@link #onBufferUnused(byte[])}.
* This is used for Camera2.
*/
public class ByteBufferFrameManager extends FrameManager<byte[]> {
/**
* Receives callbacks on buffer availability
* (when a Frame is released, we reuse its buffer).
*/
public interface BufferCallback {
void onBufferAvailable(@NonNull byte[] buffer);
}
/**
* In this mode, we have a {@link #mBufferCallback} and dispatch
* new buffers to the callback.
*/
private final static int BUFFER_MODE_DISPATCH = 0;
/**
* In this mode, we have a {@link #mBufferQueue} where we store
* buffers and only dispatch when requested.
*/
private final static int BUFFER_MODE_ENQUEUE = 1;
private LinkedBlockingQueue<byte[]> mBufferQueue;
private BufferCallback mBufferCallback;
private final int mBufferMode;
/**
* Construct a new frame manager.
* The construction must be followed by an {@link #setUp(int, Size)} call
* as soon as the parameters are known.
*
* @param poolSize the size of the backing pool.
* @param callback a callback
*/
public ByteBufferFrameManager(int poolSize, @Nullable BufferCallback callback) {
super(poolSize, byte[].class);
if (callback != null) {
mBufferCallback = callback;
mBufferMode = BUFFER_MODE_DISPATCH;
} else {
mBufferQueue = new LinkedBlockingQueue<>(poolSize);
mBufferMode = BUFFER_MODE_ENQUEUE;
}
}
@Override
public void setUp(int format, @NonNull Size size) {
super.setUp(format, size);
int bytes = getFrameBytes();
for (int i = 0; i < getPoolSize(); i++) {
if (mBufferMode == BUFFER_MODE_DISPATCH) {
mBufferCallback.onBufferAvailable(new byte[bytes]);
} else {
mBufferQueue.offer(new byte[bytes]);
}
}
}
/**
* Returns a new byte buffer than can be filled.
* This can only be called in {@link #BUFFER_MODE_ENQUEUE} mode! Where the frame
* manager also holds a queue of the byte buffers.
*
* If not null, the buffer returned by this method can be filled and used to get
* a new frame through {@link #getFrame(byte[], long, int)}.
*
* @return a buffer, or null
*/
@Nullable
public byte[] getBuffer() {
if (mBufferMode != BUFFER_MODE_ENQUEUE) {
throw new IllegalStateException("Can't call getBuffer() " +
"when not in BUFFER_MODE_ENQUEUE.");
}
return mBufferQueue.poll();
}
/**
* Can be called if the buffer obtained by {@link #getBuffer()}
* was not used to construct a frame, so it can be put back into the queue.
* @param buffer a buffer
*/
public void onBufferUnused(@NonNull byte[] buffer) {
if (mBufferMode != BUFFER_MODE_ENQUEUE) {
throw new IllegalStateException("Can't call onBufferUnused() " +
"when not in BUFFER_MODE_ENQUEUE.");
}
if (isSetUp()) {
mBufferQueue.offer(buffer);
} else {
LOG.w("onBufferUnused: buffer was returned but we're not set up anymore.");
}
}
@Override
protected void onFrameDataRecycled(@NonNull byte[] data) {
if (data.length == getFrameBytes()) {
if (mBufferMode == BUFFER_MODE_DISPATCH) {
mBufferCallback.onBufferAvailable(data);
} else {
mBufferQueue.offer(data);
}
}
}
/**
* Releases all frames controlled by this manager and
* clears the pool.
* In BUFFER_MODE_ENQUEUE, releases also all the buffers.
*/
@Override
public void release() {
super.release();
if (mBufferMode == BUFFER_MODE_ENQUEUE) {
mBufferQueue.clear();
}
}
}

@ -1,6 +1,10 @@
package com.otaliastudios.cameraview.frame;
import android.annotation.SuppressLint;
import android.media.Image;
import com.otaliastudios.cameraview.CameraLogger;
import com.otaliastudios.cameraview.controls.Engine;
import com.otaliastudios.cameraview.size.Size;
import androidx.annotation.NonNull;
@ -14,19 +18,21 @@ public class Frame {
private final static CameraLogger LOG = CameraLogger.create(TAG);
private final FrameManager mManager;
private final Class<?> mDataClass;
private byte[] mData = null;
private Object mData = null;
private long mTime = -1;
private long mLastTime = -1;
private int mRotation = 0;
private Size mSize = null;
private int mFormat = -1;
Frame(@NonNull FrameManager manager) {
Frame(@NonNull FrameManager manager, @NonNull Class<?> dataClass) {
mManager = manager;
mDataClass = dataClass;
}
void setContent(@NonNull byte[] data, long time, int rotation, @NonNull Size size, int format) {
void setContent(@NonNull Object data, long time, int rotation, @NonNull Size size, int format) {
this.mData = data;
this.mTime = time;
this.mLastTime = time;
@ -63,13 +69,23 @@ public class Frame {
*
* @return a frozen Frame
*/
@SuppressLint("NewApi")
@NonNull
public Frame freeze() {
ensureHasContent();
byte[] data = new byte[mData.length];
System.arraycopy(mData, 0, data, 0, mData.length);
Frame other = new Frame(mManager);
other.setContent(data, mTime, mRotation, mSize, mFormat);
Frame other = new Frame(mManager, mDataClass);
if (getDataClass() == byte[].class) {
byte[] input = getData();
byte[] output = new byte[input.length];
System.arraycopy(input, 0, output, 0, input.length);
other.setContent(output, mTime, mRotation, mSize, mFormat);
} else if (getDataClass() == Image.class) {
throw new RuntimeException("Cannot freeze() a Image Frame. " +
"Please consider using the frame synchronously in your process() method, " +
"which also gives better performance.");
} else {
throw new RuntimeException("Unknown data class: " + getDataClass());
}
return other;
}
@ -80,7 +96,7 @@ public class Frame {
public void release() {
if (!hasContent()) return;
LOG.v("Frame with time", mTime, "is being released.");
byte[] data = mData;
Object data = mData;
mData = null;
mRotation = 0;
mTime = -1;
@ -89,6 +105,7 @@ public class Frame {
// After the manager is notified, this frame instance can be taken by
// someone else, possibly from another thread. So this should be the
// last call in this method. If we null data after, we can have issues.
//noinspection unchecked
mManager.onFrameReleased(this, data);
}
@ -96,10 +113,24 @@ public class Frame {
* Returns the frame data.
* @return the frame data
*/
@SuppressWarnings("unchecked")
@NonNull
public byte[] getData() {
public <T> T getData() {
ensureHasContent();
return mData;
return (T) mData;
}
/**
* Returns the class returned by {@link #getData()}.
* This class depends on the engine that produced this frame.
* - {@link Engine#CAMERA1} will produce byte[] arrays
* - {@link Engine#CAMERA2} will produce {@link android.media.Image}s
* @return the data class
*/
@SuppressWarnings("WeakerAccess")
@NonNull
public Class<?> getDataClass() {
return mDataClass;
}
/**

@ -12,71 +12,29 @@ import androidx.annotation.Nullable;
import java.util.concurrent.LinkedBlockingQueue;
/**
* This class manages the allocation of byte buffers and {@link Frame} objects.
* We are interested in recycling both of them, especially byte[] buffers which can create a lot
* of overhead.
*
* This class manages the allocation of {@link Frame} objects.
* The FrameManager keeps a {@link #mPoolSize} integer that defines the number of instances to keep.
* The pool size applies to both the {@link Frame} pool and the byte[] pool - it makes sense to use
* the same number since they are consumed at the same time.
*
* Main methods are:
* - {@link #setUp(int, Size)}: to set up with size and allocate buffers
* - {@link #release()}: to release. After release, a manager can be setUp again.
* - {@link #getFrame(byte[], long, int)}: gets a new {@link Frame}.
* - {@link #getFrame(Object, long, int)}: gets a new {@link Frame}.
*
* For both byte buffers and frames to get back to the FrameManager pool, all you have to do
* For frames to get back to the FrameManager pool, all you have to do
* is call {@link Frame#release()} when done.
*
* Other than this, the FrameManager can work in two modes, depending on whether a
* {@link BufferCallback} is passed to the constructor. The modes changes the buffer behavior.
*
* 1. {@link #BUFFER_MODE_DISPATCH}: in this mode, as soon as we have a buffer, it is dispatched to
* the {@link BufferCallback}. The callback should then fill the buffer, and finally call
* {@link #getFrame(byte[], long, int)} to receive a frame.
* This is used for Camera1.
*
* 2. {@link #BUFFER_MODE_ENQUEUE}: in this mode, the manager internally keeps a queue of byte
* buffers, instead of handing them to the callback. The users can ask for buffers through
* {@link #getBuffer()}.
* This buffer can be filled with data and used to get a frame
* {@link #getFrame(byte[], long, int)}, or, in case it was not filled, returned to the queue
* using {@link #onBufferUnused(byte[])}.
* This is used for Camera2.
*/
public class FrameManager {
public abstract class FrameManager<T> {
private static final String TAG = FrameManager.class.getSimpleName();
private static final CameraLogger LOG = CameraLogger.create(TAG);
/**
* Receives callbacks on buffer availability
* (when a Frame is released, we reuse its buffer).
*/
public interface BufferCallback {
void onBufferAvailable(@NonNull byte[] buffer);
}
protected static final CameraLogger LOG = CameraLogger.create(TAG);
private final int mPoolSize;
private int mBufferSize = -1;
private int mFrameBytes = -1;
private Size mFrameSize = null;
private int mFrameFormat = -1;
private final Class<T> mFrameDataClass;
private LinkedBlockingQueue<Frame> mFrameQueue;
private LinkedBlockingQueue<byte[]> mBufferQueue;
private BufferCallback mBufferCallback;
private final int mBufferMode;
/**
* In this mode, we have a {@link #mBufferCallback} and dispatch
* new buffers to the callback.
*/
private final static int BUFFER_MODE_DISPATCH = 0;
/**
* In this mode, we have a {@link #mBufferQueue} where we store
* buffers and only dispatch when requested.
*/
private final static int BUFFER_MODE_ENQUEUE = 1;
/**
* Construct a new frame manager.
@ -84,18 +42,29 @@ public class FrameManager {
* as soon as the parameters are known.
*
* @param poolSize the size of the backing pool.
* @param callback a callback
*/
public FrameManager(int poolSize, @Nullable BufferCallback callback) {
protected FrameManager(int poolSize, @NonNull Class<T> dataClass) {
mPoolSize = poolSize;
mFrameDataClass = dataClass;
mFrameQueue = new LinkedBlockingQueue<>(mPoolSize);
if (callback != null) {
mBufferCallback = callback;
mBufferMode = BUFFER_MODE_DISPATCH;
} else {
mBufferQueue = new LinkedBlockingQueue<>(mPoolSize);
mBufferMode = BUFFER_MODE_ENQUEUE;
}
}
/**
* Returns the pool size.
* @return pool size
*/
@SuppressWarnings("WeakerAccess")
public final int getPoolSize() {
return mPoolSize;
}
/**
* Returns the frame size in bytes.
* @return frame size in bytes
*/
@SuppressWarnings("WeakerAccess")
public final int getFrameBytes() {
return mFrameBytes;
}
/**
@ -106,9 +75,8 @@ public class FrameManager {
*
* @param format the image format
* @param size the frame size
* @return the buffer size
*/
public int setUp(int format, @NonNull Size size) {
public void setUp(int format, @NonNull Size size) {
if (isSetUp()) {
// TODO throw or just reconfigure?
}
@ -116,15 +84,7 @@ public class FrameManager {
mFrameFormat = format;
int bitsPerPixel = ImageFormat.getBitsPerPixel(format);
long sizeInBits = size.getHeight() * size.getWidth() * bitsPerPixel;
mBufferSize = (int) Math.ceil(sizeInBits / 8.0d);
for (int i = 0; i < mPoolSize; i++) {
if (mBufferMode == BUFFER_MODE_DISPATCH) {
mBufferCallback.onBufferAvailable(new byte[mBufferSize]);
} else {
mBufferQueue.offer(new byte[mBufferSize]);
}
}
return mBufferSize;
mFrameBytes = (int) Math.ceil(sizeInBits / 8.0d);
}
/**
@ -134,47 +94,10 @@ public class FrameManager {
*
* @return true if set up
*/
private boolean isSetUp() {
protected boolean isSetUp() {
return mFrameSize != null;
}
/**
* Returns a new byte buffer than can be filled.
* This can only be called in {@link #BUFFER_MODE_ENQUEUE} mode! Where the frame
* manager also holds a queue of the byte buffers.
*
* If not null, the buffer returned by this method can be filled and used to get
* a new frame through {@link #getFrame(byte[], long, int)}.
*
* @return a buffer, or null
*/
@Nullable
public byte[] getBuffer() {
if (mBufferMode != BUFFER_MODE_ENQUEUE) {
throw new IllegalStateException("Can't call getBuffer() " +
"when not in BUFFER_MODE_ENQUEUE.");
}
return mBufferQueue.poll();
}
/**
* Can be called if the buffer obtained by {@link #getBuffer()}
* was not used to construct a frame, so it can be put back into the queue.
* @param buffer a buffer
*/
public void onBufferUnused(@NonNull byte[] buffer) {
if (mBufferMode != BUFFER_MODE_ENQUEUE) {
throw new IllegalStateException("Can't call onBufferUnused() " +
"when not in BUFFER_MODE_ENQUEUE.");
}
if (isSetUp()) {
mBufferQueue.offer(buffer);
} else {
LOG.w("onBufferUnused: buffer was returned but we're not set up anymore.");
}
}
/**
* Returns a new Frame for the given data. This must be called
* - after {@link #setUp(int, Size)}, which sets the buffer size
@ -190,7 +113,7 @@ public class FrameManager {
* @return a new frame
*/
@NonNull
public Frame getFrame(@NonNull byte[] data, long time, int rotation) {
public Frame getFrame(@NonNull T data, long time, int rotation) {
if (!isSetUp()) {
throw new IllegalStateException("Can't call getFrame() after releasing " +
"or before setUp.");
@ -201,7 +124,7 @@ public class FrameManager {
LOG.v("getFrame for time:", time, "RECYCLING.");
} else {
LOG.v("getFrame for time:", time, "CREATING.");
frame = new Frame(this);
frame = new Frame(this, mFrameDataClass);
}
frame.setContent(data, time, rotation, mFrameSize, mFrameFormat);
return frame;
@ -209,33 +132,29 @@ public class FrameManager {
/**
* Called by child frames when they are released.
* This might be called from old Frames that belong to an old 'setUp'
* of this FrameManager instance. So the buffer size might be different,
* for instance.
*
* @param frame the released frame
*/
void onFrameReleased(@NonNull Frame frame, @NonNull byte[] buffer) {
void onFrameReleased(@NonNull Frame frame, @NonNull T data) {
if (!isSetUp()) return;
// If frame queue is full, let's drop everything.
// If frame queue accepts this frame, let's recycle the buffer as well.
if (mFrameQueue.offer(frame)) {
int currSize = buffer.length;
int reqSize = mBufferSize;
if (currSize == reqSize) {
if (mBufferMode == BUFFER_MODE_DISPATCH) {
mBufferCallback.onBufferAvailable(buffer);
} else {
mBufferQueue.offer(buffer);
}
}
onFrameDataRecycled(data);
}
}
/**
* Called when a Frame was recycled and its data is now available.
* This might be called from old Frames that belong to an old 'setUp'
* of this FrameManager instance. So the buffer size might be different,
* for instance.
* @param data data
*/
protected abstract void onFrameDataRecycled(@NonNull T data);
/**
* Releases all frames controlled by this manager and
* clears the pool.
* In BUFFER_MODE_ENQUEUE, releases also all the buffers.
*/
public void release() {
if (!isSetUp()) {
@ -245,10 +164,7 @@ public class FrameManager {
LOG.i("release: Clearing the frame and buffer queue.");
mFrameQueue.clear();
if (mBufferMode == BUFFER_MODE_ENQUEUE) {
mBufferQueue.clear();
}
mBufferSize = -1;
mFrameBytes = -1;
mFrameSize = null;
mFrameFormat = -1;
}

@ -78,18 +78,24 @@ public class CameraActivity extends AppCompatActivity implements View.OnClickLis
lastTime = newTime;
LOG.e("Frame delayMillis:", delay, "FPS:", 1000 / delay);
if (DECODE_BITMAP) {
YuvImage yuvImage = new YuvImage(frame.getData(), ImageFormat.NV21,
frame.getSize().getWidth(),
frame.getSize().getHeight(),
null);
ByteArrayOutputStream jpegStream = new ByteArrayOutputStream();
yuvImage.compressToJpeg(new Rect(0, 0,
frame.getSize().getWidth(),
frame.getSize().getHeight()), 100, jpegStream);
byte[] jpegByteArray = jpegStream.toByteArray();
Bitmap bitmap = BitmapFactory.decodeByteArray(jpegByteArray, 0, jpegByteArray.length);
//noinspection ResultOfMethodCallIgnored
bitmap.toString();
if (frame.getFormat() == ImageFormat.NV21
&& frame.getDataClass() == byte[].class) {
byte[] data = frame.getData();
YuvImage yuvImage = new YuvImage(data,
frame.getFormat(),
frame.getSize().getWidth(),
frame.getSize().getHeight(),
null);
ByteArrayOutputStream jpegStream = new ByteArrayOutputStream();
yuvImage.compressToJpeg(new Rect(0, 0,
frame.getSize().getWidth(),
frame.getSize().getHeight()), 100, jpegStream);
byte[] jpegByteArray = jpegStream.toByteArray();
Bitmap bitmap = BitmapFactory.decodeByteArray(jpegByteArray,
0, jpegByteArray.length);
//noinspection ResultOfMethodCallIgnored
bitmap.toString();
}
}
}
});

Loading…
Cancel
Save