Add setFrameProcessingExecutors API

pull/716/head
Mattia Iavarone 6 years ago
parent d442cd3534
commit 0df28dc2bd
  1. 13
      cameraview/src/androidTest/java/com/otaliastudios/cameraview/CameraViewTest.java
  2. 114
      cameraview/src/main/java/com/otaliastudios/cameraview/CameraView.java
  3. 1
      cameraview/src/main/res/values/attrs.xml
  4. 48
      docs/_posts/2018-12-20-frame-processing.md
  5. 2
      docs/css/main.css

@ -840,6 +840,19 @@ public class CameraViewTest extends BaseTest {
assertEquals(6, cameraView.getFrameProcessingPoolSize());
}
@Test
public void testFrameProcessingExecutors() {
cameraView.setFrameProcessingExecutors(5);
assertEquals(5, cameraView.getFrameProcessingExecutors());
cameraView.setFrameProcessingExecutors(2);
assertEquals(2, cameraView.getFrameProcessingExecutors());
}
@Test(expected = RuntimeException.class)
public void testFrameProcessingExecutors_throws() {
cameraView.setFrameProcessingExecutors(0);
}
//endregion
//region Lists of listeners and processors

@ -67,7 +67,6 @@ import com.otaliastudios.cameraview.gesture.TapGestureFinder;
import com.otaliastudios.cameraview.internal.GridLinesLayout;
import com.otaliastudios.cameraview.internal.utils.CropHelper;
import com.otaliastudios.cameraview.internal.utils.OrientationHelper;
import com.otaliastudios.cameraview.internal.utils.WorkerHandler;
import com.otaliastudios.cameraview.markers.AutoFocusMarker;
import com.otaliastudios.cameraview.markers.AutoFocusTrigger;
import com.otaliastudios.cameraview.markers.MarkerLayout;
@ -89,6 +88,12 @@ import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.concurrent.CopyOnWriteArrayList;
import java.util.concurrent.Executor;
import java.util.concurrent.LinkedBlockingQueue;
import java.util.concurrent.ThreadFactory;
import java.util.concurrent.ThreadPoolExecutor;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.atomic.AtomicInteger;
import static android.view.View.MeasureSpec.AT_MOST;
import static android.view.View.MeasureSpec.EXACTLY;
@ -112,6 +117,7 @@ public class CameraView extends FrameLayout implements LifecycleObserver {
final static boolean DEFAULT_PICTURE_METERING = true;
final static boolean DEFAULT_PICTURE_SNAPSHOT_METERING = false;
final static int DEFAULT_FRAME_PROCESSING_POOL_SIZE = 2;
final static int DEFAULT_FRAME_PROCESSING_EXECUTORS = 1;
// Self managed parameters
private boolean mPlaySounds;
@ -120,8 +126,11 @@ public class CameraView extends FrameLayout implements LifecycleObserver {
private Preview mPreview;
private Engine mEngine;
private Filter mPendingFilter;
private int mFrameProcessingExecutors;
// Components
private Handler mUiHandler;
private Executor mFrameProcessingExecutor;
@VisibleForTesting CameraCallbacks mCameraCallbacks;
private CameraPreview mCameraPreview;
private OrientationHelper mOrientationHelper;
@ -149,10 +158,6 @@ public class CameraView extends FrameLayout implements LifecycleObserver {
// Overlays
@VisibleForTesting OverlayLayout mOverlayLayout;
// Threading
private Handler mUiHandler;
private WorkerHandler mFrameProcessorsHandler;
public CameraView(@NonNull Context context) {
super(context, null);
initialize(context, null);
@ -209,6 +214,8 @@ public class CameraView extends FrameLayout implements LifecycleObserver {
int frameFormat = a.getInteger(R.styleable.CameraView_cameraFrameProcessingFormat, 0);
int framePoolSize = a.getInteger(R.styleable.CameraView_cameraFrameProcessingPoolSize,
DEFAULT_FRAME_PROCESSING_POOL_SIZE);
int frameExecutors = a.getInteger(R.styleable.CameraView_cameraFrameProcessingExecutors,
DEFAULT_FRAME_PROCESSING_EXECUTORS);
// Size selectors and gestures
SizeSelectorParser sizeSelectors = new SizeSelectorParser(a);
@ -221,7 +228,6 @@ public class CameraView extends FrameLayout implements LifecycleObserver {
// Components
mCameraCallbacks = new CameraCallbacks();
mUiHandler = new Handler(Looper.getMainLooper());
mFrameProcessorsHandler = WorkerHandler.get("FrameProcessorsWorker");
// Gestures
mPinchGestureFinder = new PinchGestureFinder(mCameraCallbacks);
@ -271,7 +277,7 @@ public class CameraView extends FrameLayout implements LifecycleObserver {
setFrameProcessingMaxHeight(frameMaxHeight);
setFrameProcessingFormat(frameFormat);
setFrameProcessingPoolSize(framePoolSize);
mCameraEngine.setHasFrameProcessors(!mFrameProcessors.isEmpty());
setFrameProcessingExecutors(frameExecutors);
// Apply gestures
mapGesture(Gesture.TAP, gestures.getTapAction());
@ -987,6 +993,7 @@ public class CameraView extends FrameLayout implements LifecycleObserver {
setFrameProcessingMaxHeight(oldEngine.getFrameProcessingMaxHeight());
setFrameProcessingFormat(0 /* this is very engine specific, so do not pass */);
setFrameProcessingPoolSize(oldEngine.getFrameProcessingPoolSize());
mCameraEngine.setHasFrameProcessors(!mFrameProcessors.isEmpty());
}
/**
@ -1989,7 +1996,8 @@ public class CameraView extends FrameLayout implements LifecycleObserver {
OrientationHelper.Callback,
GestureFinder.Controller {
private CameraLogger mLogger = CameraLogger.create(CameraCallbacks.class.getSimpleName());
private final String TAG = CameraCallbacks.class.getSimpleName();
private final CameraLogger LOG = CameraLogger.create(TAG);
@NonNull
@Override
@ -2009,7 +2017,7 @@ public class CameraView extends FrameLayout implements LifecycleObserver {
@Override
public void dispatchOnCameraOpened(@NonNull final CameraOptions options) {
mLogger.i("dispatchOnCameraOpened", options);
LOG.i("dispatchOnCameraOpened", options);
mUiHandler.post(new Runnable() {
@Override
public void run() {
@ -2022,7 +2030,7 @@ public class CameraView extends FrameLayout implements LifecycleObserver {
@Override
public void dispatchOnCameraClosed() {
mLogger.i("dispatchOnCameraClosed");
LOG.i("dispatchOnCameraClosed");
mUiHandler.post(new Runnable() {
@Override
public void run() {
@ -2043,10 +2051,10 @@ public class CameraView extends FrameLayout implements LifecycleObserver {
if (previewSize == null) {
throw new RuntimeException("Preview stream size should not be null here.");
} else if (previewSize.equals(mLastPreviewStreamSize)) {
mLogger.i("onCameraPreviewStreamSizeChanged:",
LOG.i("onCameraPreviewStreamSizeChanged:",
"swallowing because the preview size has not changed.", previewSize);
} else {
mLogger.i("onCameraPreviewStreamSizeChanged: posting a requestLayout call.",
LOG.i("onCameraPreviewStreamSizeChanged: posting a requestLayout call.",
"Preview stream size:", previewSize);
mUiHandler.post(new Runnable() {
@Override
@ -2066,7 +2074,7 @@ public class CameraView extends FrameLayout implements LifecycleObserver {
@Override
public void dispatchOnPictureTaken(@NonNull final PictureResult.Stub stub) {
mLogger.i("dispatchOnPictureTaken", stub);
LOG.i("dispatchOnPictureTaken", stub);
mUiHandler.post(new Runnable() {
@Override
public void run() {
@ -2080,7 +2088,7 @@ public class CameraView extends FrameLayout implements LifecycleObserver {
@Override
public void dispatchOnVideoTaken(@NonNull final VideoResult.Stub stub) {
mLogger.i("dispatchOnVideoTaken", stub);
LOG.i("dispatchOnVideoTaken", stub);
mUiHandler.post(new Runnable() {
@Override
public void run() {
@ -2095,7 +2103,7 @@ public class CameraView extends FrameLayout implements LifecycleObserver {
@Override
public void dispatchOnFocusStart(@Nullable final Gesture gesture,
@NonNull final PointF point) {
mLogger.i("dispatchOnFocusStart", gesture, point);
LOG.i("dispatchOnFocusStart", gesture, point);
mUiHandler.post(new Runnable() {
@Override
public void run() {
@ -2117,7 +2125,7 @@ public class CameraView extends FrameLayout implements LifecycleObserver {
public void dispatchOnFocusEnd(@Nullable final Gesture gesture,
final boolean success,
@NonNull final PointF point) {
mLogger.i("dispatchOnFocusEnd", gesture, success, point);
LOG.i("dispatchOnFocusEnd", gesture, success, point);
mUiHandler.post(new Runnable() {
@Override
public void run() {
@ -2140,7 +2148,7 @@ public class CameraView extends FrameLayout implements LifecycleObserver {
@Override
public void onDeviceOrientationChanged(int deviceOrientation) {
mLogger.i("onDeviceOrientationChanged", deviceOrientation);
LOG.i("onDeviceOrientationChanged", deviceOrientation);
int displayOffset = mOrientationHelper.getLastDisplayOffset();
if (!mUseDeviceOrientation) {
// To fool the engine to return outputs in the VIEW reference system,
@ -2163,12 +2171,12 @@ public class CameraView extends FrameLayout implements LifecycleObserver {
@Override
public void onDisplayOffsetChanged(int displayOffset, boolean willRecreate) {
mLogger.i("onDisplayOffsetChanged", displayOffset, "recreate:", willRecreate);
LOG.i("onDisplayOffsetChanged", displayOffset, "recreate:", willRecreate);
if (isOpened() && !willRecreate) {
// Display offset changes when the device rotation lock is off and the activity
// is free to rotate. However, some changes will NOT recreate the activity, namely
// 180 degrees flips. In this case, we must restart the camera manually.
mLogger.w("onDisplayOffsetChanged", "restarting the camera.");
LOG.w("onDisplayOffsetChanged", "restarting the camera.");
close();
open();
}
@ -2176,7 +2184,7 @@ public class CameraView extends FrameLayout implements LifecycleObserver {
@Override
public void dispatchOnZoomChanged(final float newValue, @Nullable final PointF[] fingers) {
mLogger.i("dispatchOnZoomChanged", newValue);
LOG.i("dispatchOnZoomChanged", newValue);
mUiHandler.post(new Runnable() {
@Override
public void run() {
@ -2191,7 +2199,7 @@ public class CameraView extends FrameLayout implements LifecycleObserver {
public void dispatchOnExposureCorrectionChanged(final float newValue,
@NonNull final float[] bounds,
@Nullable final PointF[] fingers) {
mLogger.i("dispatchOnExposureCorrectionChanged", newValue);
LOG.i("dispatchOnExposureCorrectionChanged", newValue);
mUiHandler.post(new Runnable() {
@Override
public void run() {
@ -2206,23 +2214,22 @@ public class CameraView extends FrameLayout implements LifecycleObserver {
public void dispatchFrame(@NonNull final Frame frame) {
// The getTime() below might crash if developers incorrectly release
// frames asynchronously.
mLogger.v("dispatchFrame:", frame.getTime(), "processors:", mFrameProcessors.size());
LOG.v("dispatchFrame:", frame.getTime(), "processors:", mFrameProcessors.size());
if (mFrameProcessors.isEmpty()) {
// Mark as released. This instance will be reused.
frame.release();
} else {
// Dispatch this frame to frame processors.
mFrameProcessorsHandler.run(new Runnable() {
mFrameProcessingExecutor.execute(new Runnable() {
@Override
public void run() {
mLogger.v("dispatchFrame: dispatching", frame.getTime(),
LOG.v("dispatchFrame: dispatching", frame.getTime(),
"to processors.");
for (FrameProcessor processor : mFrameProcessors) {
try {
processor.process(frame);
} catch (Exception e) {
// Don't let a single processor crash the processor thread.
mLogger.w("Frame processor crashed:", e);
LOG.w("Frame processor crashed:", e);
}
}
frame.release();
@ -2233,7 +2240,7 @@ public class CameraView extends FrameLayout implements LifecycleObserver {
@Override
public void dispatchError(final CameraException exception) {
mLogger.i("dispatchError", exception);
LOG.i("dispatchError", exception);
mUiHandler.post(new Runnable() {
@Override
public void run() {
@ -2246,7 +2253,7 @@ public class CameraView extends FrameLayout implements LifecycleObserver {
@Override
public void dispatchOnVideoRecordingStart() {
mLogger.i("dispatchOnVideoRecordingStart");
LOG.i("dispatchOnVideoRecordingStart");
mUiHandler.post(new Runnable() {
@Override
public void run() {
@ -2259,7 +2266,7 @@ public class CameraView extends FrameLayout implements LifecycleObserver {
@Override
public void dispatchOnVideoRecordingEnd() {
mLogger.i("dispatchOnVideoRecordingEnd");
LOG.i("dispatchOnVideoRecordingEnd");
mUiHandler.post(new Runnable() {
@Override
public void run() {
@ -2380,6 +2387,12 @@ public class CameraView extends FrameLayout implements LifecycleObserver {
* {@link Frame} instances that can exist at a given moment in the frame pipeline,
* excluding frozen frames.
*
* Defaults to 2 - higher values will increase the memory usage with little benefit.
* Can be higher than 2 if {@link #setFrameProcessingExecutors(int)} is used.
* These values should be tuned together. We recommend setting a pool size that's equal to
* the number of executors plus 1, so that there's always a free Frame for the camera engine.
*
* Changing this value after camera initialization will have no effect.
* @param poolSize pool size
*/
public void setFrameProcessingPoolSize(int poolSize) {
@ -2395,6 +2408,49 @@ public class CameraView extends FrameLayout implements LifecycleObserver {
return mCameraEngine.getFrameProcessingPoolSize();
}
/**
* Sets the thread pool size for frame processing. This means that if the processing rate
* is slower than the preview rate, you can set this value to something bigger than 1
* to avoid losing frames.
* Defaults to 1 and this should be OK for most applications.
*
* Should be tuned depending on the task, the processor implementation, and along with
* {@link #setFrameProcessingPoolSize(int)}. We recommend choosing a pool size that is
* equal to the executors plus 1.
* @param executors thread count
*/
public void setFrameProcessingExecutors(int executors) {
if (executors < 1) {
throw new IllegalArgumentException("Need at least 1 executor, got " + executors);
}
mFrameProcessingExecutors = executors;
ThreadPoolExecutor executor = new ThreadPoolExecutor(
executors,
executors,
4,
TimeUnit.SECONDS,
new LinkedBlockingQueue<Runnable>(),
new ThreadFactory() {
private final AtomicInteger mCount = new AtomicInteger(1);
@Override
public Thread newThread(@NonNull Runnable r) {
return new Thread(r, "FrameExecutor #" + mCount.getAndIncrement());
}
}
);
executor.allowCoreThreadTimeOut(true);
mFrameProcessingExecutor = executor;
}
/**
* Returns the current executors count.
* @see #setFrameProcessingExecutors(int)
* @return thread count
*/
public int getFrameProcessingExecutors() {
return mFrameProcessingExecutors;
}
//endregion
//region Overlays

@ -29,6 +29,7 @@
<attr name="cameraFrameProcessingMaxHeight" format="integer|reference" />
<attr name="cameraFrameProcessingFormat" format="integer|reference" />
<attr name="cameraFrameProcessingPoolSize" format="integer|reference" />
<attr name="cameraFrameProcessingExecutors" format="integer|reference" />
<attr name="cameraVideoBitRate" format="integer|reference" />
<attr name="cameraAudioBitRate" format="integer|reference" />

@ -41,6 +41,7 @@ apply new data to it. So:
- you can do your job synchronously in the `process()` method. This is **recommended**.
- if you must hold the `Frame` instance longer, use `frame = frame.freeze()` to get a frozen instance
that will not be affected. This is **discouraged** because it requires copying the whole array.
Also, starting from `v2.5.0`, this is not allowed when Camera2 is used.
### Process synchronously
@ -48,8 +49,9 @@ Processing synchronously, for the duration of the `process()` method, is the rec
processors, because it solves different issues:
- avoids the need of calling `frame = frame.freeze()` which is a very expensive operation
- the engine will **automatically drop frames** if the `process()` method is busy, so you'll only receive frames that you can handle
- we have already allocated a thread for you, so there's no need to create another
- the engine will **automatically drop frames** if the `process()` method is busy, so you'll only
receive frames that you can handle
- we have already allocated background threads for you, so there's no need to create another
Some frame consumers might have a built-in asynchronous behavior.
But you can still block the `process()` thread until the consumer has returned.
@ -117,13 +119,41 @@ cameraView.setFrameProcessingFormat(ImageFormat.YUV_422_888);
With the Camera1 engine, the incoming format will always be `ImageFormat.NV21`.
You can check which formats are available for use through `CameraOptions.getSupportedFrameProcessingFormats()`.
### Advanced: Thread Control
Starting from `v2.5.1`, you can control the number of background threads that are allocated
for frame processing work. This should further push you into perform processing actions synchronously
and can be useful if processing is very slow with respect to the preview frame rate, in order to
avoid dropping too many frames.
You can change the number of threads by calling `setFrameProcessingExecutors()`. Whenever you do,
we recommend that you also change the frame processing pool size to a compatible value.
The frame processing pool size is roughly the number of `Frame` instances that can exist at any given
moment. We recommend that this value is set to the number of executors plus 1. For example:
- Single threaded (default):
```java
cameraView.setFrameProcessingExecutors(1);
cameraView.setFrameProcessingPoolSize(2);
```
- Two threads:
```java
cameraView.setFrameProcessingExecutors(2);
cameraView.setFrameProcessingPoolSize(3);
```
### XML Attributes
```xml
<com.otaliastudios.cameraview.CameraView
app:cameraFrameProcessingMaxWidth="640"
app:cameraFrameProcessingMaxHeight="640"
app:cameraFrameProcessingFormat="0x23"/>
app:cameraFrameProcessingFormat="0x23"
app:cameraFrameProcessingPoolSize="2"
app:cameraFrameProcessingExecutors="1"/>
```
### Related APIs
@ -135,16 +165,20 @@ You can check which formats are available for use through `CameraOptions.getSupp
|`camera.clearFrameProcessors()`|`-`|Removes all `FrameProcessor`s.|
|`camera.setFrameProcessingMaxWidth(int)`|`-`|Sets the max width for incoming frames.|
|`camera.setFrameProcessingMaxHeight(int)`|`-`|Sets the max height for incoming frames.|
|`camera.getFrameProcessingMaxWidth()`|`int`|Gets the max width for incoming frames.|
|`camera.getFrameProcessingMaxHeight()`|`int`|Gets the max height for incoming frames.|
|`camera.getFrameProcessingMaxWidth()`|`int`|Returns the max width for incoming frames.|
|`camera.getFrameProcessingMaxHeight()`|`int`|Returns the max height for incoming frames.|
|`camera.setFrameProcessingFormat(int)`|`-`|Sets the desired format for incoming frames. Should be one of the ImageFormat constants.|
|`camera.getFrameProcessingFormat()`|`-`|Gets the format for incoming frames. One of the ImageFormat constants.|
|`camera.getFrameProcessingFormat()`|`-`|Returns the format for incoming frames. One of the ImageFormat constants.|
|`camera.setFrameProcessingPoolSize(int)`|`-`|Sets the frame pool size, roughly the number of Frames that can exist at any given moment. Defaults to 2, which fits all use cases unless you change the executors.|
|`camera.getFrameProcessingPoolSize()`|`-`|Returns the frame pool size.|
|`camera.setFrameProcessingExecutors(int)`|`-`|Sets the processing thread size. Defaults to 1, but can be increased if your processing is slow and you are dropping too many frames. This should always be tuned together with the frame pool size.|
|`camera.getFrameProcessingExecutors()`|`-`|Returns the processing thread size.|
|`frame.getDataClass()`|`Class<T>`|The class of the data returned by `getData()`. Either `byte[]` or `android.media.Image`.|
|`frame.getData()`|`T`|The current preview frame, in its original orientation.|
|`frame.getTime()`|`long`|The preview timestamp, in `System.currentTimeMillis()` reference.|
|`frame.getRotation()`|`int`|The rotation that should be applied to the byte array in order to see what the user sees.|
|`frame.getSize()`|`Size`|The frame size, before any rotation is applied, to access data.|
|`frame.getFormat()`|`int`|The frame `ImageFormat`. This will always be `ImageFormat.NV21` for now.|
|`frame.getFormat()`|`int`|The frame `ImageFormat`. Defaults to `ImageFormat.NV21` for Camera1 and `ImageFormat.YUV_420_888` for Camera2.|
|`frame.freeze()`|`Frame`|Clones this frame and makes it immutable. Can be expensive because requires copying the byte array.|
|`frame.release()`|`-`|Disposes the content of this frame. Should be used on frozen frames to release memory.|

@ -6,7 +6,7 @@ pre, code, pre code {
border: none;
border-radius: 0;
background-color: #f9f9f9;
font-size: 0.85em;
font-size: 1em;
}
.highlight {

Loading…
Cancel
Save