From 122e4a159773321f0b087c22252190a481a8d3c7 Mon Sep 17 00:00:00 2001 From: Mattia Iavarone Date: Fri, 21 Jun 2019 17:50:28 -0500 Subject: [PATCH] Refactor encoding package --- .../cameraview/CameraPreviewTest.java | 4 +- .../cameraview/engine/Camera1Engine.java | 3 +- .../cameraview/internal/utils/Pool.java | 2 +- .../cameraview/preview/CameraPreview.java | 5 +- .../preview/TextureCameraPreview.java | 4 +- .../video/SnapshotVideoRecorder.java | 2 + .../video/encoding/AudioMediaEncoder.java | 3 + .../video/encoding/ByteBufferPool.java | 5 +- .../video/encoding/EncoderThread.java | 5 +- .../video/encoding/InputBuffer.java | 4 + .../video/encoding/InputBufferPool.java | 5 +- .../video/encoding/MediaCodecBuffers.java | 9 +- .../video/encoding/MediaEncoder.java | 6 + .../video/encoding/MediaEncoderEngine.java | 200 +++++++++++------- .../video/encoding/OutputBuffer.java | 5 + .../video/encoding/OutputBufferPool.java | 9 +- .../video/encoding/TextureMediaEncoder.java | 5 +- 17 files changed, 185 insertions(+), 91 deletions(-) diff --git a/cameraview/src/androidTest/java/com/otaliastudios/cameraview/CameraPreviewTest.java b/cameraview/src/androidTest/java/com/otaliastudios/cameraview/CameraPreviewTest.java index 7c62e844..660005ae 100644 --- a/cameraview/src/androidTest/java/com/otaliastudios/cameraview/CameraPreviewTest.java +++ b/cameraview/src/androidTest/java/com/otaliastudios/cameraview/CameraPreviewTest.java @@ -107,7 +107,7 @@ public abstract class CameraPreviewTest extends BaseTest { @Test public void testDesiredSize() { - preview.setStreamSize(160, 90, false); + preview.setStreamSize(160, 90); assertEquals(160, preview.getStreamSize().getWidth()); assertEquals(90, preview.getStreamSize().getHeight()); } @@ -159,7 +159,7 @@ public abstract class CameraPreviewTest extends BaseTest { private void setDesiredAspectRatio(float desiredAspectRatio) { preview.mCropTask.listen(); - preview.setStreamSize((int) (10f * desiredAspectRatio), 10, false); // Wait... + preview.setStreamSize((int) (10f * desiredAspectRatio), 10); // Wait... preview.mCropTask.await(); assertEquals(desiredAspectRatio, getViewAspectRatioWithScale(), 0.01f); diff --git a/cameraview/src/main/java/com/otaliastudios/cameraview/engine/Camera1Engine.java b/cameraview/src/main/java/com/otaliastudios/cameraview/engine/Camera1Engine.java index 765c80ed..383b2f83 100644 --- a/cameraview/src/main/java/com/otaliastudios/cameraview/engine/Camera1Engine.java +++ b/cameraview/src/main/java/com/otaliastudios/cameraview/engine/Camera1Engine.java @@ -207,8 +207,7 @@ public class Camera1Engine extends CameraEngine implements Camera.PreviewCallbac if (previewSize == null) { throw new IllegalStateException("previewStreamSize should not be null at this point."); } - boolean wasFlipped = flip(REF_SENSOR, REF_VIEW); - mPreview.setStreamSize(previewSize.getWidth(), previewSize.getHeight(), wasFlipped); + mPreview.setStreamSize(previewSize.getWidth(), previewSize.getHeight()); Camera.Parameters params = mCamera.getParameters(); mPreviewStreamFormat = params.getPreviewFormat(); diff --git a/cameraview/src/main/java/com/otaliastudios/cameraview/internal/utils/Pool.java b/cameraview/src/main/java/com/otaliastudios/cameraview/internal/utils/Pool.java index 29e53c59..b59a0819 100644 --- a/cameraview/src/main/java/com/otaliastudios/cameraview/internal/utils/Pool.java +++ b/cameraview/src/main/java/com/otaliastudios/cameraview/internal/utils/Pool.java @@ -12,7 +12,7 @@ import androidx.annotation.Nullable; * Base class for pools of recycleable objects. * @param the object type */ -class Pool { +public class Pool { private static final String TAG = Pool.class.getSimpleName(); private static final CameraLogger LOG = CameraLogger.create(TAG); diff --git a/cameraview/src/main/java/com/otaliastudios/cameraview/preview/CameraPreview.java b/cameraview/src/main/java/com/otaliastudios/cameraview/preview/CameraPreview.java index 27bb9375..b89272d0 100644 --- a/cameraview/src/main/java/com/otaliastudios/cameraview/preview/CameraPreview.java +++ b/cameraview/src/main/java/com/otaliastudios/cameraview/preview/CameraPreview.java @@ -58,7 +58,6 @@ public abstract class CameraPreview { // These are the preview stream dimensions, in REF_VIEW. int mInputStreamWidth; int mInputStreamHeight; - private boolean mInputFlipped; /** * Creates a new preview. @@ -134,13 +133,11 @@ public abstract class CameraPreview { * * @param width width of the preview stream, in view coordinates * @param height height of the preview stream, in view coordinates - * @param wasFlipped whether width and height were flipped before calling this */ - public void setStreamSize(int width, int height, boolean wasFlipped) { + public void setStreamSize(int width, int height) { LOG.i("setStreamSize:", "desiredW=", width, "desiredH=", height); mInputStreamWidth = width; mInputStreamHeight = height; - mInputFlipped = wasFlipped; if (mInputStreamWidth > 0 && mInputStreamHeight > 0) { crop(mCropTask); } diff --git a/cameraview/src/main/java/com/otaliastudios/cameraview/preview/TextureCameraPreview.java b/cameraview/src/main/java/com/otaliastudios/cameraview/preview/TextureCameraPreview.java index bfd47adf..a74ccb88 100644 --- a/cameraview/src/main/java/com/otaliastudios/cameraview/preview/TextureCameraPreview.java +++ b/cameraview/src/main/java/com/otaliastudios/cameraview/preview/TextureCameraPreview.java @@ -78,8 +78,8 @@ public class TextureCameraPreview extends CameraPreview { ByteBufferPool(final int bufferSize, int maxPoolSize) { diff --git a/cameraview/src/main/java/com/otaliastudios/cameraview/video/encoding/EncoderThread.java b/cameraview/src/main/java/com/otaliastudios/cameraview/video/encoding/EncoderThread.java index 2e65d1a7..3d5964f4 100644 --- a/cameraview/src/main/java/com/otaliastudios/cameraview/video/encoding/EncoderThread.java +++ b/cameraview/src/main/java/com/otaliastudios/cameraview/video/encoding/EncoderThread.java @@ -1,3 +1,6 @@ package com.otaliastudios.cameraview.video.encoding; -@interface EncoderThread {} +/** + * Indicates that some action is being executed on the encoder thread. + */ +public @interface EncoderThread {} diff --git a/cameraview/src/main/java/com/otaliastudios/cameraview/video/encoding/InputBuffer.java b/cameraview/src/main/java/com/otaliastudios/cameraview/video/encoding/InputBuffer.java index dc715699..979ef268 100644 --- a/cameraview/src/main/java/com/otaliastudios/cameraview/video/encoding/InputBuffer.java +++ b/cameraview/src/main/java/com/otaliastudios/cameraview/video/encoding/InputBuffer.java @@ -2,6 +2,10 @@ package com.otaliastudios.cameraview.video.encoding; import java.nio.ByteBuffer; +/** + * Represents an input buffer, which means, + * raw data that should be encoded by MediaCodec. + */ class InputBuffer { ByteBuffer data; ByteBuffer source; diff --git a/cameraview/src/main/java/com/otaliastudios/cameraview/video/encoding/InputBufferPool.java b/cameraview/src/main/java/com/otaliastudios/cameraview/video/encoding/InputBufferPool.java index 65f4ebde..d8124e47 100644 --- a/cameraview/src/main/java/com/otaliastudios/cameraview/video/encoding/InputBufferPool.java +++ b/cameraview/src/main/java/com/otaliastudios/cameraview/video/encoding/InputBufferPool.java @@ -1,7 +1,10 @@ package com.otaliastudios.cameraview.video.encoding; -import com.otaliastudios.cameraview.utils.Pool; +import com.otaliastudios.cameraview.internal.utils.Pool; +/** + * A simple {@link Pool(int, Factory)} implementation for input buffers. + */ class InputBufferPool extends Pool { InputBufferPool() { diff --git a/cameraview/src/main/java/com/otaliastudios/cameraview/video/encoding/MediaCodecBuffers.java b/cameraview/src/main/java/com/otaliastudios/cameraview/video/encoding/MediaCodecBuffers.java index 3cec938a..1b293cc4 100644 --- a/cameraview/src/main/java/com/otaliastudios/cameraview/video/encoding/MediaCodecBuffers.java +++ b/cameraview/src/main/java/com/otaliastudios/cameraview/video/encoding/MediaCodecBuffers.java @@ -5,10 +5,13 @@ import android.os.Build; import java.nio.ByteBuffer; +import androidx.annotation.RequiresApi; + /** * A Wrapper to MediaCodec that facilitates the use of API-dependent get{Input/Output}Buffer methods, * in order to prevent: http://stackoverflow.com/q/30646885 */ +@RequiresApi(Build.VERSION_CODES.JELLY_BEAN_MR2) class MediaCodecBuffers { private final MediaCodec mMediaCodec; @@ -26,7 +29,7 @@ class MediaCodecBuffers { } } - public ByteBuffer getInputBuffer(final int index) { + ByteBuffer getInputBuffer(final int index) { if (Build.VERSION.SDK_INT >= 21) { return mMediaCodec.getInputBuffer(index); } @@ -35,14 +38,14 @@ class MediaCodecBuffers { return buffer; } - public ByteBuffer getOutputBuffer(final int index) { + ByteBuffer getOutputBuffer(final int index) { if (Build.VERSION.SDK_INT >= 21) { return mMediaCodec.getOutputBuffer(index); } return mOutputBuffers[index]; } - public void onOutputBuffersChanged() { + void onOutputBuffersChanged() { if (Build.VERSION.SDK_INT < 21) { mOutputBuffers = mMediaCodec.getOutputBuffers(); } diff --git a/cameraview/src/main/java/com/otaliastudios/cameraview/video/encoding/MediaEncoder.java b/cameraview/src/main/java/com/otaliastudios/cameraview/video/encoding/MediaEncoder.java index c0b5023e..21fea433 100644 --- a/cameraview/src/main/java/com/otaliastudios/cameraview/video/encoding/MediaEncoder.java +++ b/cameraview/src/main/java/com/otaliastudios/cameraview/video/encoding/MediaEncoder.java @@ -14,6 +14,9 @@ import com.otaliastudios.cameraview.internal.utils.WorkerHandler; import java.nio.ByteBuffer; +/** + * Base class for single-track encoders, coordinated by a {@link MediaEncoderEngine}. + */ // https://github.com/saki4510t/AudioVideoRecordingSample/blob/master/app/src/main/java/com/serenegiant/encoder/MediaEncoder.java @RequiresApi(api = Build.VERSION_CODES.JELLY_BEAN_MR2) abstract class MediaEncoder { @@ -296,6 +299,9 @@ abstract class MediaEncoder { // TODO fix the mBufferInfo being the same, then implement delayed writing in Controller // and remove the isStarted() check here. OutputBuffer buffer = mOutputBufferPool.get(); + if (buffer == null) { + throw new IllegalStateException("buffer is null!"); + } buffer.info = mBufferInfo; buffer.trackIndex = mTrackIndex; buffer.data = encodedData; diff --git a/cameraview/src/main/java/com/otaliastudios/cameraview/video/encoding/MediaEncoderEngine.java b/cameraview/src/main/java/com/otaliastudios/cameraview/video/encoding/MediaEncoderEngine.java index e719e191..8ba29f56 100644 --- a/cameraview/src/main/java/com/otaliastudios/cameraview/video/encoding/MediaEncoderEngine.java +++ b/cameraview/src/main/java/com/otaliastudios/cameraview/video/encoding/MediaEncoderEngine.java @@ -14,9 +14,28 @@ import java.io.File; import java.io.IOException; import java.util.ArrayList; +/** + * The entry point for encoding video files. + */ @RequiresApi(api = Build.VERSION_CODES.JELLY_BEAN_MR2) public class MediaEncoderEngine { + /** + * Receives the stop event callback to know when the video + * was written (or what went wrong). + */ + public interface Listener { + + /** + * Called when encoding stopped for some reason. + * If there's an exception, it failed. + * @param stopReason the reason + * @param e the error, if present + */ + @EncoderThread + void onEncoderStop(int stopReason, @Nullable Exception e); + } + private final static String TAG = MediaEncoderEngine.class.getSimpleName(); private final static CameraLogger LOG = CameraLogger.create(TAG); @@ -30,14 +49,30 @@ public class MediaEncoderEngine { private int mStartedEncodersCount; private int mStoppedEncodersCount; private boolean mMediaMuxerStarted; + @SuppressWarnings("FieldCanBeLocal") private Controller mController; private Listener mListener; private int mStopReason = STOP_BY_USER; private int mPossibleStopReason; private final Object mControllerLock = new Object(); - public MediaEncoderEngine(@NonNull File file, @NonNull VideoMediaEncoder videoEncoder, @Nullable AudioMediaEncoder audioEncoder, - final int maxDuration, final long maxSize, @Nullable Listener listener) { + /** + * Creates a new engine for the given file, with the given encoders and max limits, + * and listener to receive events. + * + * @param file output file + * @param videoEncoder video encoder to use + * @param audioEncoder audio encoder to use + * @param maxDuration max duration in millis + * @param maxSize max size + * @param listener a listener + */ + public MediaEncoderEngine(@NonNull File file, + @NonNull VideoMediaEncoder videoEncoder, + @Nullable AudioMediaEncoder audioEncoder, + final int maxDuration, + final long maxSize, + @Nullable Listener listener) { mListener = listener; mController = new Controller(); mEncoders = new ArrayList<>(); @@ -81,7 +116,95 @@ public class MediaEncoderEngine { } } - // Stuff here might be called from multiple threads. + /** + * Asks encoders to start (each one on its own track). + */ + public final void start() { + for (MediaEncoder encoder : mEncoders) { + encoder.start(); + } + } + + /** + * Notifies encoders of some event with the given payload. + * Can be used for example to notify the video encoder of new frame available. + * @param event an event string + * @param data an event payload + */ + @SuppressWarnings("SameParameterValue") + public final void notify(final String event, final Object data) { + for (MediaEncoder encoder : mEncoders) { + encoder.notify(event, data); + } + } + + /** + * Asks encoders to stop. This is not sync, of course we will ask for encoders + * to call {@link Controller#requestRelease(int)} before actually stop the muxer. + * When all encoders request a release, {@link #release()} is called to do cleanup + * and notify the listener. + */ + public final void stop() { + for (MediaEncoder encoder : mEncoders) { + encoder.stop(); + } + } + + /** + * Called after all encoders have requested a release using {@link Controller#requestRelease(int)}. + * At this point we will do cleanup and notify the listener. + */ + private void release() { + Exception error = null; + if (mMediaMuxer != null) { + // stop() throws an exception if you haven't fed it any data. + // But also in other occasions. So this is a signal that something + // went wrong, and we propagate that to the listener. + try { + mMediaMuxer.stop(); + mMediaMuxer.release(); + } catch (Exception e) { + error = e; + } + mMediaMuxer = null; + } + if (mListener != null) { + mListener.onEncoderStop(mStopReason, error); + mListener = null; + } + mStopReason = STOP_BY_USER; + mStartedEncodersCount = 0; + mStoppedEncodersCount = 0; + mMediaMuxerStarted = false; + } + + /** + * Returns the current video encoder. + * @return the current video encoder + */ + @NonNull + public VideoMediaEncoder getVideoEncoder() { + return (VideoMediaEncoder) mEncoders.get(0); + } + + /** + * Returns the current audio encoder. + * @return the current audio encoder + */ + @SuppressWarnings("unused") + @Nullable + public AudioMediaEncoder getAudioEncoder() { + if (mEncoders.size() > 1) { + return (AudioMediaEncoder) mEncoders.get(1); + } else { + return null; + } + } + + /** + * A handle for {@link MediaEncoder}s to pass information to this engine. + * All methods here can be called for multiple threads. + */ class Controller { /** @@ -90,7 +213,7 @@ public class MediaEncoderEngine { * @param format the media format * @return the encoder track index */ - int requestStart(MediaFormat format) { + int requestStart(@NonNull MediaFormat format) { synchronized (mControllerLock) { if (mMediaMuxerStarted) { throw new IllegalStateException("Trying to start but muxer started already"); @@ -121,7 +244,7 @@ public class MediaEncoderEngine { * TODO cache values if not started yet, then apply later. Read comments in drain(). * Currently they are recycled instantly. */ - void write(OutputBufferPool pool, OutputBuffer buffer) { + void write(@NonNull OutputBufferPool pool, @NonNull OutputBuffer buffer) { if (!mMediaMuxerStarted) { throw new IllegalStateException("Trying to write before muxer started"); } @@ -163,71 +286,4 @@ public class MediaEncoderEngine { } } } - - public final void start() { - for (MediaEncoder encoder : mEncoders) { - encoder.start(); - } - } - - @SuppressWarnings("SameParameterValue") - public final void notify(final String event, final Object data) { - for (MediaEncoder encoder : mEncoders) { - encoder.notify(event, data); - } - } - - /** - * This just asks the encoder to stop. We will wait for them to call {@link Controller#requestRelease(int)} - * to actually stop the muxer, as there might be async stuff going on. - */ - public final void stop() { - for (MediaEncoder encoder : mEncoders) { - encoder.stop(); - } - } - - private void release() { - Exception error = null; - if (mMediaMuxer != null) { - // stop() throws an exception if you haven't fed it any data. - // But also in other occasions. So this is a signal that something - // went wrong, and we propagate that to the listener. - try { - mMediaMuxer.stop(); - mMediaMuxer.release(); - } catch (Exception e) { - error = e; - } - mMediaMuxer = null; - } - if (mListener != null) { - mListener.onEncoderStop(mStopReason, error); - mListener = null; - } - mStopReason = STOP_BY_USER; - mStartedEncodersCount = 0; - mStoppedEncodersCount = 0; - mMediaMuxerStarted = false; - } - - @NonNull - public VideoMediaEncoder getVideoEncoder() { - return (VideoMediaEncoder) mEncoders.get(0); - } - - @Nullable - public AudioMediaEncoder getAudioEncoder() { - if (mEncoders.size() > 1) { - return (AudioMediaEncoder) mEncoders.get(1); - } else { - return null; - } - } - - public interface Listener { - - @EncoderThread - void onEncoderStop(int stopReason, @Nullable Exception e); - } } diff --git a/cameraview/src/main/java/com/otaliastudios/cameraview/video/encoding/OutputBuffer.java b/cameraview/src/main/java/com/otaliastudios/cameraview/video/encoding/OutputBuffer.java index 357e73b8..711acfdb 100644 --- a/cameraview/src/main/java/com/otaliastudios/cameraview/video/encoding/OutputBuffer.java +++ b/cameraview/src/main/java/com/otaliastudios/cameraview/video/encoding/OutputBuffer.java @@ -4,6 +4,11 @@ import android.media.MediaCodec; import java.nio.ByteBuffer; +/** + * Represents an output buffer, which means, + * an encoded buffer of data that should be passed + * to the muxer. + */ class OutputBuffer { MediaCodec.BufferInfo info; int trackIndex; diff --git a/cameraview/src/main/java/com/otaliastudios/cameraview/video/encoding/OutputBufferPool.java b/cameraview/src/main/java/com/otaliastudios/cameraview/video/encoding/OutputBufferPool.java index 301503df..d1a145a4 100644 --- a/cameraview/src/main/java/com/otaliastudios/cameraview/video/encoding/OutputBufferPool.java +++ b/cameraview/src/main/java/com/otaliastudios/cameraview/video/encoding/OutputBufferPool.java @@ -1,9 +1,16 @@ package com.otaliastudios.cameraview.video.encoding; import android.media.MediaCodec; +import android.os.Build; -import com.otaliastudios.cameraview.utils.Pool; +import com.otaliastudios.cameraview.internal.utils.Pool; +import androidx.annotation.RequiresApi; + +/** + * A simple {@link Pool(int, Factory)} implementation for output buffers. + */ +@RequiresApi(Build.VERSION_CODES.JELLY_BEAN_MR2) class OutputBufferPool extends Pool { OutputBufferPool(final int trackIndex) { diff --git a/cameraview/src/main/java/com/otaliastudios/cameraview/video/encoding/TextureMediaEncoder.java b/cameraview/src/main/java/com/otaliastudios/cameraview/video/encoding/TextureMediaEncoder.java index e15a954d..19cdf45a 100644 --- a/cameraview/src/main/java/com/otaliastudios/cameraview/video/encoding/TextureMediaEncoder.java +++ b/cameraview/src/main/java/com/otaliastudios/cameraview/video/encoding/TextureMediaEncoder.java @@ -8,12 +8,15 @@ import com.otaliastudios.cameraview.CameraLogger; import com.otaliastudios.cameraview.internal.egl.EglCore; import com.otaliastudios.cameraview.internal.egl.EglViewport; import com.otaliastudios.cameraview.internal.egl.EglWindowSurface; -import com.otaliastudios.cameraview.utils.Pool; +import com.otaliastudios.cameraview.internal.utils.Pool; import androidx.annotation.NonNull; import androidx.annotation.Nullable; import androidx.annotation.RequiresApi; +/** + * Default implementation for video encoding. + */ @RequiresApi(api = Build.VERSION_CODES.JELLY_BEAN_MR2) public class TextureMediaEncoder extends VideoMediaEncoder {