From 1d8b8d0ea29a42d364ae12540184a5bd2553d04b Mon Sep 17 00:00:00 2001 From: Mattia Iavarone Date: Mon, 15 Jul 2019 20:58:54 -0300 Subject: [PATCH] Reorder code and add long comments --- .../video/encoding/AudioMediaEncoder.java | 10 +- .../video/encoding/MediaEncoder.java | 176 +++++++++++++----- .../video/encoding/MediaEncoderEngine.java | 45 ++++- .../video/encoding/TextureMediaEncoder.java | 10 +- .../video/encoding/VideoMediaEncoder.java | 25 ++- 5 files changed, 188 insertions(+), 78 deletions(-) diff --git a/cameraview/src/main/java/com/otaliastudios/cameraview/video/encoding/AudioMediaEncoder.java b/cameraview/src/main/java/com/otaliastudios/cameraview/video/encoding/AudioMediaEncoder.java index 5978629c..076c43de 100644 --- a/cameraview/src/main/java/com/otaliastudios/cameraview/video/encoding/AudioMediaEncoder.java +++ b/cameraview/src/main/java/com/otaliastudios/cameraview/video/encoding/AudioMediaEncoder.java @@ -80,15 +80,10 @@ public class AudioMediaEncoder extends MediaEncoder { } public AudioMediaEncoder(@NonNull Config config) { + super("AudioEncoder"); mConfig = config; } - @NonNull - @Override - String getName() { - return "AudioEncoder"; - } - @EncoderThread @Override void onPrepare(@NonNull MediaEncoderEngine.Controller controller, long maxLengthMillis) { @@ -127,7 +122,8 @@ public class AudioMediaEncoder extends MediaEncoder { } @Override - void onRelease() { + protected void onStopped() { + super.onStopped(); mRequestStop = false; mEncoder = null; mRecorder = null; diff --git a/cameraview/src/main/java/com/otaliastudios/cameraview/video/encoding/MediaEncoder.java b/cameraview/src/main/java/com/otaliastudios/cameraview/video/encoding/MediaEncoder.java index b7881416..04831d9b 100644 --- a/cameraview/src/main/java/com/otaliastudios/cameraview/video/encoding/MediaEncoder.java +++ b/cameraview/src/main/java/com/otaliastudios/cameraview/video/encoding/MediaEncoder.java @@ -5,6 +5,7 @@ import android.media.MediaCodec; import android.media.MediaFormat; import android.os.Build; +import androidx.annotation.CallSuper; import androidx.annotation.NonNull; import androidx.annotation.Nullable; import androidx.annotation.RequiresApi; @@ -16,6 +17,44 @@ import java.nio.ByteBuffer; /** * Base class for single-track encoders, coordinated by a {@link MediaEncoderEngine}. + * For the lifecycle of this class, read comments in the engine class. + * + * This class manages a background thread and streamlines events on this thread + * which we call the {@link EncoderThread}: + * + * 1. When {@link #prepare(MediaEncoderEngine.Controller, long)} is called, we call + * {@link #onPrepare(MediaEncoderEngine.Controller, long)} on the encoder thread. + * + * 2. When {@link #start()} is called, we call {@link #onStart()} on the encoder thread. + * + * 3. When {@link #notify(String, Object)} is called, we call {@link #onEvent(String, Object)} + * on the encoder thread. + * + * 4. After starting, encoders are free to acquire an input buffer with + * {@link #tryAcquireInputBuffer(InputBuffer)} or {@link #acquireInputBuffer(InputBuffer)}. + * + * 5. After getting the input buffer, they are free to fill it with data. + * + * 6. After filling it with data, they are required to call {@link #encodeInputBuffer(InputBuffer)} + * for encoding to take place. + * + * 7. After this happens, or at regular intervals, or whenever they want, encoders can then + * call {@link #drainOutput(boolean)} with a false parameter to fetch the encoded data + * and pass it to the engine (so it can be written to the muxer). + * + * 8. When {@link #stop()} is called - either by the engine user, or as a consequence of having + * called {@link MediaEncoderEngine.Controller#requestStop(int)} - we call + * {@link #onStop()} on the encoder thread. + * + * 9. The {@link #onStop()} implementation should, as fast as possible, stop reading, signal the + * end of input stream (there are two ways to do so), and finally call + * {@link #drainOutput(boolean)} for the last time, with a true parameter. + * + * 10. Once everything is drained, we will call {@link #onStopped()}, on a unspecified thread. + * There, subclasses can perform extra cleanup of their own resources. + * + * For VIDEO encoders, things are much easier because we skip the whole input part. + * See description in {@link VideoMediaEncoder}. */ // https://github.com/saki4510t/AudioVideoRecordingSample/blob/master/app/src/main/java/com/serenegiant/encoder/MediaEncoder.java @RequiresApi(api = Build.VERSION_CODES.JELLY_BEAN_MR2) @@ -36,6 +75,20 @@ abstract class MediaEncoder { // Can't go too high or this is a bottleneck for the audio encoder. private final static int OUTPUT_TIMEOUT_US = 0; + private final static int STATE_NONE = 0; + private final static int STATE_PREPARING = 1; + private final static int STATE_PREPARED = 2; + private final static int STATE_STARTING = 3; + private final static int STATE_STARTED = 4; + // max timestamp was reached. we will keep draining but have asked the engine to stop us. + // this step can be skipped in case stop() is called from outside before a limit is reached. + private final static int STATE_LIMIT_REACHED = 5; + private final static int STATE_STOPPING = 6; + private final static int STATE_STOPPED = 7; + + private int mState = STATE_NONE; + private final String mName; + @SuppressWarnings("WeakerAccess") protected MediaCodec mMediaCodec; @@ -51,31 +104,55 @@ abstract class MediaEncoder { private boolean mMaxLengthReached; /** - * A readable name for the thread. + * Needs a readable name for the thread and for logging. + * @param name a name */ - @NonNull - abstract String getName(); + MediaEncoder(@NonNull String name) { + mName = name; + } + + private void setState(int newState) { + String newStateName = null; + switch (newState) { + case STATE_NONE: newStateName = "NONE"; break; + case STATE_PREPARING: newStateName = "PREPARING"; break; + case STATE_PREPARED: newStateName = "PREPARED"; break; + case STATE_STARTING: newStateName = "STARTING"; break; + case STATE_STARTED: newStateName = "STARTED"; break; + case STATE_LIMIT_REACHED: newStateName = "LIMIT_REACHED"; break; + case STATE_STOPPING: newStateName = "STOPPING"; break; + case STATE_STOPPED: newStateName = "STOPPED"; break; + } + LOG.w(mName, "setState:", newStateName); + mState = newState; + } /** * This encoder was attached to the engine. Keep the controller * and run the internal thread. * * NOTE: it's important to call {@link WorkerHandler#post(Runnable)} instead of run()! - * The internal actions can cause a stop/release, and due to how {@link WorkerHandler#run(Runnable)} - * works, we might have {@link #onStop()} or {@link #onRelease()} to be executed before + * The internal actions can cause a stop, and due to how {@link WorkerHandler#run(Runnable)} + * works, we might have {@link #onStop()} or {@link #onStopped()} to be executed before * the previous step has completed. */ final void prepare(@NonNull final MediaEncoderEngine.Controller controller, final long maxLengthMillis) { + if (mState >= STATE_PREPARING) { + LOG.e("Wrong state while preparing. Aborting.", mState); + return; + } + setState(STATE_PREPARING); mController = controller; mBufferInfo = new MediaCodec.BufferInfo(); mMaxLengthMillis = maxLengthMillis; - mWorker = WorkerHandler.get(getName()); - LOG.i(getName(), "Prepare was called. Posting."); + mWorker = WorkerHandler.get(mName); + LOG.i(mName, "Prepare was called. Posting."); mWorker.post(new Runnable() { @Override public void run() { - LOG.i(getName(), "Prepare was called. Executing."); + LOG.i(mName, "Prepare was called. Executing."); onPrepare(controller, maxLengthMillis); + setState(STATE_PREPARED); } }); } @@ -85,14 +162,22 @@ abstract class MediaEncoder { * in case the encoder needs to wait for a certain event * like a "frame available". * + * The {@link #STATE_STARTED} state will be set when draining for the + * first time (not when onStart ends). + * * NOTE: it's important to call {@link WorkerHandler#post(Runnable)} instead of run()! */ final void start() { - LOG.w(getName(), "Start was called. Posting."); + if (mState < STATE_PREPARED || mState >= STATE_STARTING) { + LOG.e("Wrong state while starting. Aborting.", mState); + return; + } + setState(STATE_STARTING); + LOG.w(mName, "Start was called. Posting."); mWorker.post(new Runnable() { @Override public void run() { - LOG.w(getName(), "Start was called. Executing."); + LOG.w(mName, "Start was called. Executing."); onStart(); } }); @@ -108,27 +193,36 @@ abstract class MediaEncoder { * @param data object */ final void notify(final @NonNull String event, final @Nullable Object data) { - LOG.v(getName(), "Notify was called. Posting."); + LOG.v(mName, "Notify was called. Posting."); mWorker.post(new Runnable() { @Override public void run() { - LOG.v(getName(), "Notify was called. Executing."); + LOG.v(mName, "Notify was called. Executing."); onEvent(event, data); } }); } /** - * Stop recording. + * Stop recording. This involves signaling the end of stream and draining + * all output left. + * + * The {@link #STATE_STOPPED} state will be set when draining for the + * last time (not when onStart ends). * * NOTE: it's important to call {@link WorkerHandler#post(Runnable)} instead of run()! */ final void stop() { - LOG.w(getName(), "Stop was called. Posting."); + if (mState >= STATE_LIMIT_REACHED) { + LOG.e("Wrong state while stopping. Aborting.", mState); + return; + } + setState(STATE_STOPPING); + LOG.w(mName, "Stop was called. Posting."); mWorker.post(new Runnable() { @Override public void run() { - LOG.w(getName(), "Stop was called. Executing."); + LOG.w(mName, "Stop was called. Executing."); onStop(); } }); @@ -165,7 +259,8 @@ abstract class MediaEncoder { abstract void onEvent(@NonNull String event, @Nullable Object data); /** - * Stop recording. + * Stop recording. This involves signaling the end of stream and draining + * all output left. */ @EncoderThread abstract void onStop(); @@ -173,27 +268,24 @@ abstract class MediaEncoder { /** * Called by {@link #drainOutput(boolean)} when we get an EOS signal (not necessarily in the * parameters, might also be through an input buffer flag). + * + * This is a good moment to release all resources, although the muxer might still + * be alive (we wait for the other Encoder, see MediaEncoderEngine.Controller). */ - private void release() { - LOG.w(getName(), "is being released. Notifying controller and releasing codecs."); + @CallSuper + protected void onStopped() { + LOG.w(mName, "is being released. Notifying controller and releasing codecs."); // TODO should we notify after this method? - mController.notifyReleased(mTrackIndex); + mController.notifyStopped(mTrackIndex); mMediaCodec.stop(); mMediaCodec.release(); mMediaCodec = null; mOutputBufferPool.clear(); mOutputBufferPool = null; mBuffers = null; - onRelease(); + setState(STATE_STOPPED); } - /** - * This is called when we are stopped. - * It is a good moment to release all resources, although the muxer - * might still be alive (we wait for the other Encoder, see Controller). - */ - abstract void onRelease(); - /** * Returns a new input buffer and index, waiting at most {@link #INPUT_TIMEOUT_US} if none is available. * Callers should check the boolean result - true if the buffer was filled. @@ -234,7 +326,7 @@ abstract class MediaEncoder { */ @SuppressWarnings("WeakerAccess") protected void encodeInputBuffer(InputBuffer buffer) { - LOG.v(getName(), "ENCODING - Buffer:", buffer.index, "Bytes:", buffer.length, "Presentation:", buffer.timestamp); + LOG.v(mName, "ENCODING - Buffer:", buffer.index, "Bytes:", buffer.length, "Presentation:", buffer.timestamp); if (buffer.isEndOfStream) { // send EOS mMediaCodec.queueInputBuffer(buffer.index, 0, 0, buffer.timestamp, MediaCodec.BUFFER_FLAG_END_OF_STREAM); @@ -244,16 +336,6 @@ abstract class MediaEncoder { } } - /** - * Signals the end of input stream. This is a Video only API, as in the normal case, - * we use input buffers to signal the end. In the video case, we don't have input buffers - * because we use an input surface instead. - */ - @SuppressWarnings("WeakerAccess") - protected void signalEndOfInputStream() { - mMediaCodec.signalEndOfInputStream(); - } - /** * Extracts all pending data that was written and encoded into {@link #mMediaCodec}, * and forwards it to the muxer. @@ -267,7 +349,7 @@ abstract class MediaEncoder { @SuppressLint("LogNotTimber") @SuppressWarnings("WeakerAccess") protected void drainOutput(boolean drainAll) { - LOG.v(getName(), "DRAINING - EOS:", drainAll); + LOG.v(mName, "DRAINING - EOS:", drainAll); if (mMediaCodec == null) { LOG.e("drain() was called before prepare() or after releasing."); return; @@ -289,7 +371,8 @@ abstract class MediaEncoder { // should happen before receiving buffers, and should only happen once if (mController.isStarted()) throw new RuntimeException("MediaFormat changed twice."); MediaFormat newFormat = mMediaCodec.getOutputFormat(); - mTrackIndex = mController.requestStart(newFormat); + mTrackIndex = mController.notifyStarted(newFormat); + setState(STATE_STARTED); mOutputBufferPool = new OutputBufferPool(mTrackIndex); } else if (encoderStatus < 0) { LOG.e("Unexpected result from dequeueOutputBuffer: " + encoderStatus); @@ -316,7 +399,7 @@ abstract class MediaEncoder { // and should be used for offsets only. // TODO find a better way, this causes sync issues. (+ note: this sends pts=0 at first) // mBufferInfo.presentationTimeUs = mLastPresentationTimeUs - mStartPresentationTimeUs; - LOG.v(getName(), "DRAINING - About to write(). Presentation:", mBufferInfo.presentationTimeUs); + LOG.v(mName, "DRAINING - About to write(). Presentation:", mBufferInfo.presentationTimeUs); // TODO fix the mBufferInfo being the same, then implement delayed writing in Controller // and remove the isStarted() check here. @@ -335,19 +418,20 @@ abstract class MediaEncoder { && !mMaxLengthReached && mStartPresentationTimeUs != Long.MIN_VALUE && mLastPresentationTimeUs - mStartPresentationTimeUs > mMaxLengthMillis * 1000) { - LOG.w(getName(), "DRAINING - Reached maxLength! mLastPresentationTimeUs:", mLastPresentationTimeUs, + LOG.w(mName, "DRAINING - Reached maxLength! mLastPresentationTimeUs:", mLastPresentationTimeUs, "mStartPresentationTimeUs:", mStartPresentationTimeUs, "mMaxLengthUs:", mMaxLengthMillis * 1000); mMaxLengthReached = true; - LOG.w(getName(), "DRAINING - Requesting a stop."); + LOG.w(mName, "DRAINING - Requesting a stop."); + setState(STATE_LIMIT_REACHED); mController.requestStop(mTrackIndex); break; } - // Check for the EOS flag so we can release the encoder. + // Check for the EOS flag so we can call onStopped. if ((mBufferInfo.flags & MediaCodec.BUFFER_FLAG_END_OF_STREAM) != 0) { - LOG.w(getName(), "DRAINING - Got EOS. Releasing the codec."); - release(); + LOG.w(mName, "DRAINING - Got EOS. Releasing the codec."); + onStopped(); break; } } diff --git a/cameraview/src/main/java/com/otaliastudios/cameraview/video/encoding/MediaEncoderEngine.java b/cameraview/src/main/java/com/otaliastudios/cameraview/video/encoding/MediaEncoderEngine.java index cdc079c1..8842e609 100644 --- a/cameraview/src/main/java/com/otaliastudios/cameraview/video/encoding/MediaEncoderEngine.java +++ b/cameraview/src/main/java/com/otaliastudios/cameraview/video/encoding/MediaEncoderEngine.java @@ -16,6 +16,35 @@ import java.util.ArrayList; /** * The entry point for encoding video files. + * + * The external API is simple but the internal mechanism is not easy. Basically the engine + * controls a {@link MediaEncoder} instance for each track (e.g. one for video, one for audio). + * + * 1. We prepare the MediaEncoders: {@link MediaEncoder#prepare(Controller, long)} + * MediaEncoders can be prepared synchronously or not. + * + * 2. Someone calls {@link #start()} from any thread. + * As a consequence, we start the MediaEncoders: {@link MediaEncoder#start()}. + * + * 3. MediaEncoders do not start synchronously. Instead, they call + * {@link Controller#notifyStarted(MediaFormat)} when they have a legit format, + * and we keep track of who has started. + * + * 4. When all MediaEncoders have started, we actually start the muxer. + * + * 5. Someone calls {@link #stop()} from any thread. + * As a consequence, we stop the MediaEncoders: {@link MediaEncoder#stop()}. + * + * 6. MediaEncoders do not stop synchronously. Instead, they will stop reading but + * keep draining the codec until there's no data left. At that point, they can + * call {@link Controller#notifyStopped(int)}. + * + * 7. When all MediaEncoders have been released, we actually stop the muxer and notify. + * + * There is another possibility where MediaEncoders themselves want to stop, for example + * because they reach some limit or constraint (e.g. max duration). For this, they should + * call {@link Controller#requestStop(int)}. Once all MediaEncoders have stopped, we will + * actually call {@link #stop()} on ourselves. */ @RequiresApi(api = Build.VERSION_CODES.JELLY_BEAN_MR2) public class MediaEncoderEngine { @@ -148,7 +177,7 @@ public class MediaEncoderEngine { /** * Asks encoders to stop. This is not sync, of course we will ask for encoders - * to call {@link Controller#notifyReleased(int)} before actually stop the muxer. + * to call {@link Controller#notifyStopped(int)} before actually stop the muxer. * When all encoders request a release, {@link #end()} is called to do cleanup * and notify the listener. */ @@ -160,7 +189,7 @@ public class MediaEncoderEngine { } /** - * Called after all encoders have requested a release using {@link Controller#notifyReleased(int)}. + * Called after all encoders have requested a release using {@link Controller#notifyStopped(int)}. * At this point we will do cleanup and notify the listener. */ private void end() { @@ -225,15 +254,15 @@ public class MediaEncoderEngine { * @param format the media format * @return the encoder track index */ - int requestStart(@NonNull MediaFormat format) { + int notifyStarted(@NonNull MediaFormat format) { synchronized (mControllerLock) { if (mMediaMuxerStarted) { throw new IllegalStateException("Trying to start but muxer started already"); } int track = mMediaMuxer.addTrack(format); - LOG.w("requestStart:", "Assigned track", track, "to format", format.getString(MediaFormat.KEY_MIME)); + LOG.w("notifyStarted:", "Assigned track", track, "to format", format.getString(MediaFormat.KEY_MIME)); if (++mStartedEncodersCount == mEncoders.size()) { - LOG.w("requestStart:", "All encoders have started. Starting muxer and dispatching onEncodingStart()."); + LOG.w("notifyStarted:", "All encoders have started. Starting muxer and dispatching onEncodingStart()."); mMediaMuxer.start(); mMediaMuxerStarted = true; if (mListener != null) { @@ -274,7 +303,7 @@ public class MediaEncoderEngine { /** * Requests that the engine stops. This is not executed until all encoders call - * this method, so it is a kind of soft request, just like {@link #requestStart(MediaFormat)}. + * this method, so it is a kind of soft request, just like {@link #notifyStarted(MediaFormat)}. * To be used when maxLength / maxSize constraints are reached, for example. * * When this succeeds, {@link MediaEncoder#stop()} is called. @@ -294,9 +323,9 @@ public class MediaEncoderEngine { * Notifies that the encoder was stopped. After this is called by all encoders, * we will actually stop the muxer. */ - void notifyReleased(int track) { + void notifyStopped(int track) { synchronized (mControllerLock) { - LOG.w("notifyReleased:", "Called for track", track); + LOG.w("notifyStopped:", "Called for track", track); if (++mReleasedEncodersCount == mEncoders.size()) { LOG.w("requestStop:", "All encoders have been released. Stopping the muxer."); end(); diff --git a/cameraview/src/main/java/com/otaliastudios/cameraview/video/encoding/TextureMediaEncoder.java b/cameraview/src/main/java/com/otaliastudios/cameraview/video/encoding/TextureMediaEncoder.java index 2e9e275b..e97d97c6 100644 --- a/cameraview/src/main/java/com/otaliastudios/cameraview/video/encoding/TextureMediaEncoder.java +++ b/cameraview/src/main/java/com/otaliastudios/cameraview/video/encoding/TextureMediaEncoder.java @@ -100,13 +100,6 @@ public class TextureMediaEncoder extends VideoMediaEncoder the config object. */ @@ -53,15 +62,10 @@ abstract class VideoMediaEncoder extends Med } VideoMediaEncoder(@NonNull C config) { + super("VideoEncoder"); mConfig = config; } - @NonNull - @Override - String getName() { - return "VideoEncoder"; - } - @EncoderThread @Override void onPrepare(@NonNull MediaEncoderEngine.Controller controller, long maxLengthMillis) { @@ -99,7 +103,10 @@ abstract class VideoMediaEncoder extends Med void onStop() { LOG.i("onStop", "setting mFrameNum to 1 and signaling the end of input stream."); mFrameNum = -1; - signalEndOfInputStream(); + // Signals the end of input stream. This is a Video only API, as in the normal case, + // we use input buffers to signal the end. In the video case, we don't have input buffers + // because we use an input surface instead. + mMediaCodec.signalEndOfInputStream(); drainOutput(true); }