Reorder code and add long comments

pull/506/head
Mattia Iavarone 6 years ago
parent ea952d1497
commit 1d8b8d0ea2
  1. 10
      cameraview/src/main/java/com/otaliastudios/cameraview/video/encoding/AudioMediaEncoder.java
  2. 176
      cameraview/src/main/java/com/otaliastudios/cameraview/video/encoding/MediaEncoder.java
  3. 45
      cameraview/src/main/java/com/otaliastudios/cameraview/video/encoding/MediaEncoderEngine.java
  4. 10
      cameraview/src/main/java/com/otaliastudios/cameraview/video/encoding/TextureMediaEncoder.java
  5. 25
      cameraview/src/main/java/com/otaliastudios/cameraview/video/encoding/VideoMediaEncoder.java

@ -80,15 +80,10 @@ public class AudioMediaEncoder extends MediaEncoder {
}
public AudioMediaEncoder(@NonNull Config config) {
super("AudioEncoder");
mConfig = config;
}
@NonNull
@Override
String getName() {
return "AudioEncoder";
}
@EncoderThread
@Override
void onPrepare(@NonNull MediaEncoderEngine.Controller controller, long maxLengthMillis) {
@ -127,7 +122,8 @@ public class AudioMediaEncoder extends MediaEncoder {
}
@Override
void onRelease() {
protected void onStopped() {
super.onStopped();
mRequestStop = false;
mEncoder = null;
mRecorder = null;

@ -5,6 +5,7 @@ import android.media.MediaCodec;
import android.media.MediaFormat;
import android.os.Build;
import androidx.annotation.CallSuper;
import androidx.annotation.NonNull;
import androidx.annotation.Nullable;
import androidx.annotation.RequiresApi;
@ -16,6 +17,44 @@ import java.nio.ByteBuffer;
/**
* Base class for single-track encoders, coordinated by a {@link MediaEncoderEngine}.
* For the lifecycle of this class, read comments in the engine class.
*
* This class manages a background thread and streamlines events on this thread
* which we call the {@link EncoderThread}:
*
* 1. When {@link #prepare(MediaEncoderEngine.Controller, long)} is called, we call
* {@link #onPrepare(MediaEncoderEngine.Controller, long)} on the encoder thread.
*
* 2. When {@link #start()} is called, we call {@link #onStart()} on the encoder thread.
*
* 3. When {@link #notify(String, Object)} is called, we call {@link #onEvent(String, Object)}
* on the encoder thread.
*
* 4. After starting, encoders are free to acquire an input buffer with
* {@link #tryAcquireInputBuffer(InputBuffer)} or {@link #acquireInputBuffer(InputBuffer)}.
*
* 5. After getting the input buffer, they are free to fill it with data.
*
* 6. After filling it with data, they are required to call {@link #encodeInputBuffer(InputBuffer)}
* for encoding to take place.
*
* 7. After this happens, or at regular intervals, or whenever they want, encoders can then
* call {@link #drainOutput(boolean)} with a false parameter to fetch the encoded data
* and pass it to the engine (so it can be written to the muxer).
*
* 8. When {@link #stop()} is called - either by the engine user, or as a consequence of having
* called {@link MediaEncoderEngine.Controller#requestStop(int)} - we call
* {@link #onStop()} on the encoder thread.
*
* 9. The {@link #onStop()} implementation should, as fast as possible, stop reading, signal the
* end of input stream (there are two ways to do so), and finally call
* {@link #drainOutput(boolean)} for the last time, with a true parameter.
*
* 10. Once everything is drained, we will call {@link #onStopped()}, on a unspecified thread.
* There, subclasses can perform extra cleanup of their own resources.
*
* For VIDEO encoders, things are much easier because we skip the whole input part.
* See description in {@link VideoMediaEncoder}.
*/
// https://github.com/saki4510t/AudioVideoRecordingSample/blob/master/app/src/main/java/com/serenegiant/encoder/MediaEncoder.java
@RequiresApi(api = Build.VERSION_CODES.JELLY_BEAN_MR2)
@ -36,6 +75,20 @@ abstract class MediaEncoder {
// Can't go too high or this is a bottleneck for the audio encoder.
private final static int OUTPUT_TIMEOUT_US = 0;
private final static int STATE_NONE = 0;
private final static int STATE_PREPARING = 1;
private final static int STATE_PREPARED = 2;
private final static int STATE_STARTING = 3;
private final static int STATE_STARTED = 4;
// max timestamp was reached. we will keep draining but have asked the engine to stop us.
// this step can be skipped in case stop() is called from outside before a limit is reached.
private final static int STATE_LIMIT_REACHED = 5;
private final static int STATE_STOPPING = 6;
private final static int STATE_STOPPED = 7;
private int mState = STATE_NONE;
private final String mName;
@SuppressWarnings("WeakerAccess")
protected MediaCodec mMediaCodec;
@ -51,31 +104,55 @@ abstract class MediaEncoder {
private boolean mMaxLengthReached;
/**
* A readable name for the thread.
* Needs a readable name for the thread and for logging.
* @param name a name
*/
@NonNull
abstract String getName();
MediaEncoder(@NonNull String name) {
mName = name;
}
private void setState(int newState) {
String newStateName = null;
switch (newState) {
case STATE_NONE: newStateName = "NONE"; break;
case STATE_PREPARING: newStateName = "PREPARING"; break;
case STATE_PREPARED: newStateName = "PREPARED"; break;
case STATE_STARTING: newStateName = "STARTING"; break;
case STATE_STARTED: newStateName = "STARTED"; break;
case STATE_LIMIT_REACHED: newStateName = "LIMIT_REACHED"; break;
case STATE_STOPPING: newStateName = "STOPPING"; break;
case STATE_STOPPED: newStateName = "STOPPED"; break;
}
LOG.w(mName, "setState:", newStateName);
mState = newState;
}
/**
* This encoder was attached to the engine. Keep the controller
* and run the internal thread.
*
* NOTE: it's important to call {@link WorkerHandler#post(Runnable)} instead of run()!
* The internal actions can cause a stop/release, and due to how {@link WorkerHandler#run(Runnable)}
* works, we might have {@link #onStop()} or {@link #onRelease()} to be executed before
* The internal actions can cause a stop, and due to how {@link WorkerHandler#run(Runnable)}
* works, we might have {@link #onStop()} or {@link #onStopped()} to be executed before
* the previous step has completed.
*/
final void prepare(@NonNull final MediaEncoderEngine.Controller controller, final long maxLengthMillis) {
if (mState >= STATE_PREPARING) {
LOG.e("Wrong state while preparing. Aborting.", mState);
return;
}
setState(STATE_PREPARING);
mController = controller;
mBufferInfo = new MediaCodec.BufferInfo();
mMaxLengthMillis = maxLengthMillis;
mWorker = WorkerHandler.get(getName());
LOG.i(getName(), "Prepare was called. Posting.");
mWorker = WorkerHandler.get(mName);
LOG.i(mName, "Prepare was called. Posting.");
mWorker.post(new Runnable() {
@Override
public void run() {
LOG.i(getName(), "Prepare was called. Executing.");
LOG.i(mName, "Prepare was called. Executing.");
onPrepare(controller, maxLengthMillis);
setState(STATE_PREPARED);
}
});
}
@ -85,14 +162,22 @@ abstract class MediaEncoder {
* in case the encoder needs to wait for a certain event
* like a "frame available".
*
* The {@link #STATE_STARTED} state will be set when draining for the
* first time (not when onStart ends).
*
* NOTE: it's important to call {@link WorkerHandler#post(Runnable)} instead of run()!
*/
final void start() {
LOG.w(getName(), "Start was called. Posting.");
if (mState < STATE_PREPARED || mState >= STATE_STARTING) {
LOG.e("Wrong state while starting. Aborting.", mState);
return;
}
setState(STATE_STARTING);
LOG.w(mName, "Start was called. Posting.");
mWorker.post(new Runnable() {
@Override
public void run() {
LOG.w(getName(), "Start was called. Executing.");
LOG.w(mName, "Start was called. Executing.");
onStart();
}
});
@ -108,27 +193,36 @@ abstract class MediaEncoder {
* @param data object
*/
final void notify(final @NonNull String event, final @Nullable Object data) {
LOG.v(getName(), "Notify was called. Posting.");
LOG.v(mName, "Notify was called. Posting.");
mWorker.post(new Runnable() {
@Override
public void run() {
LOG.v(getName(), "Notify was called. Executing.");
LOG.v(mName, "Notify was called. Executing.");
onEvent(event, data);
}
});
}
/**
* Stop recording.
* Stop recording. This involves signaling the end of stream and draining
* all output left.
*
* The {@link #STATE_STOPPED} state will be set when draining for the
* last time (not when onStart ends).
*
* NOTE: it's important to call {@link WorkerHandler#post(Runnable)} instead of run()!
*/
final void stop() {
LOG.w(getName(), "Stop was called. Posting.");
if (mState >= STATE_LIMIT_REACHED) {
LOG.e("Wrong state while stopping. Aborting.", mState);
return;
}
setState(STATE_STOPPING);
LOG.w(mName, "Stop was called. Posting.");
mWorker.post(new Runnable() {
@Override
public void run() {
LOG.w(getName(), "Stop was called. Executing.");
LOG.w(mName, "Stop was called. Executing.");
onStop();
}
});
@ -165,7 +259,8 @@ abstract class MediaEncoder {
abstract void onEvent(@NonNull String event, @Nullable Object data);
/**
* Stop recording.
* Stop recording. This involves signaling the end of stream and draining
* all output left.
*/
@EncoderThread
abstract void onStop();
@ -173,27 +268,24 @@ abstract class MediaEncoder {
/**
* Called by {@link #drainOutput(boolean)} when we get an EOS signal (not necessarily in the
* parameters, might also be through an input buffer flag).
*
* This is a good moment to release all resources, although the muxer might still
* be alive (we wait for the other Encoder, see MediaEncoderEngine.Controller).
*/
private void release() {
LOG.w(getName(), "is being released. Notifying controller and releasing codecs.");
@CallSuper
protected void onStopped() {
LOG.w(mName, "is being released. Notifying controller and releasing codecs.");
// TODO should we notify after this method?
mController.notifyReleased(mTrackIndex);
mController.notifyStopped(mTrackIndex);
mMediaCodec.stop();
mMediaCodec.release();
mMediaCodec = null;
mOutputBufferPool.clear();
mOutputBufferPool = null;
mBuffers = null;
onRelease();
setState(STATE_STOPPED);
}
/**
* This is called when we are stopped.
* It is a good moment to release all resources, although the muxer
* might still be alive (we wait for the other Encoder, see Controller).
*/
abstract void onRelease();
/**
* Returns a new input buffer and index, waiting at most {@link #INPUT_TIMEOUT_US} if none is available.
* Callers should check the boolean result - true if the buffer was filled.
@ -234,7 +326,7 @@ abstract class MediaEncoder {
*/
@SuppressWarnings("WeakerAccess")
protected void encodeInputBuffer(InputBuffer buffer) {
LOG.v(getName(), "ENCODING - Buffer:", buffer.index, "Bytes:", buffer.length, "Presentation:", buffer.timestamp);
LOG.v(mName, "ENCODING - Buffer:", buffer.index, "Bytes:", buffer.length, "Presentation:", buffer.timestamp);
if (buffer.isEndOfStream) { // send EOS
mMediaCodec.queueInputBuffer(buffer.index, 0, 0,
buffer.timestamp, MediaCodec.BUFFER_FLAG_END_OF_STREAM);
@ -244,16 +336,6 @@ abstract class MediaEncoder {
}
}
/**
* Signals the end of input stream. This is a Video only API, as in the normal case,
* we use input buffers to signal the end. In the video case, we don't have input buffers
* because we use an input surface instead.
*/
@SuppressWarnings("WeakerAccess")
protected void signalEndOfInputStream() {
mMediaCodec.signalEndOfInputStream();
}
/**
* Extracts all pending data that was written and encoded into {@link #mMediaCodec},
* and forwards it to the muxer.
@ -267,7 +349,7 @@ abstract class MediaEncoder {
@SuppressLint("LogNotTimber")
@SuppressWarnings("WeakerAccess")
protected void drainOutput(boolean drainAll) {
LOG.v(getName(), "DRAINING - EOS:", drainAll);
LOG.v(mName, "DRAINING - EOS:", drainAll);
if (mMediaCodec == null) {
LOG.e("drain() was called before prepare() or after releasing.");
return;
@ -289,7 +371,8 @@ abstract class MediaEncoder {
// should happen before receiving buffers, and should only happen once
if (mController.isStarted()) throw new RuntimeException("MediaFormat changed twice.");
MediaFormat newFormat = mMediaCodec.getOutputFormat();
mTrackIndex = mController.requestStart(newFormat);
mTrackIndex = mController.notifyStarted(newFormat);
setState(STATE_STARTED);
mOutputBufferPool = new OutputBufferPool(mTrackIndex);
} else if (encoderStatus < 0) {
LOG.e("Unexpected result from dequeueOutputBuffer: " + encoderStatus);
@ -316,7 +399,7 @@ abstract class MediaEncoder {
// and should be used for offsets only.
// TODO find a better way, this causes sync issues. (+ note: this sends pts=0 at first)
// mBufferInfo.presentationTimeUs = mLastPresentationTimeUs - mStartPresentationTimeUs;
LOG.v(getName(), "DRAINING - About to write(). Presentation:", mBufferInfo.presentationTimeUs);
LOG.v(mName, "DRAINING - About to write(). Presentation:", mBufferInfo.presentationTimeUs);
// TODO fix the mBufferInfo being the same, then implement delayed writing in Controller
// and remove the isStarted() check here.
@ -335,19 +418,20 @@ abstract class MediaEncoder {
&& !mMaxLengthReached
&& mStartPresentationTimeUs != Long.MIN_VALUE
&& mLastPresentationTimeUs - mStartPresentationTimeUs > mMaxLengthMillis * 1000) {
LOG.w(getName(), "DRAINING - Reached maxLength! mLastPresentationTimeUs:", mLastPresentationTimeUs,
LOG.w(mName, "DRAINING - Reached maxLength! mLastPresentationTimeUs:", mLastPresentationTimeUs,
"mStartPresentationTimeUs:", mStartPresentationTimeUs,
"mMaxLengthUs:", mMaxLengthMillis * 1000);
mMaxLengthReached = true;
LOG.w(getName(), "DRAINING - Requesting a stop.");
LOG.w(mName, "DRAINING - Requesting a stop.");
setState(STATE_LIMIT_REACHED);
mController.requestStop(mTrackIndex);
break;
}
// Check for the EOS flag so we can release the encoder.
// Check for the EOS flag so we can call onStopped.
if ((mBufferInfo.flags & MediaCodec.BUFFER_FLAG_END_OF_STREAM) != 0) {
LOG.w(getName(), "DRAINING - Got EOS. Releasing the codec.");
release();
LOG.w(mName, "DRAINING - Got EOS. Releasing the codec.");
onStopped();
break;
}
}

@ -16,6 +16,35 @@ import java.util.ArrayList;
/**
* The entry point for encoding video files.
*
* The external API is simple but the internal mechanism is not easy. Basically the engine
* controls a {@link MediaEncoder} instance for each track (e.g. one for video, one for audio).
*
* 1. We prepare the MediaEncoders: {@link MediaEncoder#prepare(Controller, long)}
* MediaEncoders can be prepared synchronously or not.
*
* 2. Someone calls {@link #start()} from any thread.
* As a consequence, we start the MediaEncoders: {@link MediaEncoder#start()}.
*
* 3. MediaEncoders do not start synchronously. Instead, they call
* {@link Controller#notifyStarted(MediaFormat)} when they have a legit format,
* and we keep track of who has started.
*
* 4. When all MediaEncoders have started, we actually start the muxer.
*
* 5. Someone calls {@link #stop()} from any thread.
* As a consequence, we stop the MediaEncoders: {@link MediaEncoder#stop()}.
*
* 6. MediaEncoders do not stop synchronously. Instead, they will stop reading but
* keep draining the codec until there's no data left. At that point, they can
* call {@link Controller#notifyStopped(int)}.
*
* 7. When all MediaEncoders have been released, we actually stop the muxer and notify.
*
* There is another possibility where MediaEncoders themselves want to stop, for example
* because they reach some limit or constraint (e.g. max duration). For this, they should
* call {@link Controller#requestStop(int)}. Once all MediaEncoders have stopped, we will
* actually call {@link #stop()} on ourselves.
*/
@RequiresApi(api = Build.VERSION_CODES.JELLY_BEAN_MR2)
public class MediaEncoderEngine {
@ -148,7 +177,7 @@ public class MediaEncoderEngine {
/**
* Asks encoders to stop. This is not sync, of course we will ask for encoders
* to call {@link Controller#notifyReleased(int)} before actually stop the muxer.
* to call {@link Controller#notifyStopped(int)} before actually stop the muxer.
* When all encoders request a release, {@link #end()} is called to do cleanup
* and notify the listener.
*/
@ -160,7 +189,7 @@ public class MediaEncoderEngine {
}
/**
* Called after all encoders have requested a release using {@link Controller#notifyReleased(int)}.
* Called after all encoders have requested a release using {@link Controller#notifyStopped(int)}.
* At this point we will do cleanup and notify the listener.
*/
private void end() {
@ -225,15 +254,15 @@ public class MediaEncoderEngine {
* @param format the media format
* @return the encoder track index
*/
int requestStart(@NonNull MediaFormat format) {
int notifyStarted(@NonNull MediaFormat format) {
synchronized (mControllerLock) {
if (mMediaMuxerStarted) {
throw new IllegalStateException("Trying to start but muxer started already");
}
int track = mMediaMuxer.addTrack(format);
LOG.w("requestStart:", "Assigned track", track, "to format", format.getString(MediaFormat.KEY_MIME));
LOG.w("notifyStarted:", "Assigned track", track, "to format", format.getString(MediaFormat.KEY_MIME));
if (++mStartedEncodersCount == mEncoders.size()) {
LOG.w("requestStart:", "All encoders have started. Starting muxer and dispatching onEncodingStart().");
LOG.w("notifyStarted:", "All encoders have started. Starting muxer and dispatching onEncodingStart().");
mMediaMuxer.start();
mMediaMuxerStarted = true;
if (mListener != null) {
@ -274,7 +303,7 @@ public class MediaEncoderEngine {
/**
* Requests that the engine stops. This is not executed until all encoders call
* this method, so it is a kind of soft request, just like {@link #requestStart(MediaFormat)}.
* this method, so it is a kind of soft request, just like {@link #notifyStarted(MediaFormat)}.
* To be used when maxLength / maxSize constraints are reached, for example.
*
* When this succeeds, {@link MediaEncoder#stop()} is called.
@ -294,9 +323,9 @@ public class MediaEncoderEngine {
* Notifies that the encoder was stopped. After this is called by all encoders,
* we will actually stop the muxer.
*/
void notifyReleased(int track) {
void notifyStopped(int track) {
synchronized (mControllerLock) {
LOG.w("notifyReleased:", "Called for track", track);
LOG.w("notifyStopped:", "Called for track", track);
if (++mReleasedEncodersCount == mEncoders.size()) {
LOG.w("requestStop:", "All encoders have been released. Stopping the muxer.");
end();

@ -100,13 +100,6 @@ public class TextureMediaEncoder extends VideoMediaEncoder<TextureMediaEncoder.C
mViewport = new EglViewport();
}
@EncoderThread
@Override
void onStart() {
super.onStart();
// Nothing to do here. Waiting for the first frame.
}
@EncoderThread
@Override
void onEvent(@NonNull String event, @Nullable Object data) {
@ -163,7 +156,8 @@ public class TextureMediaEncoder extends VideoMediaEncoder<TextureMediaEncoder.C
}
@Override
void onRelease() {
protected void onStopped() {
super.onStopped();
mFramePool.clear();
if (mWindow != null) {
mWindow.release();

@ -14,8 +14,17 @@ import com.otaliastudios.cameraview.CameraLogger;
import java.io.IOException;
/**
* This alone does nothing.
* Subclasses must make sure they write each frame onto the given Surface {@link #mSurface}.
* Base class for video encoding.
*
* This uses {@link MediaCodec#createInputSurface()} to create an input {@link Surface}
* into which we can write and that MediaCodec itself can read.
*
* This makes everything easier with respect to the process explained in {@link MediaEncoder}
* docs. We can skip the whole input part of acquiring an InputBuffer, filling it with data
* and returning it to the encoder with {@link #encodeInputBuffer(InputBuffer)}.
*
* All of this is automatically done by MediaCodec as long as we keep writing data into the
* given {@link Surface}. This class alone does not do this - subclasses are required to do so.
*
* @param <C> the config object.
*/
@ -53,15 +62,10 @@ abstract class VideoMediaEncoder<C extends VideoMediaEncoder.Config> extends Med
}
VideoMediaEncoder(@NonNull C config) {
super("VideoEncoder");
mConfig = config;
}
@NonNull
@Override
String getName() {
return "VideoEncoder";
}
@EncoderThread
@Override
void onPrepare(@NonNull MediaEncoderEngine.Controller controller, long maxLengthMillis) {
@ -99,7 +103,10 @@ abstract class VideoMediaEncoder<C extends VideoMediaEncoder.Config> extends Med
void onStop() {
LOG.i("onStop", "setting mFrameNum to 1 and signaling the end of input stream.");
mFrameNum = -1;
signalEndOfInputStream();
// Signals the end of input stream. This is a Video only API, as in the normal case,
// we use input buffers to signal the end. In the video case, we don't have input buffers
// because we use an input surface instead.
mMediaCodec.signalEndOfInputStream();
drainOutput(true);
}

Loading…
Cancel
Save