diff --git a/cameraview/src/main/gles/com/otaliastudios/cameraview/AudioMediaEncoder.java b/cameraview/src/main/gles/com/otaliastudios/cameraview/AudioMediaEncoder.java index 58cae799..32dce9c8 100644 --- a/cameraview/src/main/gles/com/otaliastudios/cameraview/AudioMediaEncoder.java +++ b/cameraview/src/main/gles/com/otaliastudios/cameraview/AudioMediaEncoder.java @@ -1,34 +1,64 @@ package com.otaliastudios.cameraview; +import android.annotation.SuppressLint; import android.media.AudioFormat; import android.media.AudioRecord; +import android.media.AudioTimestamp; import android.media.MediaCodec; import android.media.MediaCodecInfo; import android.media.MediaFormat; import android.media.MediaRecorder; import android.os.Build; +import android.os.Handler; +import android.os.Message; +import android.util.Log; + import androidx.annotation.NonNull; import androidx.annotation.Nullable; import androidx.annotation.RequiresApi; import java.io.IOException; import java.nio.ByteBuffer; +import java.util.concurrent.LinkedBlockingQueue; +// TODO create onVideoRecordingStart/onVideoRecordingEnd callbacks @RequiresApi(api = Build.VERSION_CODES.JELLY_BEAN_MR2) class AudioMediaEncoder extends MediaEncoder { + private static final String TAG = AudioMediaEncoder.class.getSimpleName(); + private static final CameraLogger LOG = CameraLogger.create(TAG); + private static final String MIME_TYPE = "audio/mp4a-latm"; - private static final int SAMPLE_RATE = 44100; // 44.1[KHz] is only setting guaranteed to be available on all devices. - public static final int SAMPLES_PER_FRAME = 1024; // AAC, bytes/frame/channel - public static final int FRAMES_PER_BUFFER = 25; // AAC, frame/buffer/sec + private static final int ENCODING = AudioFormat.ENCODING_PCM_16BIT; // Determines the SAMPLE_SIZE + private static final int CHANNELS = AudioFormat.CHANNEL_IN_MONO; // AudioFormat.CHANNEL_IN_STEREO; + + // The 44.1KHz frequency is the only setting guaranteed to be available on all devices. + private static final int SAMPLING_FREQUENCY = 44100; // samples/sec + private static final int CHANNELS_COUNT = 1; // 2; + + private static final int SAMPLE_SIZE = 2; // byte/sample/channel + private static final int BYTE_RATE_PER_CHANNEL = SAMPLING_FREQUENCY * SAMPLE_SIZE; // byte/sec/channel + private static final int BYTE_RATE = BYTE_RATE_PER_CHANNEL * CHANNELS_COUNT; // byte/sec + + static final int BIT_RATE = BYTE_RATE * 8; // bit/sec + + // We call FRAME here the chunk of data that we want to read at each loop cycle + private static final int FRAME_SIZE_PER_CHANNEL = 1024; // bytes/frame/channel [AAC constant] + private static final int FRAME_SIZE = FRAME_SIZE_PER_CHANNEL * CHANNELS_COUNT; // bytes/frame + + // We allocate buffers of 1KB each, which is not so much. I would say that allocating + // at most 200 of them is a reasonable value. With the current setup, in device tests, + // we manage to use 50 at most. + private static final int BUFFER_POOL_MAX_SIZE = 200; - private final Object mLock = new Object(); private boolean mRequestStop = false; + private AudioEncodingHandler mEncoder; + private AudioRecordingThread mRecorder; + private ByteBufferPool mByteBufferPool; private Config mConfig; static class Config { int bitRate; - Config(int bitRate) { this.bitRate = bitRate; } @@ -38,15 +68,20 @@ class AudioMediaEncoder extends MediaEncoder { mConfig = config; } + @NonNull + @Override + String getName() { + return "AudioEncoder"; + } + @EncoderThread @Override - void prepare(@NonNull MediaEncoderEngine.Controller controller, long maxLengthMillis) { - super.prepare(controller, maxLengthMillis); - final MediaFormat audioFormat = MediaFormat.createAudioFormat(MIME_TYPE, SAMPLE_RATE, 1); + void onPrepare(@NonNull MediaEncoderEngine.Controller controller, long maxLengthMillis) { + final MediaFormat audioFormat = MediaFormat.createAudioFormat(MIME_TYPE, SAMPLING_FREQUENCY, CHANNELS_COUNT); audioFormat.setInteger(MediaFormat.KEY_AAC_PROFILE, MediaCodecInfo.CodecProfileLevel.AACObjectLC); - audioFormat.setInteger(MediaFormat.KEY_CHANNEL_MASK, AudioFormat.CHANNEL_IN_MONO); + audioFormat.setInteger(MediaFormat.KEY_CHANNEL_MASK, CHANNELS); audioFormat.setInteger(MediaFormat.KEY_BIT_RATE, mConfig.bitRate); - audioFormat.setInteger(MediaFormat.KEY_CHANNEL_COUNT, 1); + audioFormat.setInteger(MediaFormat.KEY_CHANNEL_COUNT, CHANNELS_COUNT); try { mMediaCodec = MediaCodec.createEncoderByType(MIME_TYPE); } catch (IOException e) { @@ -54,86 +89,228 @@ class AudioMediaEncoder extends MediaEncoder { } mMediaCodec.configure(audioFormat, null, null, MediaCodec.CONFIGURE_FLAG_ENCODE); mMediaCodec.start(); + mByteBufferPool = new ByteBufferPool(FRAME_SIZE, BUFFER_POOL_MAX_SIZE); + mEncoder = new AudioEncodingHandler(); + mRecorder = new AudioRecordingThread(); } @EncoderThread @Override - void start() { + void onStart() { mRequestStop = false; - new AudioThread().start(); + mRecorder.start(); } @EncoderThread @Override - void notify(@NonNull String event, @Nullable Object data) { } + void onEvent(@NonNull String event, @Nullable Object data) { } @EncoderThread @Override - void stop() { + void onStop() { mRequestStop = true; - synchronized (mLock) { - try { - mLock.wait(); - } catch (InterruptedException e) { - // do nothing - } - } } @Override - void release() { - super.release(); + void onRelease() { mRequestStop = false; + mEncoder = null; + mRecorder = null; + if (mByteBufferPool != null) { + mByteBufferPool.clear(); + mByteBufferPool = null; + } } - class AudioThread extends Thread { + @Override + int getEncodedBitRate() { + return mConfig.bitRate; + } + + class AudioRecordingThread extends Thread { private AudioRecord mAudioRecord; + private ByteBuffer mCurrentBuffer; + private int mReadBytes; + private long mLastTimeUs; - AudioThread() { - final int minBufferSize = AudioRecord.getMinBufferSize( - SAMPLE_RATE, AudioFormat.CHANNEL_IN_MONO, - AudioFormat.ENCODING_PCM_16BIT); - int bufferSize = SAMPLES_PER_FRAME * FRAMES_PER_BUFFER; - if (bufferSize < minBufferSize) { - bufferSize = ((minBufferSize / SAMPLES_PER_FRAME) + 1) * SAMPLES_PER_FRAME * 2; + AudioRecordingThread() { + final int minBufferSize = AudioRecord.getMinBufferSize(SAMPLING_FREQUENCY, CHANNELS, ENCODING); + int bufferSize = FRAME_SIZE * 25; // Make this bigger so we don't skip frames. + while (bufferSize < minBufferSize) { + bufferSize += FRAME_SIZE; // Unlikely I think. } - mAudioRecord = new AudioRecord(MediaRecorder.AudioSource.CAMCORDER, SAMPLE_RATE, - AudioFormat.CHANNEL_IN_MONO, AudioFormat.ENCODING_PCM_16BIT, bufferSize); + mAudioRecord = new AudioRecord(MediaRecorder.AudioSource.CAMCORDER, + SAMPLING_FREQUENCY, CHANNELS, ENCODING, bufferSize); + setPriority(Thread.MAX_PRIORITY); } @Override public void run() { - super.run(); + mLastTimeUs = System.nanoTime() / 1000L; mAudioRecord.startRecording(); - final ByteBuffer buffer = ByteBuffer.allocateDirect(SAMPLES_PER_FRAME); - int readBytes; while (!mRequestStop) { - buffer.clear(); - readBytes = mAudioRecord.read(buffer, SAMPLES_PER_FRAME); - if (readBytes > 0) { - // set audio data to encoder - buffer.position(readBytes); - buffer.flip(); - encode(buffer, readBytes, getPresentationTime()); - drain(false); - } + read(false); } - // This will signal the endOfStream. + LOG.w("RECORDER: Stop was requested. We're out of the loop. Will post an endOfStream."); + // Last input with 0 length. This will signal the endOfStream. // Can't use drain(true); it is only available when writing to the codec InputSurface. - encode(null, 0, getPresentationTime()); - drain(false); + read(true); mAudioRecord.stop(); mAudioRecord.release(); mAudioRecord = null; - synchronized (mLock) { - mLock.notify(); + } + + private void read(boolean endOfStream) { + mCurrentBuffer = mByteBufferPool.get(); + if (mCurrentBuffer == null) { + LOG.e("Skipping audio frame, encoding is too slow."); + // TODO should fix the next presentation time here. However this is + // extremely unlikely based on my tests. The mByteBufferPool should be big enough. + } else { + mCurrentBuffer.clear(); + mReadBytes = mAudioRecord.read(mCurrentBuffer, FRAME_SIZE); + if (mReadBytes > 0) { // Good read: increase PTS. + increaseTime(mReadBytes); + mCurrentBuffer.limit(mReadBytes); + onBuffer(endOfStream); + } else if (mReadBytes == AudioRecord.ERROR_INVALID_OPERATION) { + LOG.e("Got AudioRecord.ERROR_INVALID_OPERATION"); + } else if (mReadBytes == AudioRecord.ERROR_BAD_VALUE) { + LOG.e("Got AudioRecord.ERROR_BAD_VALUE"); + } + } + } + + /** + * New data at position buffer.position() of size buffer.remaining() + * has been written into this buffer. This method should pass the data + * to the consumer. + */ + private void onBuffer(boolean endOfStream) { + mEncoder.sendInputBuffer(mCurrentBuffer, mLastTimeUs, endOfStream); + } + + private void increaseTime(int readBytes) { + increaseTime3(readBytes); + LOG.v("Read", readBytes, "bytes, increasing PTS to", mLastTimeUs); + } + + /** + * This method simply assumes that we read everything without losing a single US. + * It will use System.nanoTime() just once, as the starting point. + * Of course we don't as there are things going on in this thread. + */ + private void increaseTime1(int readBytes) { + mLastTimeUs += (1000000L * readBytes) / BYTE_RATE; + } + + /** + * Just for testing, this method will use Api 24 method to retrieve the timestamp. + * This way we let the platform choose instead of making assumptions. + */ + @RequiresApi(24) + private void increaseTime2(int readBytes) { + if (mApi24Timestamp == null) { + mApi24Timestamp = new AudioTimestamp(); } + mAudioRecord.getTimestamp(mApi24Timestamp, AudioTimestamp.TIMEBASE_MONOTONIC); + mLastTimeUs = mApi24Timestamp.nanoTime / 1000; } + private AudioTimestamp mApi24Timestamp; + + /** + * This method looks like an improvement over {@link #increaseTime1(int)} as it + * accounts for the current time as well. Adapted & improved. from Kickflip. + */ + private void increaseTime3(int readBytes) { + long currentTime = System.nanoTime() / 1000; + long correctedTime; + long bufferDuration = (1000000 * readBytes) / BYTE_RATE; + long bufferTime = currentTime - bufferDuration; // delay of acquiring the audio buffer + if (mTotalReadBytes == 0) { + mStartTimeUs = bufferTime; + } + // Recompute time assuming that we are respecting the sampling frequency. + // However, if the correction is too big (> 2*bufferDuration), reset to this point. + correctedTime = mStartTimeUs + (1000000 * mTotalReadBytes) / BYTE_RATE; + if(bufferTime - correctedTime >= 2 * bufferDuration) { + mStartTimeUs = bufferTime; + mTotalReadBytes = 0; + correctedTime = mStartTimeUs; + } + mTotalReadBytes += readBytes; + mLastTimeUs = correctedTime; + } + private long mStartTimeUs; + private long mTotalReadBytes; } - @Override - int getBitRate() { - return mConfig.bitRate; + /** + * This will be a super busy thread. It's important for it to be: + * - different than the recording thread: or we would miss a lot of audio + * - different than the 'encoder' thread: we want that to be reactive. + * For example, a stop() must become onStop() soon, can't wait for all this draining. + */ + @SuppressLint("HandlerLeak") + class AudioEncodingHandler extends Handler { + + InputBufferPool mInputBufferPool = new InputBufferPool(); + LinkedBlockingQueue mPendingOps = new LinkedBlockingQueue<>(); + + AudioEncodingHandler() { + super(WorkerHandler.get("AudioEncodingHandler").getLooper()); + } + + void sendInputBuffer(ByteBuffer buffer, long presentationTimeUs, boolean endOfStream) { + int presentation1 = (int) (presentationTimeUs >> 32); + int presentation2 = (int) (presentationTimeUs); + sendMessage(obtainMessage(endOfStream ? 1 : 0, presentation1, presentation2, buffer)); + } + + @Override + public void handleMessage(Message msg) { + super.handleMessage(msg); + boolean endOfStream = msg.what == 1; + long timestamp = (((long) msg.arg1) << 32) | (((long) msg.arg2) & 0xffffffffL); + ByteBuffer buffer = (ByteBuffer) msg.obj; + int readBytes = buffer.remaining(); + InputBuffer inputBuffer = mInputBufferPool.get(); + inputBuffer.source = buffer; + inputBuffer.timestamp = timestamp; + inputBuffer.length = readBytes; + inputBuffer.isEndOfStream = endOfStream; + mPendingOps.add(inputBuffer); + performPendingOps(endOfStream); + } + + private void performPendingOps(boolean force) { + LOG.v("Performing", mPendingOps.size(), "Pending operations."); + InputBuffer buffer; + while ((buffer = mPendingOps.peek()) != null) { + if (force) { + acquireInputBuffer(buffer); + performPendingOp(buffer); + } else if (tryAcquireInputBuffer(buffer)) { + performPendingOp(buffer); + } else { + break; // Will try later. + } + } + } + + private void performPendingOp(InputBuffer buffer) { + buffer.data.put(buffer.source); + mByteBufferPool.recycle(buffer.source); + mPendingOps.remove(buffer); + encodeInputBuffer(buffer); + boolean eos = buffer.isEndOfStream; + mInputBufferPool.recycle(buffer); + drainOutput(eos); + if (eos) { + mInputBufferPool.clear(); + WorkerHandler.get("AudioEncodingHandler").getThread().interrupt(); + } + } } } diff --git a/cameraview/src/main/gles/com/otaliastudios/cameraview/ByteBufferPool.java b/cameraview/src/main/gles/com/otaliastudios/cameraview/ByteBufferPool.java new file mode 100644 index 00000000..8e2a2aac --- /dev/null +++ b/cameraview/src/main/gles/com/otaliastudios/cameraview/ByteBufferPool.java @@ -0,0 +1,15 @@ +package com.otaliastudios.cameraview; + +import java.nio.ByteBuffer; + +class ByteBufferPool extends Pool { + + ByteBufferPool(final int bufferSize, int maxPoolSize) { + super(maxPoolSize, new Factory() { + @Override + public ByteBuffer create() { + return ByteBuffer.allocateDirect(bufferSize); + } + }); + } +} diff --git a/cameraview/src/main/gles/com/otaliastudios/cameraview/EglBaseSurface.java b/cameraview/src/main/gles/com/otaliastudios/cameraview/EglBaseSurface.java index ea842618..4e3393e2 100644 --- a/cameraview/src/main/gles/com/otaliastudios/cameraview/EglBaseSurface.java +++ b/cameraview/src/main/gles/com/otaliastudios/cameraview/EglBaseSurface.java @@ -151,6 +151,7 @@ class EglBaseSurface extends EglElement { /** * Sends the presentation time stamp to EGL. + * https://www.khronos.org/registry/EGL/extensions/ANDROID/EGL_ANDROID_presentation_time.txt * * @param nsecs Timestamp, in nanoseconds. */ diff --git a/cameraview/src/main/gles/com/otaliastudios/cameraview/EglCore.java b/cameraview/src/main/gles/com/otaliastudios/cameraview/EglCore.java index e0691b19..2483978d 100644 --- a/cameraview/src/main/gles/com/otaliastudios/cameraview/EglCore.java +++ b/cameraview/src/main/gles/com/otaliastudios/cameraview/EglCore.java @@ -314,6 +314,7 @@ final class EglCore { /** * Sends the presentation time stamp to EGL. Time is expressed in nanoseconds. + * https://www.khronos.org/registry/EGL/extensions/ANDROID/EGL_ANDROID_presentation_time.txt */ public void setPresentationTime(EGLSurface eglSurface, long nsecs) { EGLExt.eglPresentationTimeANDROID(mEGLDisplay, eglSurface, nsecs); diff --git a/cameraview/src/main/gles/com/otaliastudios/cameraview/InputBuffer.java b/cameraview/src/main/gles/com/otaliastudios/cameraview/InputBuffer.java new file mode 100644 index 00000000..9cc1369d --- /dev/null +++ b/cameraview/src/main/gles/com/otaliastudios/cameraview/InputBuffer.java @@ -0,0 +1,12 @@ +package com.otaliastudios.cameraview; + +import java.nio.ByteBuffer; + +class InputBuffer { + ByteBuffer data; + ByteBuffer source; + int index; + int length; + long timestamp; + boolean isEndOfStream; +} diff --git a/cameraview/src/main/gles/com/otaliastudios/cameraview/InputBufferPool.java b/cameraview/src/main/gles/com/otaliastudios/cameraview/InputBufferPool.java new file mode 100644 index 00000000..c99d0780 --- /dev/null +++ b/cameraview/src/main/gles/com/otaliastudios/cameraview/InputBufferPool.java @@ -0,0 +1,15 @@ +package com.otaliastudios.cameraview; + +import java.nio.ByteBuffer; + +class InputBufferPool extends Pool { + + InputBufferPool() { + super(Integer.MAX_VALUE, new Factory() { + @Override + public InputBuffer create() { + return new InputBuffer(); + } + }); + } +} diff --git a/cameraview/src/main/gles/com/otaliastudios/cameraview/MediaCodecBuffers.java b/cameraview/src/main/gles/com/otaliastudios/cameraview/MediaCodecBuffers.java new file mode 100644 index 00000000..4f134b08 --- /dev/null +++ b/cameraview/src/main/gles/com/otaliastudios/cameraview/MediaCodecBuffers.java @@ -0,0 +1,50 @@ +package com.otaliastudios.cameraview; + +import android.media.MediaCodec; +import android.os.Build; + +import java.nio.ByteBuffer; + +/** + * A Wrapper to MediaCodec that facilitates the use of API-dependent get{Input/Output}Buffer methods, + * in order to prevent: http://stackoverflow.com/q/30646885 + */ +class MediaCodecBuffers { + + private final MediaCodec mMediaCodec; + private final ByteBuffer[] mInputBuffers; + private ByteBuffer[] mOutputBuffers; + + MediaCodecBuffers(MediaCodec mediaCodec) { + mMediaCodec = mediaCodec; + + if (Build.VERSION.SDK_INT < 21) { + mInputBuffers = mediaCodec.getInputBuffers(); + mOutputBuffers = mediaCodec.getOutputBuffers(); + } else { + mInputBuffers = mOutputBuffers = null; + } + } + + public ByteBuffer getInputBuffer(final int index) { + if (Build.VERSION.SDK_INT >= 21) { + return mMediaCodec.getInputBuffer(index); + } + ByteBuffer buffer = mInputBuffers[index]; + buffer.clear(); + return buffer; + } + + public ByteBuffer getOutputBuffer(final int index) { + if (Build.VERSION.SDK_INT >= 21) { + return mMediaCodec.getOutputBuffer(index); + } + return mOutputBuffers[index]; + } + + public void onOutputBuffersChanged() { + if (Build.VERSION.SDK_INT < 21) { + mOutputBuffers = mMediaCodec.getOutputBuffers(); + } + } +} diff --git a/cameraview/src/main/gles/com/otaliastudios/cameraview/MediaEncoder.java b/cameraview/src/main/gles/com/otaliastudios/cameraview/MediaEncoder.java index b1b0e8bc..e82e8c1e 100644 --- a/cameraview/src/main/gles/com/otaliastudios/cameraview/MediaEncoder.java +++ b/cameraview/src/main/gles/com/otaliastudios/cameraview/MediaEncoder.java @@ -1,8 +1,10 @@ package com.otaliastudios.cameraview; +import android.annotation.SuppressLint; import android.media.MediaCodec; import android.media.MediaFormat; import android.os.Build; + import androidx.annotation.NonNull; import androidx.annotation.Nullable; import androidx.annotation.RequiresApi; @@ -14,17 +16,107 @@ import java.nio.ByteBuffer; @RequiresApi(api = Build.VERSION_CODES.JELLY_BEAN_MR2) abstract class MediaEncoder { - private final static int TIMEOUT_USEC = 10000; // 10 msec + private final static String TAG = MediaEncoder.class.getSimpleName(); + private final static CameraLogger LOG = CameraLogger.create(TAG); + + // Did some test to see which value would maximize our performance in the current setup (infinite audio pool). + // Measured the time it would take to write a 30 seconds video. Based on this, we'll go with TIMEOUT=0 for now. + // INPUT_TIMEOUT_US 10000: 46 seconds + // INPUT_TIMEOUT_US 1000: 37 seconds + // INPUT_TIMEOUT_US 100: 33 seconds + // INPUT_TIMEOUT_US 0: 32 seconds + private final static int INPUT_TIMEOUT_US = 0; + + // 0 also seems to be the best, although it does not change so much. + // Can't go too high or this is a bottleneck for the audio encoder. + private final static int OUTPUT_TIMEOUT_US = 0; @SuppressWarnings("WeakerAccess") protected MediaCodec mMediaCodec; - private MediaCodec.BufferInfo mBufferInfo; + @SuppressWarnings("WeakerAccess") + protected WorkerHandler mWorker; + private MediaEncoderEngine.Controller mController; private int mTrackIndex; + private OutputBufferPool mOutputBufferPool; + private MediaCodec.BufferInfo mBufferInfo; + private MediaCodecBuffers mBuffers; private long mMaxLengthMillis; private boolean mMaxLengthReached; + /** + * A readable name for the thread. + */ + @NonNull + abstract String getName(); + + /** + * This encoder was attached to the engine. Keep the controller + * and run the internal thread. + */ + final void prepare(@NonNull final MediaEncoderEngine.Controller controller, final long maxLengthMillis) { + mController = controller; + mBufferInfo = new MediaCodec.BufferInfo(); + mMaxLengthMillis = maxLengthMillis; + mWorker = WorkerHandler.get(getName()); + LOG.i(getName(), "Prepare was called. Posting."); + mWorker.post(new Runnable() { + @Override + public void run() { + LOG.i(getName(), "Prepare was called. Executing."); + onPrepare(controller, maxLengthMillis); + } + }); + } + + /** + * Start recording. This might be a lightweight operation + * in case the encoder needs to wait for a certain event + * like a "frame available". + */ + final void start() { + LOG.i(getName(), "Start was called. Posting."); + mWorker.post(new Runnable() { + @Override + public void run() { + LOG.i(getName(), "Start was called. Executing."); + onStart(); + } + }); + } + + /** + * The caller notifying of a certain event occurring. + * Should analyze the string and see if the event is important. + * @param event what happened + * @param data object + */ + final void notify(final @NonNull String event, final @Nullable Object data) { + LOG.i(getName(), "Notify was called. Posting."); + mWorker.post(new Runnable() { + @Override + public void run() { + LOG.i(getName(), "Notify was called. Executing."); + onEvent(event, data); + } + }); + } + + /** + * Stop recording. + */ + final void stop() { + LOG.i(getName(), "Stop was called. Posting."); + mWorker.post(new Runnable() { + @Override + public void run() { + LOG.i(getName(), "Stop was called. Executing."); + onStop(); + } + }); + } + /** * Called to prepare this encoder before starting. * Any initialization should be done here as it does not interfere with the original @@ -33,13 +125,10 @@ abstract class MediaEncoder { * At this point subclasses MUST create the {@link #mMediaCodec} object. * * @param controller the muxer controller + * @param maxLengthMillis the maxLength in millis */ @EncoderThread - void prepare(@NonNull MediaEncoderEngine.Controller controller, long maxLengthMillis) { - mController = controller; - mBufferInfo = new MediaCodec.BufferInfo(); - mMaxLengthMillis = maxLengthMillis; - } + abstract void onPrepare(@NonNull final MediaEncoderEngine.Controller controller, final long maxLengthMillis); /** * Start recording. This might be a lightweight operation @@ -47,7 +136,7 @@ abstract class MediaEncoder { * like a "frame available". */ @EncoderThread - abstract void start(); + abstract void onStart(); /** * The caller notifying of a certain event occurring. @@ -56,97 +145,130 @@ abstract class MediaEncoder { * @param data object */ @EncoderThread - abstract void notify(@NonNull String event, @Nullable Object data); + abstract void onEvent(@NonNull String event, @Nullable Object data); /** * Stop recording. - * This MUST happen SYNCHRONOUSLY! */ @EncoderThread - abstract void stop(); + abstract void onStop(); /** - * Release resources here. + * Called by {@link #drainOutput(boolean)} when we get an EOS signal (not necessarily in the + * parameters, might also be through an input buffer flag). */ - @EncoderThread - void release() { - if (mMediaCodec != null) { - mMediaCodec.stop(); - mMediaCodec.release(); - mMediaCodec = null; + private void release() { + LOG.w("Subclass", getName(), "Notified that it is released."); + mController.requestRelease(mTrackIndex); + mMediaCodec.stop(); + mMediaCodec.release(); + mMediaCodec = null; + mOutputBufferPool.clear(); + mOutputBufferPool = null; + mBuffers = null; + onRelease(); + } + + /** + * This is called when we are stopped. + * It is a good moment to release all resources, although the muxer + * might still be alive (we wait for the other Encoder, see Controller). + */ + abstract void onRelease(); + + /** + * Returns a new input buffer and index, waiting at most {@link #INPUT_TIMEOUT_US} if none is available. + * Callers should check the boolean result - true if the buffer was filled. + */ + @SuppressWarnings("WeakerAccess") + protected boolean tryAcquireInputBuffer(@NonNull InputBuffer holder) { + if (mBuffers == null) { + mBuffers = new MediaCodecBuffers(mMediaCodec); + } + int inputBufferIndex = mMediaCodec.dequeueInputBuffer(INPUT_TIMEOUT_US); + if (inputBufferIndex < 0) { + return false; + } else { + holder.index = inputBufferIndex; + holder.data = mBuffers.getInputBuffer(inputBufferIndex); + return true; } } + /** + * Returns a new input buffer and index, waiting indefinitely if none is available. + * The buffer should be written into, then the index should be passed to {@link #encodeInputBuffer(InputBuffer)}. + */ + @SuppressWarnings({"StatementWithEmptyBody", "WeakerAccess"}) + protected void acquireInputBuffer(@NonNull InputBuffer holder) { + while (!tryAcquireInputBuffer(holder)) {} + } + /** * Encode data into the {@link #mMediaCodec}. */ @SuppressWarnings("WeakerAccess") - protected void encode(@Nullable final ByteBuffer buffer, final int length, final long presentationTimeUs) { - final ByteBuffer[] inputBuffers = mMediaCodec.getInputBuffers(); - while (true) { - final int inputBufferIndex = mMediaCodec.dequeueInputBuffer(TIMEOUT_USEC); - if (inputBufferIndex >= 0) { - final ByteBuffer inputBuffer = inputBuffers[inputBufferIndex]; - inputBuffer.clear(); - if (buffer != null) { - inputBuffer.put(buffer); - } - if (length <= 0) { // send EOS - mMediaCodec.queueInputBuffer(inputBufferIndex, 0, 0, - presentationTimeUs, MediaCodec.BUFFER_FLAG_END_OF_STREAM); - } else { - mMediaCodec.queueInputBuffer(inputBufferIndex, 0, length, - presentationTimeUs, 0); - } - break; - } else if (inputBufferIndex == MediaCodec.INFO_TRY_AGAIN_LATER) { - // wait for MediaCodec encoder is ready to encode - // nothing to do here because MediaCodec#dequeueInputBuffer(TIMEOUT_USEC) - // will wait for maximum TIMEOUT_USEC(10msec) on each call - } + protected void encodeInputBuffer(InputBuffer buffer) { + LOG.w("ENCODING:", getName(), "Buffer:", buffer.index, "Bytes:", buffer.length, "Presentation:", buffer.timestamp); + if (buffer.isEndOfStream) { // send EOS + mMediaCodec.queueInputBuffer(buffer.index, 0, 0, + buffer.timestamp, MediaCodec.BUFFER_FLAG_END_OF_STREAM); + } else { + mMediaCodec.queueInputBuffer(buffer.index, 0, buffer.length, + buffer.timestamp, 0); } } + /** + * Signals the end of input stream. This is a Video only API, as in the normal case, + * we use input buffers to signal the end. In the video case, we don't have input buffers + * because we use an input surface instead. + */ + @SuppressWarnings("WeakerAccess") + protected void signalEndOfInputStream() { + mMediaCodec.signalEndOfInputStream(); + } + /** * Extracts all pending data that was written and encoded into {@link #mMediaCodec}, * and forwards it to the muxer. - *

- * If endOfStream is not set, this returns when there is no more data to drain. If it - * is set, we send EOS to the encoder, and then iterate until we see EOS on the output. - * Calling this with endOfStream set should be done once, right before stopping the muxer. + * + * If drainAll is not set, this returns after TIMEOUT_USEC if there is no more data to drain. + * If drainAll is set, we wait until we see EOS on the output. + * Calling this with drainAll set should be done once, right before stopping the muxer. */ + @SuppressLint("LogNotTimber") @SuppressWarnings("WeakerAccess") - protected void drain(boolean endOfStream) { - if (endOfStream) { - mMediaCodec.signalEndOfInputStream(); + protected void drainOutput(boolean drainAll) { + LOG.w("DRAINING:", getName(), "EOS:", drainAll); + if (mMediaCodec == null) { + LOG.e("drain() was called before prepare() or after releasing."); + return; + } + if (mBuffers == null) { + mBuffers = new MediaCodecBuffers(mMediaCodec); } - - ByteBuffer[] encoderOutputBuffers = mMediaCodec.getOutputBuffers(); while (true) { - int encoderStatus = mMediaCodec.dequeueOutputBuffer(mBufferInfo, TIMEOUT_USEC); + int encoderStatus = mMediaCodec.dequeueOutputBuffer(mBufferInfo, OUTPUT_TIMEOUT_US); if (encoderStatus == MediaCodec.INFO_TRY_AGAIN_LATER) { // no output available yet - if (!endOfStream) break; // out of while + if (!drainAll) break; // out of while } else if (encoderStatus == MediaCodec.INFO_OUTPUT_BUFFERS_CHANGED) { // not expected for an encoder - encoderOutputBuffers = mMediaCodec.getOutputBuffers(); + mBuffers.onOutputBuffersChanged(); } else if (encoderStatus == MediaCodec.INFO_OUTPUT_FORMAT_CHANGED) { // should happen before receiving buffers, and should only happen once - if (mController.isStarted()) throw new RuntimeException("format changed twice"); + if (mController.isStarted()) throw new RuntimeException("MediaFormat changed twice."); MediaFormat newFormat = mMediaCodec.getOutputFormat(); - - // now that we have the Magic Goodies, start the muxer - mTrackIndex = mController.start(newFormat); + mTrackIndex = mController.requestStart(newFormat); + mOutputBufferPool = new OutputBufferPool(mTrackIndex); } else if (encoderStatus < 0) { - Log.w("VideoMediaEncoder", "unexpected result from encoder.dequeueOutputBuffer: " + encoderStatus); + LOG.e("Unexpected result from dequeueOutputBuffer: " + encoderStatus); // let's ignore it } else { - ByteBuffer encodedData = encoderOutputBuffers[encoderStatus]; - if (encodedData == null) { - throw new RuntimeException("encoderOutputBuffer " + encoderStatus + " was null"); - } + ByteBuffer encodedData = mBuffers.getOutputBuffer(encoderStatus); // Codec config means that config data was pulled out and fed to the muxer when we got // the INFO_OUTPUT_FORMAT_CHANGED status. Ignore it. @@ -155,41 +277,56 @@ abstract class MediaEncoder { // adjust the ByteBuffer values to match BufferInfo (not needed?) encodedData.position(mBufferInfo.offset); encodedData.limit(mBufferInfo.offset + mBufferInfo.size); - mController.write(mTrackIndex, encodedData, mBufferInfo); - mLastPresentationTime = mBufferInfo.presentationTimeUs; - if (mStartPresentationTime == 0) { - mStartPresentationTime = mLastPresentationTime; + // Store startPresentationTime and lastPresentationTime, useful for example to + // detect the mMaxLengthReached and stop recording. + if (mStartPresentationTimeUs == Long.MIN_VALUE) { + mStartPresentationTimeUs = mBufferInfo.presentationTimeUs; } + mLastPresentationTimeUs = mBufferInfo.presentationTimeUs; + // Pass presentation times as offets with respect to the mStartPresentationTimeUs. + // This ensures consistency between audio pts (coming from System.nanoTime()) and + // video pts (coming from SurfaceTexture) both of which have no meaningful time-base + // and should be used for offsets only. + // TODO find a better way, this causes sync issues. (+ note: this sends pts=0 at first) + // mBufferInfo.presentationTimeUs = mLastPresentationTimeUs - mStartPresentationTimeUs; + LOG.i("DRAINING:", getName(), "Dispatching write(). Presentation:", mBufferInfo.presentationTimeUs); + + // TODO fix the mBufferInfo being the same, then implement delayed writing in Controller + // and remove the isStarted() check here. + OutputBuffer buffer = mOutputBufferPool.get(); + buffer.info = mBufferInfo; + buffer.trackIndex = mTrackIndex; + buffer.data = encodedData; + mController.write(mOutputBufferPool, buffer); } mMediaCodec.releaseOutputBuffer(encoderStatus, false); - if (!mMaxLengthReached) { - if (mLastPresentationTime / 1000 - mStartPresentationTime / 1000 > mMaxLengthMillis) { - mMaxLengthReached = true; - // Log.e("MediaEncoder", this.getClass().getSimpleName() + " requested stop at " + (mLastPresentationTime * 1000 * 1000)); - mController.requestStop(); - break; - } + + // Check for the maxLength constraint (with appropriate conditions) + // Not needed if drainAll because we already were asked to stop + if (!drainAll + && !mMaxLengthReached + && mStartPresentationTimeUs != Long.MIN_VALUE + && mLastPresentationTimeUs - mStartPresentationTimeUs > mMaxLengthMillis * 1000) { + LOG.w("DRAINING: Reached maxLength! mLastPresentationTimeUs:", mLastPresentationTimeUs, + "mStartPresentationTimeUs:", mStartPresentationTimeUs, + "mMaxLengthUs:", mMaxLengthMillis * 1000); + mMaxLengthReached = true; + mController.requestStop(mTrackIndex); + break; } + // Check for the EOS flag so we can release the encoder. if ((mBufferInfo.flags & MediaCodec.BUFFER_FLAG_END_OF_STREAM) != 0) { - break; // out of while + LOG.w("DRAINING:", getName(), "Dispatching release()."); + release(); + break; } } } } - private long mStartPresentationTime = 0; - private long mLastPresentationTime = 0; - - long getPresentationTime() { - long result = System.nanoTime() / 1000L; - // presentationTimeUs should be monotonic - // otherwise muxer fail to write - if (result < mLastPresentationTime) { - result = (mLastPresentationTime - result) + result; - } - return result; - } + private long mStartPresentationTimeUs = Long.MIN_VALUE; + private long mLastPresentationTimeUs = 0; - abstract int getBitRate(); + abstract int getEncodedBitRate(); } diff --git a/cameraview/src/main/gles/com/otaliastudios/cameraview/MediaEncoderEngine.java b/cameraview/src/main/gles/com/otaliastudios/cameraview/MediaEncoderEngine.java index efb97e4c..0b9fa771 100644 --- a/cameraview/src/main/gles/com/otaliastudios/cameraview/MediaEncoderEngine.java +++ b/cameraview/src/main/gles/com/otaliastudios/cameraview/MediaEncoderEngine.java @@ -1,6 +1,5 @@ package com.otaliastudios.cameraview; -import android.media.MediaCodec; import android.media.MediaFormat; import android.media.MediaMuxer; import android.os.Build; @@ -10,13 +9,12 @@ import androidx.annotation.RequiresApi; import java.io.File; import java.io.IOException; -import java.nio.ByteBuffer; import java.util.ArrayList; @RequiresApi(api = Build.VERSION_CODES.JELLY_BEAN_MR2) class MediaEncoderEngine { - private final static String TAG = MediaEncoder.class.getSimpleName(); + private final static String TAG = MediaEncoderEngine.class.getSimpleName(); private final static CameraLogger LOG = CameraLogger.create(TAG); @SuppressWarnings("WeakerAccess") @@ -24,20 +22,19 @@ class MediaEncoderEngine { final static int STOP_BY_MAX_DURATION = 1; final static int STOP_BY_MAX_SIZE = 2; - private WorkerHandler mWorker; private ArrayList mEncoders; private MediaMuxer mMediaMuxer; - private int mMediaMuxerStartCount; + private int mStartedEncodersCount; + private int mStoppedEncodersCount; private boolean mMediaMuxerStarted; private Controller mController; private Listener mListener; private int mStopReason = STOP_BY_USER; private int mPossibleStopReason; - private final Object mLock = new Object(); + private final Object mControllerLock = new Object(); MediaEncoderEngine(@NonNull File file, @NonNull VideoMediaEncoder videoEncoder, @Nullable AudioMediaEncoder audioEncoder, final int maxDuration, final long maxSize, @Nullable Listener listener) { - mWorker = WorkerHandler.get("EncoderEngine"); mListener = listener; mController = new Controller(); mEncoders = new ArrayList<>(); @@ -50,51 +47,54 @@ class MediaEncoderEngine { } catch (IOException e) { throw new RuntimeException(e); } - mMediaMuxerStartCount = 0; + mStartedEncodersCount = 0; mMediaMuxerStarted = false; - mWorker.post(new Runnable() { - @Override - public void run() { - // Trying to convert the size constraints to duration constraints, - // because they are super easy to check. - // This is really naive & probably not accurate, but... - int bitRate = 0; - for (MediaEncoder encoder : mEncoders) { - bitRate += encoder.getBitRate(); - } - int bytePerSecond = bitRate / 8; - long sizeMaxDuration = (maxSize / bytePerSecond) * 1000L; - - long finalMaxDuration = Long.MAX_VALUE; - if (maxSize > 0 && maxDuration > 0) { - mPossibleStopReason = sizeMaxDuration < maxDuration ? STOP_BY_MAX_SIZE : STOP_BY_MAX_DURATION; - finalMaxDuration = Math.min(sizeMaxDuration, maxDuration); - } else if (maxSize > 0) { - mPossibleStopReason = STOP_BY_MAX_SIZE; - finalMaxDuration = sizeMaxDuration; - } else if (maxDuration > 0) { - mPossibleStopReason = STOP_BY_MAX_DURATION; - finalMaxDuration = maxDuration; - } - LOG.i("Computed a max duration of", (finalMaxDuration / 1000F)); - for (MediaEncoder encoder : mEncoders) { - encoder.prepare(mController, finalMaxDuration); - } - } - }); + mStoppedEncodersCount = 0; + + // Trying to convert the size constraints to duration constraints, + // because they are super easy to check. + // This is really naive & probably not accurate, but... + int bitRate = 0; + for (MediaEncoder encoder : mEncoders) { + bitRate += encoder.getEncodedBitRate(); + } + int bytePerSecond = bitRate / 8; + long sizeMaxDuration = (maxSize / bytePerSecond) * 1000L; + + long finalMaxDuration = Long.MAX_VALUE; + if (maxSize > 0 && maxDuration > 0) { + mPossibleStopReason = sizeMaxDuration < maxDuration ? STOP_BY_MAX_SIZE : STOP_BY_MAX_DURATION; + finalMaxDuration = Math.min(sizeMaxDuration, maxDuration); + } else if (maxSize > 0) { + mPossibleStopReason = STOP_BY_MAX_SIZE; + finalMaxDuration = sizeMaxDuration; + } else if (maxDuration > 0) { + mPossibleStopReason = STOP_BY_MAX_DURATION; + finalMaxDuration = maxDuration; + } + LOG.w("Computed a max duration of", (finalMaxDuration / 1000F)); + for (MediaEncoder encoder : mEncoders) { + encoder.prepare(mController, finalMaxDuration); + } } // Stuff here might be called from multiple threads. class Controller { - int start(MediaFormat format) { - synchronized (mLock) { + /** + * Request that the muxer should start. This is not guaranteed to be executed: + * we wait for all encoders to call this method, and only then, start the muxer. + * @param format the media format + * @return the encoder track index + */ + int requestStart(MediaFormat format) { + synchronized (mControllerLock) { if (mMediaMuxerStarted) { throw new IllegalStateException("Trying to start but muxer started already"); } int track = mMediaMuxer.addTrack(format); - mMediaMuxerStartCount++; - if (mMediaMuxerStartCount == mEncoders.size()) { + LOG.w("Controller:", "Assigned track", track, "to format", format.getString(MediaFormat.KEY_MIME)); + if (++mStartedEncodersCount == mEncoders.size()) { mMediaMuxer.start(); mMediaMuxerStarted = true; } @@ -102,83 +102,124 @@ class MediaEncoderEngine { } } + /** + * Whether the muxer is started. + * @return true if muxer was started + */ boolean isStarted() { - synchronized (mLock) { + synchronized (mControllerLock) { return mMediaMuxerStarted; } } - // Synchronization does not seem needed here. - void write(int track, ByteBuffer encodedData, MediaCodec.BufferInfo info) { + /** + * Writes the given data to the muxer. Should be called after {@link #isStarted()} + * returns true. Note: this seems to be thread safe, no lock. + * TODO cache values if not started yet, then apply later. Read comments in drain(). + * Currently they are recycled instantly. + */ + void write(OutputBufferPool pool, OutputBuffer buffer) { if (!mMediaMuxerStarted) { throw new IllegalStateException("Trying to write before muxer started"); } - mMediaMuxer.writeSampleData(track, encodedData, info); + // This is a bad idea and causes crashes. + // if (info.presentationTimeUs < mLastTimestampUs) info.presentationTimeUs = mLastTimestampUs; + // mLastTimestampUs = info.presentationTimeUs; + LOG.v("Writing for track", buffer.trackIndex, ". Presentation:", buffer.info.presentationTimeUs); + mMediaMuxer.writeSampleData(buffer.trackIndex, buffer.data, buffer.info); + pool.recycle(buffer); } - void requestStop() { - synchronized (mLock) { - mMediaMuxerStartCount--; - if (mMediaMuxerStartCount == 0) { + /** + * Requests that the engine stops. This is not executed until all encoders call + * this method, so it is a kind of soft request, just like {@link #requestStart(MediaFormat)}. + * To be used when maxLength / maxSize constraints are reached, for example. + * + * When this succeeds, {@link MediaEncoder#stop()} is called. + */ + void requestStop(int track) { + LOG.i("RequestStop was called for track", track); + synchronized (mControllerLock) { + if (--mStartedEncodersCount == 0) { mStopReason = mPossibleStopReason; stop(); } } } - } - void start() { - mWorker.post(new Runnable() { - @Override - public void run() { - for (MediaEncoder encoder : mEncoders) { - encoder.start(); + /** + * Notifies that the encoder was stopped. After this is called by all encoders, + * we will actually stop the muxer. + */ + void requestRelease(int track) { + LOG.i("requestRelease was called for track", track); + synchronized (mControllerLock) { + if (++mStoppedEncodersCount == mEncoders.size()) { + release(); } } - }); + } } - void notify(final String event, final Object data) { - mWorker.post(new Runnable() { - @Override - public void run() { - for (MediaEncoder encoder : mEncoders) { - encoder.notify(event, data); - } - } - }); + final void start() { + for (MediaEncoder encoder : mEncoders) { + encoder.start(); + } } - void stop() { - mWorker.post(new Runnable() { - @Override - public void run() { - for (MediaEncoder encoder : mEncoders) { - encoder.stop(); - } - for (MediaEncoder encoder : mEncoders) { - encoder.release(); - } - Exception error = null; - if (mMediaMuxer != null) { - // stop() throws an exception if you haven't fed it any data. - // But also in other occasions. So this is a signal that something - // went wrong, and we propagate that to the listener. - try { - mMediaMuxer.stop(); - mMediaMuxer.release(); - } catch (Exception e) { - error = e; - } - mMediaMuxer = null; - } - if (mListener != null) mListener.onEncoderStop(mStopReason, error); - mStopReason = STOP_BY_USER; - mListener = null; - mMediaMuxerStartCount = 0; - mMediaMuxerStarted = false; + @SuppressWarnings("SameParameterValue") + final void notify(final String event, final Object data) { + for (MediaEncoder encoder : mEncoders) { + encoder.notify(event, data); + } + } + + /** + * This just asks the encoder to stop. We will wait for them to call {@link Controller#requestRelease(int)} + * to actually stop the muxer, as there might be async stuff going on. + */ + final void stop() { + for (MediaEncoder encoder : mEncoders) { + encoder.stop(); + } + } + + private void release() { + Exception error = null; + if (mMediaMuxer != null) { + // stop() throws an exception if you haven't fed it any data. + // But also in other occasions. So this is a signal that something + // went wrong, and we propagate that to the listener. + try { + mMediaMuxer.stop(); + mMediaMuxer.release(); + } catch (Exception e) { + error = e; } - }); + mMediaMuxer = null; + } + if (mListener != null) { + mListener.onEncoderStop(mStopReason, error); + mListener = null; + } + mStopReason = STOP_BY_USER; + mStartedEncodersCount = 0; + mStoppedEncodersCount = 0; + mMediaMuxerStarted = false; + } + + @NonNull + VideoMediaEncoder getVideoEncoder() { + return (VideoMediaEncoder) mEncoders.get(0); + } + + @Nullable + AudioMediaEncoder getAudioEncoder() { + if (mEncoders.size() > 1) { + return (AudioMediaEncoder) mEncoders.get(1); + } else { + return null; + } } interface Listener { diff --git a/cameraview/src/main/gles/com/otaliastudios/cameraview/OutputBuffer.java b/cameraview/src/main/gles/com/otaliastudios/cameraview/OutputBuffer.java new file mode 100644 index 00000000..b3f72a97 --- /dev/null +++ b/cameraview/src/main/gles/com/otaliastudios/cameraview/OutputBuffer.java @@ -0,0 +1,11 @@ +package com.otaliastudios.cameraview; + +import android.media.MediaCodec; + +import java.nio.ByteBuffer; + +class OutputBuffer { + MediaCodec.BufferInfo info; + int trackIndex; + ByteBuffer data; +} diff --git a/cameraview/src/main/gles/com/otaliastudios/cameraview/OutputBufferPool.java b/cameraview/src/main/gles/com/otaliastudios/cameraview/OutputBufferPool.java new file mode 100644 index 00000000..9e77f862 --- /dev/null +++ b/cameraview/src/main/gles/com/otaliastudios/cameraview/OutputBufferPool.java @@ -0,0 +1,18 @@ +package com.otaliastudios.cameraview; + +import android.media.MediaCodec; + +class OutputBufferPool extends Pool { + + OutputBufferPool(final int trackIndex) { + super(Integer.MAX_VALUE, new Factory() { + @Override + public OutputBuffer create() { + OutputBuffer buffer = new OutputBuffer(); + buffer.trackIndex = trackIndex; + buffer.info = new MediaCodec.BufferInfo(); + return buffer; + } + }); + } +} diff --git a/cameraview/src/main/gles/com/otaliastudios/cameraview/Pool.java b/cameraview/src/main/gles/com/otaliastudios/cameraview/Pool.java new file mode 100644 index 00000000..4429d55c --- /dev/null +++ b/cameraview/src/main/gles/com/otaliastudios/cameraview/Pool.java @@ -0,0 +1,89 @@ +package com.otaliastudios.cameraview; + +import java.util.concurrent.LinkedBlockingQueue; + +import androidx.annotation.CallSuper; +import androidx.annotation.NonNull; +import androidx.annotation.Nullable; + +class Pool { + + private static final String TAG = Pool.class.getSimpleName(); + private static final CameraLogger LOG = CameraLogger.create(TAG); + + private int maxPoolSize; + private int activeCount; + private LinkedBlockingQueue mQueue; + private Factory factory; + + interface Factory { + T create(); + } + + Pool(int maxPoolSize, Factory factory) { + this.maxPoolSize = maxPoolSize; + this.mQueue = new LinkedBlockingQueue<>(maxPoolSize); + this.factory = factory; + } + + boolean canGet() { + return count() < maxPoolSize; + } + + @Nullable + T get() { + T buffer = mQueue.poll(); + if (buffer != null) { + activeCount++; // poll decreases, this fixes + LOG.v("GET: Reusing recycled item.", this); + return buffer; + } + + if (!canGet()) { + LOG.v("GET: Returning null. Too much items requested.", this); + return null; + } + + activeCount++; + LOG.v("GET: Creating a new item.", this); + return factory.create(); + } + + + void recycle(@NonNull T item) { + LOG.v("RECYCLE: Recycling item.", this); + if (--activeCount < 0) { + throw new IllegalStateException("Trying to recycle an item which makes activeCount < 0." + + "This means that this or some previous items being recycled were not coming from " + + "this pool, or some item was recycled more than once. " + this); + } + if (!mQueue.offer(item)) { + throw new IllegalStateException("Trying to recycle an item while the queue is full. " + + "This means that this or some previous items being recycled were not coming from " + + "this pool, or some item was recycled more than once. " + this); + } + } + + @NonNull + @Override + public String toString() { + return getClass().getSimpleName() + " -- count:" + count() + ", active:" + activeCount() + ", cached:" + cachedCount(); + } + + final int count() { + return activeCount() + cachedCount(); + } + + final int activeCount() { + return activeCount; + } + + final int cachedCount() { + return mQueue.size(); + } + + @CallSuper + void clear() { + mQueue.clear(); + } +} diff --git a/cameraview/src/main/gles/com/otaliastudios/cameraview/TextureMediaEncoder.java b/cameraview/src/main/gles/com/otaliastudios/cameraview/TextureMediaEncoder.java index 5e72a878..6c40b58a 100644 --- a/cameraview/src/main/gles/com/otaliastudios/cameraview/TextureMediaEncoder.java +++ b/cameraview/src/main/gles/com/otaliastudios/cameraview/TextureMediaEncoder.java @@ -3,6 +3,8 @@ package com.otaliastudios.cameraview; import android.opengl.EGLContext; import android.opengl.Matrix; import android.os.Build; +import android.widget.TextView; + import androidx.annotation.NonNull; import androidx.annotation.Nullable; import androidx.annotation.RequiresApi; @@ -15,10 +17,6 @@ class TextureMediaEncoder extends VideoMediaEncoder final static String FRAME_EVENT = "frame"; - static class Frame { - float[] transform; - long timestamp; - } static class Config extends VideoMediaEncoder.Config { int textureId; float scaleX; @@ -44,15 +42,40 @@ class TextureMediaEncoder extends VideoMediaEncoder private EglCore mEglCore; private EglWindowSurface mWindow; private EglViewport mViewport; + private Pool mFramePool = new Pool<>(100, new Pool.Factory() { + @Override + public TextureFrame create() { + return new TextureFrame(); + } + }); TextureMediaEncoder(@NonNull Config config) { super(config); } + static class TextureFrame { + private TextureFrame() {} + // Nanoseconds, in no meaningful time-base. Should be for offsets only. + // Typically coming from SurfaceTexture.getTimestamp(). + long timestamp; + float[] transform = new float[16]; + } + + @NonNull + TextureFrame acquireFrame() { + if (!mFramePool.canGet()) { + throw new RuntimeException("Need more frames than this! Please increase the pool size."); + } else { + //noinspection ConstantConditions + return mFramePool.get(); + } + } + + @EncoderThread @Override - void prepare(@NonNull MediaEncoderEngine.Controller controller, long maxLengthMillis) { - super.prepare(controller, maxLengthMillis); + void onPrepare(@NonNull MediaEncoderEngine.Controller controller, long maxLengthMillis) { + super.onPrepare(controller, maxLengthMillis); mEglCore = new EglCore(mConfig.eglContext, EglCore.FLAG_RECORDABLE); mWindow = new EglWindowSurface(mEglCore, mSurface, true); mWindow.makeCurrent(); // drawing will happen on the InputWindowSurface, which @@ -62,8 +85,57 @@ class TextureMediaEncoder extends VideoMediaEncoder @EncoderThread @Override - void release() { - super.release(); + void onStart() { + super.onStart(); + // Nothing to do here. Waiting for the first frame. + } + + @EncoderThread + @Override + void onEvent(@NonNull String event, @Nullable Object data) { + if (!event.equals(FRAME_EVENT)) return; + TextureFrame frame = (TextureFrame) data; + if (frame == null) return; // Should not happen + if (frame.timestamp == 0 || mFrameNum < 0) { + // The first condition comes from grafika. + // The second condition means we were asked to stop. + mFramePool.recycle(frame); + return; + } + + mFrameNum++; + LOG.v("Incoming frame timestamp:", frame.timestamp); + // We must scale this matrix like GlCameraPreview does, because it might have some cropping. + // Scaling takes place with respect to the (0, 0, 0) point, so we must apply a Translation to compensate. + float[] transform = frame.transform; + float scaleX = mConfig.scaleX; + float scaleY = mConfig.scaleY; + float scaleTranslX = (1F - scaleX) / 2F; + float scaleTranslY = (1F - scaleY) / 2F; + Matrix.translateM(transform, 0, scaleTranslX, scaleTranslY, 0); + Matrix.scaleM(transform, 0, scaleX, scaleY, 1); + + // We also must rotate this matrix. In GlCameraPreview it is not needed because it is a live + // stream, but the output video, must be correctly rotated based on the device rotation at the moment. + // Rotation also takes place with respect to the origin (the Z axis), so we must + // translate to origin, rotate, then back to where we were. + + Matrix.translateM(transform, 0, 0.5F, 0.5F, 0); + Matrix.rotateM(transform, 0, mConfig.transformRotation, 0, 0, 1); + Matrix.translateM(transform, 0, -0.5F, -0.5F, 0); + + drainOutput(false); + // Future note: passing scale values to the viewport? They are scaleX and scaleY, + // but flipped based on the mConfig.scaleFlipped boolean. + mViewport.drawFrame(mConfig.textureId, transform); + mWindow.setPresentationTime(frame.timestamp); + mWindow.swapBuffers(); + mFramePool.recycle(frame); + } + + @Override + void onRelease() { + mFramePool.clear(); if (mWindow != null) { mWindow.release(); mWindow = null; @@ -77,58 +149,4 @@ class TextureMediaEncoder extends VideoMediaEncoder mEglCore = null; } } - - @EncoderThread - @Override - void start() { - super.start(); - // Nothing to do here. Waiting for the first frame. - } - - @EncoderThread - @Override - void notify(@NonNull String event, @Nullable Object data) { - if (event.equals(FRAME_EVENT)) { - Frame frame = (Frame) data; - - // Seeing this after device is toggled off/on with power button. The - // first frame back has a zero timestamp. - // MPEG4Writer thinks this is cause to abort() in native code, so it's very - // important that we just ignore the frame. - if (frame.timestamp == 0) return; - if (mFrameNum < 0) return; - mFrameNum++; - - int arg1 = (int) (frame.timestamp >> 32); - int arg2 = (int) frame.timestamp; - long timestamp = (((long) arg1) << 32) | (((long) arg2) & 0xffffffffL); - float[] transform = frame.transform; - - // We must scale this matrix like GlCameraPreview does, because it might have some cropping. - // Scaling takes place with respect to the (0, 0, 0) point, so we must apply a Translation to compensate. - - float scaleX = mConfig.scaleX; - float scaleY = mConfig.scaleY; - float scaleTranslX = (1F - scaleX) / 2F; - float scaleTranslY = (1F - scaleY) / 2F; - Matrix.translateM(transform, 0, scaleTranslX, scaleTranslY, 0); - Matrix.scaleM(transform, 0, scaleX, scaleY, 1); - - // We also must rotate this matrix. In GlCameraPreview it is not needed because it is a live - // stream, but the output video, must be correctly rotated based on the device rotation at the moment. - // Rotation also takes place with respect to the origin (the Z axis), so we must - // translate to origin, rotate, then back to where we were. - - Matrix.translateM(transform, 0, 0.5F, 0.5F, 0); - Matrix.rotateM(transform, 0, mConfig.transformRotation, 0, 0, 1); - Matrix.translateM(transform, 0, -0.5F, -0.5F, 0); - - drain(false); - // Future note: passing scale values to the viewport? They are scaleX and scaleY, - // but flipped based on the mConfig.scaleFlipped boolean. - mViewport.drawFrame(mConfig.textureId, transform); - mWindow.setPresentationTime(timestamp); - mWindow.swapBuffers(); - } - } } diff --git a/cameraview/src/main/gles/com/otaliastudios/cameraview/VideoMediaEncoder.java b/cameraview/src/main/gles/com/otaliastudios/cameraview/VideoMediaEncoder.java index 5c3c58db..13c4599a 100644 --- a/cameraview/src/main/gles/com/otaliastudios/cameraview/VideoMediaEncoder.java +++ b/cameraview/src/main/gles/com/otaliastudios/cameraview/VideoMediaEncoder.java @@ -51,10 +51,15 @@ abstract class VideoMediaEncoder extends Med mConfig = config; } + @NonNull + @Override + String getName() { + return "VideoEncoder"; + } + @EncoderThread @Override - void prepare(@NonNull MediaEncoderEngine.Controller controller, long maxLengthMillis) { - super.prepare(controller, maxLengthMillis); + void onPrepare(@NonNull MediaEncoderEngine.Controller controller, long maxLengthMillis) { MediaFormat format = MediaFormat.createVideoFormat(mConfig.mimeType, mConfig.width, mConfig.height); // Set some properties. Failing to specify some of these can cause the MediaCodec @@ -62,6 +67,7 @@ abstract class VideoMediaEncoder extends Med format.setInteger(MediaFormat.KEY_COLOR_FORMAT, MediaCodecInfo.CodecCapabilities.COLOR_FormatSurface); format.setInteger(MediaFormat.KEY_BIT_RATE, mConfig.bitRate); format.setInteger(MediaFormat.KEY_FRAME_RATE, mConfig.frameRate); + format.setInteger(MediaFormat.KEY_FRAME_RATE, 6); // TODO format.setInteger(MediaFormat.KEY_I_FRAME_INTERVAL, 2); format.setInteger("rotation-degrees", mConfig.rotation); @@ -79,20 +85,21 @@ abstract class VideoMediaEncoder extends Med @EncoderThread @Override - void start() { + void onStart() { // Nothing to do here. Waiting for the first frame. mFrameNum = 0; } @EncoderThread @Override - void stop() { + void onStop() { mFrameNum = -1; - drain(true); + signalEndOfInputStream(); + drainOutput(true); } @Override - int getBitRate() { + int getEncodedBitRate() { return mConfig.bitRate; } } diff --git a/cameraview/src/main/java/com/otaliastudios/cameraview/SnapshotVideoRecorder.java b/cameraview/src/main/java/com/otaliastudios/cameraview/SnapshotVideoRecorder.java index c3003f18..e707dec9 100644 --- a/cameraview/src/main/java/com/otaliastudios/cameraview/SnapshotVideoRecorder.java +++ b/cameraview/src/main/java/com/otaliastudios/cameraview/SnapshotVideoRecorder.java @@ -58,8 +58,13 @@ class SnapshotVideoRecorder extends VideoRecorder implements GlCameraPreview.Ren @Override public void onRendererFrame(@NonNull SurfaceTexture surfaceTexture, float scaleX, float scaleY) { if (mCurrentState == STATE_NOT_RECORDING && mDesiredState == STATE_RECORDING) { + // Set default options + if (mResult.videoBitRate <= 0) mResult.videoBitRate = DEFAULT_VIDEO_BITRATE; + if (mResult.videoFrameRate <= 0) mResult.videoFrameRate = DEFAULT_VIDEO_FRAMERATE; + if (mResult.audioBitRate <= 0) mResult.audioBitRate = DEFAULT_AUDIO_BITRATE; + + // Video. Ensure width and height are divisible by 2, as I have read somewhere. Size size = mResult.getSize(); - // Ensure width and height are divisible by 2, as I have read somewhere. int width = size.getWidth(); int height = size.getHeight(); width = width % 2 == 0 ? width : width + 1; @@ -70,9 +75,6 @@ class SnapshotVideoRecorder extends VideoRecorder implements GlCameraPreview.Ren case H_264: type = "video/avc"; break; // MediaFormat.MIMETYPE_VIDEO_AVC: case DEVICE_DEFAULT: type = "video/avc"; break; } - if (mResult.videoBitRate <= 0) mResult.videoBitRate = DEFAULT_VIDEO_BITRATE; - if (mResult.audioBitRate <= 0) mResult.audioBitRate = DEFAULT_AUDIO_BITRATE; - if (mResult.videoFrameRate <= 0) mResult.videoFrameRate = DEFAULT_VIDEO_FRAMERATE; LOG.w("Creating frame encoder. Rotation:", mResult.rotation); TextureMediaEncoder.Config config = new TextureMediaEncoder.Config(width, height, mResult.videoBitRate, @@ -84,10 +86,14 @@ class SnapshotVideoRecorder extends VideoRecorder implements GlCameraPreview.Ren EGL14.eglGetCurrentContext() ); TextureMediaEncoder videoEncoder = new TextureMediaEncoder(config); + + // Audio AudioMediaEncoder audioEncoder = null; if (mResult.audio == Audio.ON) { audioEncoder = new AudioMediaEncoder(new AudioMediaEncoder.Config(mResult.audioBitRate)); } + + // Engine mEncoderEngine = new MediaEncoderEngine(mResult.file, videoEncoder, audioEncoder, mResult.maxDuration, mResult.maxSize, SnapshotVideoRecorder.this); mEncoderEngine.start(); @@ -96,11 +102,11 @@ class SnapshotVideoRecorder extends VideoRecorder implements GlCameraPreview.Ren } if (mCurrentState == STATE_RECORDING) { - TextureMediaEncoder.Frame frame = new TextureMediaEncoder.Frame(); - frame.timestamp = surfaceTexture.getTimestamp(); - frame.transform = new float[16]; // TODO would be cool to avoid this at every frame. But it's not easy. - surfaceTexture.getTransformMatrix(frame.transform); - mEncoderEngine.notify(TextureMediaEncoder.FRAME_EVENT, frame); + TextureMediaEncoder textureEncoder = (TextureMediaEncoder) mEncoderEngine.getVideoEncoder(); + TextureMediaEncoder.TextureFrame textureFrame = textureEncoder.acquireFrame(); + textureFrame.timestamp = surfaceTexture.getTimestamp(); + surfaceTexture.getTransformMatrix(textureFrame.transform); + mEncoderEngine.notify(TextureMediaEncoder.FRAME_EVENT, textureFrame); } if (mCurrentState == STATE_RECORDING && mDesiredState == STATE_NOT_RECORDING) { @@ -113,7 +119,6 @@ class SnapshotVideoRecorder extends VideoRecorder implements GlCameraPreview.Ren } - @EncoderThread @Override public void onEncoderStop(int stopReason, @Nullable Exception e) { // If something failed, undo the result, since this is the mechanism diff --git a/cameraview/src/main/java/com/otaliastudios/cameraview/VideoRecorder.java b/cameraview/src/main/java/com/otaliastudios/cameraview/VideoRecorder.java index 5b2c4d1b..22d512b0 100644 --- a/cameraview/src/main/java/com/otaliastudios/cameraview/VideoRecorder.java +++ b/cameraview/src/main/java/com/otaliastudios/cameraview/VideoRecorder.java @@ -22,6 +22,7 @@ abstract class VideoRecorder { abstract void stop(); + @SuppressWarnings("WeakerAccess") protected void dispatchResult() { if (mListener != null) { mListener.onVideoResult(mResult); diff --git a/cameraview/src/main/utils/com/otaliastudios/cameraview/WorkerHandler.java b/cameraview/src/main/utils/com/otaliastudios/cameraview/WorkerHandler.java index db2142b2..aeb1e040 100644 --- a/cameraview/src/main/utils/com/otaliastudios/cameraview/WorkerHandler.java +++ b/cameraview/src/main/utils/com/otaliastudios/cameraview/WorkerHandler.java @@ -2,6 +2,8 @@ package com.otaliastudios.cameraview; import android.os.Handler; import android.os.HandlerThread; +import android.os.Looper; + import androidx.annotation.NonNull; import java.lang.ref.WeakReference; @@ -63,16 +65,22 @@ class WorkerHandler { } @NonNull - public Thread getThread() { + public HandlerThread getThread() { return mThread; } + @NonNull + public Looper getLooper() { + return mThread.getLooper(); + } + static void destroy() { for (String key : sCache.keySet()) { WeakReference ref = sCache.get(key); WorkerHandler handler = ref.get(); if (handler != null && handler.getThread().isAlive()) { handler.getThread().interrupt(); + // handler.getThread().quit(); } ref.clear(); } diff --git a/demo/src/main/java/com/otaliastudios/cameraview/demo/CameraActivity.java b/demo/src/main/java/com/otaliastudios/cameraview/demo/CameraActivity.java index 125e1315..0b0d348a 100644 --- a/demo/src/main/java/com/otaliastudios/cameraview/demo/CameraActivity.java +++ b/demo/src/main/java/com/otaliastudios/cameraview/demo/CameraActivity.java @@ -2,6 +2,7 @@ package com.otaliastudios.cameraview.demo; import android.content.Intent; import android.content.pm.PackageManager; +import android.net.Uri; import android.os.Bundle; import androidx.annotation.NonNull; import com.google.android.material.bottomsheet.BottomSheetBehavior;