From 8ec369998ca2b81e239a7c757a07aaf81aeb5689 Mon Sep 17 00:00:00 2001 From: Mattia Iavarone Date: Tue, 18 Sep 2018 11:43:44 -0300 Subject: [PATCH] Add max size and duration constraints to videos --- .../cameraview/AudioMediaEncoder.java | 21 +++-- .../cameraview/MediaEncoder.java | 34 +++++-- .../cameraview/MediaEncoderEngine.java | 94 +++++++++++++++---- .../cameraview/TextureMediaEncoder.java | 4 +- .../cameraview/VideoMediaEncoder.java | 11 +-- .../com/otaliastudios/cameraview/Camera1.java | 2 + .../cameraview/CameraException.java | 17 ++++ .../cameraview/SnapshotVideoRecorder.java | 43 ++++++--- .../cameraview/demo/CameraActivity.java | 8 ++ 9 files changed, 181 insertions(+), 53 deletions(-) diff --git a/cameraview/src/main/gles/com/otaliastudios/cameraview/AudioMediaEncoder.java b/cameraview/src/main/gles/com/otaliastudios/cameraview/AudioMediaEncoder.java index 1a39c8e7..077da416 100644 --- a/cameraview/src/main/gles/com/otaliastudios/cameraview/AudioMediaEncoder.java +++ b/cameraview/src/main/gles/com/otaliastudios/cameraview/AudioMediaEncoder.java @@ -19,29 +19,33 @@ class AudioMediaEncoder extends MediaEncoder { private static final String MIME_TYPE = "audio/mp4a-latm"; private static final int SAMPLE_RATE = 44100; // 44.1[KHz] is only setting guaranteed to be available on all devices. - private static final int BIT_RATE = 64000; public static final int SAMPLES_PER_FRAME = 1024; // AAC, bytes/frame/channel public static final int FRAMES_PER_BUFFER = 25; // AAC, frame/buffer/sec private final Object mLock = new Object(); private boolean mRequestStop = false; + private Config mConfig; static class Config { - Config() { } + int bitRate; + + Config(int bitRate) { + this.bitRate = bitRate; + } } AudioMediaEncoder(@NonNull Config config) { - + mConfig = config; } @EncoderThread @Override - void prepare(MediaEncoderEngine.Controller controller) { - super.prepare(controller); + void prepare(MediaEncoderEngine.Controller controller, long maxLengthMillis) { + super.prepare(controller, maxLengthMillis); final MediaFormat audioFormat = MediaFormat.createAudioFormat(MIME_TYPE, SAMPLE_RATE, 1); audioFormat.setInteger(MediaFormat.KEY_AAC_PROFILE, MediaCodecInfo.CodecProfileLevel.AACObjectLC); audioFormat.setInteger(MediaFormat.KEY_CHANNEL_MASK, AudioFormat.CHANNEL_IN_MONO); - audioFormat.setInteger(MediaFormat.KEY_BIT_RATE, BIT_RATE); + audioFormat.setInteger(MediaFormat.KEY_BIT_RATE, mConfig.bitRate); audioFormat.setInteger(MediaFormat.KEY_CHANNEL_COUNT, 1); try { mMediaCodec = MediaCodec.createEncoderByType(MIME_TYPE); @@ -125,4 +129,9 @@ class AudioMediaEncoder extends MediaEncoder { } } } + + @Override + int getBitRate() { + return mConfig.bitRate; + } } diff --git a/cameraview/src/main/gles/com/otaliastudios/cameraview/MediaEncoder.java b/cameraview/src/main/gles/com/otaliastudios/cameraview/MediaEncoder.java index 977e777b..429e7dd0 100644 --- a/cameraview/src/main/gles/com/otaliastudios/cameraview/MediaEncoder.java +++ b/cameraview/src/main/gles/com/otaliastudios/cameraview/MediaEncoder.java @@ -2,7 +2,6 @@ package com.otaliastudios.cameraview; import android.media.MediaCodec; import android.media.MediaFormat; -import android.media.MediaMuxer; import android.os.Build; import android.support.annotation.RequiresApi; import android.util.Log; @@ -20,6 +19,8 @@ abstract class MediaEncoder { private MediaCodec.BufferInfo mBufferInfo; private MediaEncoderEngine.Controller mController; private int mTrackIndex; + private long mMaxLengthMillis; + private boolean mMaxLengthReached; /** * Called to prepare this encoder before starting. @@ -31,9 +32,10 @@ abstract class MediaEncoder { * @param controller the muxer controller */ @EncoderThread - void prepare(MediaEncoderEngine.Controller controller) { + void prepare(MediaEncoderEngine.Controller controller, long maxLengthMillis) { mController = controller; mBufferInfo = new MediaCodec.BufferInfo(); + mMaxLengthMillis = maxLengthMillis; } /** @@ -149,28 +151,40 @@ abstract class MediaEncoder { encodedData.position(mBufferInfo.offset); encodedData.limit(mBufferInfo.offset + mBufferInfo.size); mController.write(mTrackIndex, encodedData, mBufferInfo); - mPreviousTime = mBufferInfo.presentationTimeUs; + mLastPresentationTime = mBufferInfo.presentationTimeUs; + if (mStartPresentationTime == 0) { + mStartPresentationTime = mLastPresentationTime; + } } mMediaCodec.releaseOutputBuffer(encoderStatus, false); - if ((mBufferInfo.flags & MediaCodec.BUFFER_FLAG_END_OF_STREAM) != 0) { - if (!endOfStream) { - Log.w("VideoMediaEncoder", "reached end of stream unexpectedly"); + if (!mMaxLengthReached) { + if (mLastPresentationTime / 1000 - mStartPresentationTime / 1000 > mMaxLengthMillis) { + mMaxLengthReached = true; + // Log.e("MediaEncoder", this.getClass().getSimpleName() + " requested stop at " + (mLastPresentationTime * 1000 * 1000)); + mController.requestStop(); + break; } + } + + if ((mBufferInfo.flags & MediaCodec.BUFFER_FLAG_END_OF_STREAM) != 0) { break; // out of while } } } } - private long mPreviousTime = 0; + private long mStartPresentationTime = 0; + private long mLastPresentationTime = 0; - protected long getPresentationTime() { + long getPresentationTime() { long result = System.nanoTime() / 1000L; // presentationTimeUs should be monotonic // otherwise muxer fail to write - if (result < mPreviousTime) { - result = (mPreviousTime - result) + result; + if (result < mLastPresentationTime) { + result = (mLastPresentationTime - result) + result; } return result; } + + abstract int getBitRate(); } diff --git a/cameraview/src/main/gles/com/otaliastudios/cameraview/MediaEncoderEngine.java b/cameraview/src/main/gles/com/otaliastudios/cameraview/MediaEncoderEngine.java index ff963d6d..8e5e90f6 100644 --- a/cameraview/src/main/gles/com/otaliastudios/cameraview/MediaEncoderEngine.java +++ b/cameraview/src/main/gles/com/otaliastudios/cameraview/MediaEncoderEngine.java @@ -19,15 +19,25 @@ import java.util.List; @RequiresApi(api = Build.VERSION_CODES.JELLY_BEAN_MR2) class MediaEncoderEngine { + final static int STOP_BY_USER = 0; + final static int STOP_BY_MAX_DURATION = 1; + final static int STOP_BY_MAX_SIZE = 2; + private WorkerHandler mWorker; private ArrayList mEncoders; private MediaMuxer mMediaMuxer; private int mMediaMuxerStartCount; private boolean mMediaMuxerStarted; private Controller mController; + private Listener mListener; + private int mStopReason = STOP_BY_USER; + private int mPossibleStopReason; + private final Object mLock = new Object(); - MediaEncoderEngine(@NonNull File file, @NonNull VideoMediaEncoder videoEncoder, @Nullable AudioMediaEncoder audioEncoder) { + MediaEncoderEngine(@NonNull File file, @NonNull VideoMediaEncoder videoEncoder, @Nullable AudioMediaEncoder audioEncoder, + final int maxDuration, final long maxSize, @Nullable Listener listener) { mWorker = WorkerHandler.get("EncoderEngine"); + mListener = listener; mController = new Controller(); mEncoders = new ArrayList<>(); mEncoders.add(videoEncoder); @@ -44,32 +54,58 @@ class MediaEncoderEngine { mWorker.post(new Runnable() { @Override public void run() { + // Trying to convert the size constraints to duration constraints, + // because they are super easy to check. + int bitRate = 0; + for (MediaEncoder encoder : mEncoders) { + bitRate += encoder.getBitRate(); + } + int bytePerSecond = bitRate / 8; + long sizeMaxDuration = (maxSize / bytePerSecond) * 1000L; + long finalMaxDuration = Long.MAX_VALUE; + if (maxSize > 0 && maxDuration > 0) { + mPossibleStopReason = sizeMaxDuration < maxDuration ? STOP_BY_MAX_SIZE : STOP_BY_MAX_DURATION; + finalMaxDuration = Math.min(sizeMaxDuration, maxDuration); + } else if (maxSize > 0) { + mPossibleStopReason = STOP_BY_MAX_SIZE; + finalMaxDuration = sizeMaxDuration; + } else if (maxDuration > 0) { + mPossibleStopReason = STOP_BY_MAX_DURATION; + finalMaxDuration = maxDuration; + } + Log.e("MediaEncoderEngine", "Computed a max duration of " + (finalMaxDuration / 1000F)); for (MediaEncoder encoder : mEncoders) { - encoder.prepare(mController); + encoder.prepare(mController, finalMaxDuration); } } }); } + // Stuff here might be called from multiple threads. class Controller { int start(MediaFormat format) { - if (mMediaMuxerStarted) { - throw new IllegalStateException("Trying to start but muxer started already"); - } - int track = mMediaMuxer.addTrack(format); - mMediaMuxerStartCount++; - if (mMediaMuxerStartCount == mEncoders.size()) { - mMediaMuxer.start(); - mMediaMuxerStarted = true; + synchronized (mLock) { + if (mMediaMuxerStarted) { + throw new IllegalStateException("Trying to start but muxer started already"); + } + int track = mMediaMuxer.addTrack(format); + mMediaMuxerStartCount++; + if (mMediaMuxerStartCount == mEncoders.size()) { + mMediaMuxer.start(); + mMediaMuxerStarted = true; + } + return track; } - return track; } boolean isStarted() { - return mMediaMuxerStarted; + synchronized (mLock) { + return mMediaMuxerStarted; + } } + // Synchronization does not seem needed here. void write(int track, ByteBuffer encodedData, MediaCodec.BufferInfo info) { if (!mMediaMuxerStarted) { throw new IllegalStateException("Trying to write before muxer started"); @@ -77,6 +113,16 @@ class MediaEncoderEngine { Log.e("MediaEncoderEngine", "Writing data." + track); mMediaMuxer.writeSampleData(track, encodedData, info); } + + void requestStop() { + synchronized (mLock) { + mMediaMuxerStartCount--; + if (mMediaMuxerStartCount == 0) { + mStopReason = mPossibleStopReason; + stop(); + } + } + } } void start() { @@ -101,7 +147,7 @@ class MediaEncoderEngine { }); } - void stop(final Runnable onStop) { + void stop() { mWorker.post(new Runnable() { @Override public void run() { @@ -111,17 +157,31 @@ class MediaEncoderEngine { for (MediaEncoder encoder : mEncoders) { encoder.release(); } + Exception error = null; if (mMediaMuxer != null) { // stop() throws an exception if you haven't fed it any data. - // We can just swallow I think. - mMediaMuxer.stop(); - mMediaMuxer.release(); + // But also in other occasions. So this is a signal that something + // went wrong, and we propagate that to the listener. + try { + mMediaMuxer.stop(); + mMediaMuxer.release(); + } catch (Exception e) { + error = e; + } mMediaMuxer = null; } - onStop.run(); + if (mListener != null) mListener.onEncoderStop(mStopReason, error); + mStopReason = STOP_BY_USER; + mListener = null; mMediaMuxerStartCount = 0; mMediaMuxerStarted = false; } }); } + + interface Listener { + + @EncoderThread + void onEncoderStop(int stopReason, @Nullable Exception e); + } } diff --git a/cameraview/src/main/gles/com/otaliastudios/cameraview/TextureMediaEncoder.java b/cameraview/src/main/gles/com/otaliastudios/cameraview/TextureMediaEncoder.java index eb590bca..0229194f 100644 --- a/cameraview/src/main/gles/com/otaliastudios/cameraview/TextureMediaEncoder.java +++ b/cameraview/src/main/gles/com/otaliastudios/cameraview/TextureMediaEncoder.java @@ -50,8 +50,8 @@ class TextureMediaEncoder extends VideoMediaEncoder @EncoderThread @Override - void prepare(MediaEncoderEngine.Controller controller) { - super.prepare(controller); + void prepare(MediaEncoderEngine.Controller controller, long maxLengthMillis) { + super.prepare(controller, maxLengthMillis); mEglCore = new EglCore(mConfig.eglContext, EglCore.FLAG_RECORDABLE); mWindow = new EglWindowSurface(mEglCore, mSurface, true); mWindow.makeCurrent(); // drawing will happen on the InputWindowSurface, which diff --git a/cameraview/src/main/gles/com/otaliastudios/cameraview/VideoMediaEncoder.java b/cameraview/src/main/gles/com/otaliastudios/cameraview/VideoMediaEncoder.java index f6c97316..d5af49d7 100644 --- a/cameraview/src/main/gles/com/otaliastudios/cameraview/VideoMediaEncoder.java +++ b/cameraview/src/main/gles/com/otaliastudios/cameraview/VideoMediaEncoder.java @@ -50,8 +50,8 @@ abstract class VideoMediaEncoder extends Med @EncoderThread @Override - void prepare(MediaEncoderEngine.Controller controller) { - super.prepare(controller); + void prepare(MediaEncoderEngine.Controller controller, long maxLengthMillis) { + super.prepare(controller, maxLengthMillis); MediaFormat format = MediaFormat.createVideoFormat(mConfig.mimeType, mConfig.width, mConfig.height); // Set some properties. Failing to specify some of these can cause the MediaCodec @@ -59,7 +59,7 @@ abstract class VideoMediaEncoder extends Med format.setInteger(MediaFormat.KEY_COLOR_FORMAT, MediaCodecInfo.CodecCapabilities.COLOR_FormatSurface); format.setInteger(MediaFormat.KEY_BIT_RATE, mConfig.bitRate); format.setInteger(MediaFormat.KEY_FRAME_RATE, mConfig.frameRate); - format.setInteger(MediaFormat.KEY_I_FRAME_INTERVAL, 5); + format.setInteger(MediaFormat.KEY_I_FRAME_INTERVAL, 2); format.setInteger("rotation-degrees", mConfig.rotation); // Create a MediaCodec encoder, and configure it with our format. Get a Surface @@ -88,9 +88,8 @@ abstract class VideoMediaEncoder extends Med drain(true); } - @EncoderThread @Override - void release() { - super.release(); + int getBitRate() { + return mConfig.bitRate; } } diff --git a/cameraview/src/main/java/com/otaliastudios/cameraview/Camera1.java b/cameraview/src/main/java/com/otaliastudios/cameraview/Camera1.java index 7b8c7ddf..f2184412 100644 --- a/cameraview/src/main/java/com/otaliastudios/cameraview/Camera1.java +++ b/cameraview/src/main/java/com/otaliastudios/cameraview/Camera1.java @@ -512,6 +512,7 @@ class Camera1 extends CameraController implements Camera.PreviewCallback, Camera mCameraCallbacks.dispatchOnPictureTaken(result); } else { // Something went wrong. + mCameraCallbacks.dispatchError(new CameraException(CameraException.REASON_PICTURE_FAILED)); LOG.e("onPictureResult", "result is null: something went wrong."); } } @@ -603,6 +604,7 @@ class Camera1 extends CameraController implements Camera.PreviewCallback, Camera mCameraCallbacks.dispatchOnVideoTaken(result); } else { // Something went wrong, lock the camera again. + mCameraCallbacks.dispatchError(new CameraException(CameraException.REASON_VIDEO_FAILED)); mCamera.lock(); } } diff --git a/cameraview/src/main/java/com/otaliastudios/cameraview/CameraException.java b/cameraview/src/main/java/com/otaliastudios/cameraview/CameraException.java index 62aa6cca..58111706 100644 --- a/cameraview/src/main/java/com/otaliastudios/cameraview/CameraException.java +++ b/cameraview/src/main/java/com/otaliastudios/cameraview/CameraException.java @@ -30,6 +30,18 @@ public class CameraException extends RuntimeException { */ public static final int REASON_DISCONNECTED = 3; + /** + * Could not take a picture or a picture snapshot, + * for some not specified reason. + */ + public static final int REASON_PICTURE_FAILED = 4; + + /** + * Could not take a video or a video snapshot, + * for some not specified reason. + */ + public static final int REASON_VIDEO_FAILED = 5; + private int reason = REASON_UNKNOWN; CameraException(Throwable cause) { @@ -41,6 +53,11 @@ public class CameraException extends RuntimeException { this.reason = reason; } + CameraException(int reason) { + super(); + this.reason = reason; + } + public int getReason() { return reason; } diff --git a/cameraview/src/main/java/com/otaliastudios/cameraview/SnapshotVideoRecorder.java b/cameraview/src/main/java/com/otaliastudios/cameraview/SnapshotVideoRecorder.java index a368e92a..b359c9ce 100644 --- a/cameraview/src/main/java/com/otaliastudios/cameraview/SnapshotVideoRecorder.java +++ b/cameraview/src/main/java/com/otaliastudios/cameraview/SnapshotVideoRecorder.java @@ -4,13 +4,15 @@ import android.graphics.SurfaceTexture; import android.opengl.EGL14; import android.os.Build; import android.os.Handler; +import android.support.annotation.Nullable; import android.support.annotation.RequiresApi; /** * A {@link VideoRecorder} that uses {@link android.media.MediaCodec} APIs. */ @RequiresApi(api = Build.VERSION_CODES.JELLY_BEAN_MR2) -class SnapshotVideoRecorder extends VideoRecorder implements GlCameraPreview.RendererFrameCallback { +class SnapshotVideoRecorder extends VideoRecorder implements GlCameraPreview.RendererFrameCallback, + MediaEncoderEngine.Listener { private static final String TAG = SnapshotVideoRecorder.class.getSimpleName(); private static final CameraLogger LOG = CameraLogger.create(TAG); @@ -36,14 +38,14 @@ class SnapshotVideoRecorder extends VideoRecorder implements GlCameraPreview.Ren mDesiredState = STATE_RECORDING; // TODO respect maxSize by doing inspecting frameRate, bitRate and frame size? // TODO do this at the encoder level, not here with a handler. - if (mResult.maxDuration > 0) { + /* if (mResult.maxDuration > 0) { new Handler().postDelayed(new Runnable() { @Override public void run() { mDesiredState = STATE_NOT_RECORDING; } }, (long) mResult.maxDuration); - } + } */ } @Override @@ -85,9 +87,10 @@ class SnapshotVideoRecorder extends VideoRecorder implements GlCameraPreview.Ren TextureMediaEncoder videoEncoder = new TextureMediaEncoder(config); AudioMediaEncoder audioEncoder = null; if (mResult.audio == Audio.ON) { - audioEncoder = new AudioMediaEncoder(new AudioMediaEncoder.Config()); + audioEncoder = new AudioMediaEncoder(new AudioMediaEncoder.Config(64000)); } - mEncoderEngine = new MediaEncoderEngine(mResult.file, videoEncoder, audioEncoder); + mEncoderEngine = new MediaEncoderEngine(mResult.file, videoEncoder, audioEncoder, + mResult.maxDuration, mResult.maxSize, SnapshotVideoRecorder.this); mEncoderEngine.start(); mResult.rotation = 0; // We will rotate the result instead. mCurrentState = STATE_RECORDING; @@ -102,13 +105,7 @@ class SnapshotVideoRecorder extends VideoRecorder implements GlCameraPreview.Ren } if (mCurrentState == STATE_RECORDING && mDesiredState == STATE_NOT_RECORDING) { - mEncoderEngine.stop(new Runnable() { - @Override - public void run() { - // We are in the encoder thread. - dispatchResult(); - } - }); + mEncoderEngine.stop(); mEncoderEngine = null; mCurrentState = STATE_NOT_RECORDING; mPreview.removeRendererFrameCallback(SnapshotVideoRecorder.this); @@ -116,4 +113,26 @@ class SnapshotVideoRecorder extends VideoRecorder implements GlCameraPreview.Ren } } + + @EncoderThread + @Override + public void onEncoderStop(int stopReason, @Nullable Exception e) { + // If something failed, undo the result, since this is the mechanism + // to notify Camera1 about this. + if (e != null) { + mResult = null; + } else { + if (stopReason == MediaEncoderEngine.STOP_BY_MAX_DURATION) { + mResult.endReason = VideoResult.REASON_MAX_DURATION_REACHED; + } else if (stopReason == MediaEncoderEngine.STOP_BY_MAX_SIZE) { + mResult.endReason = VideoResult.REASON_MAX_SIZE_REACHED; + } + } + mEncoderEngine = null; + if (mPreview != null) { + mPreview.removeRendererFrameCallback(SnapshotVideoRecorder.this); + mPreview = null; + } + dispatchResult(); + } } diff --git a/demo/src/main/java/com/otaliastudios/cameraview/demo/CameraActivity.java b/demo/src/main/java/com/otaliastudios/cameraview/demo/CameraActivity.java index c1252214..7eedab53 100644 --- a/demo/src/main/java/com/otaliastudios/cameraview/demo/CameraActivity.java +++ b/demo/src/main/java/com/otaliastudios/cameraview/demo/CameraActivity.java @@ -13,6 +13,7 @@ import android.view.ViewTreeObserver; import android.view.WindowManager; import android.widget.Toast; +import com.otaliastudios.cameraview.CameraException; import com.otaliastudios.cameraview.CameraListener; import com.otaliastudios.cameraview.CameraLogger; import com.otaliastudios.cameraview.CameraOptions; @@ -47,6 +48,9 @@ public class CameraActivity extends AppCompatActivity implements View.OnClickLis public void onCameraOpened(CameraOptions options) { onOpened(); } public void onPictureTaken(PictureResult result) { onPicture(result); } public void onVideoTaken(VideoResult result) { onVideo(result.getFile()); } + public void onCameraError(@NonNull CameraException exception) { + onError(exception); + } }); findViewById(R.id.edit).setOnClickListener(this); @@ -88,6 +92,10 @@ public class CameraActivity extends AppCompatActivity implements View.OnClickLis } } + private void onError(@NonNull CameraException exception) { + message("Got CameraException #" + exception.getReason(), true); + } + private void onPicture(PictureResult result) { if (camera.isTakingVideo()) { message("Captured while taking video. Size=" + result.getSize(), false);