From d150bd268134d375b3f2b51004dbebf1d9fd6d88 Mon Sep 17 00:00:00 2001 From: Mattia Iavarone Date: Wed, 31 Jul 2019 20:35:00 +0200 Subject: [PATCH] Warmup worker threads, enlarge audio buffers, use EncoderEngine thread --- .../internal/utils/WorkerHandler.java | 16 +++++- .../video/encoding/AudioConfig.java | 4 +- .../video/encoding/MediaEncoder.java | 1 + .../video/encoding/MediaEncoderEngine.java | 57 +++++++++++++------ .../video/encoding/TextureMediaEncoder.java | 18 +++--- 5 files changed, 68 insertions(+), 28 deletions(-) diff --git a/cameraview/src/main/java/com/otaliastudios/cameraview/internal/utils/WorkerHandler.java b/cameraview/src/main/java/com/otaliastudios/cameraview/internal/utils/WorkerHandler.java index fe33edbd..6d0a2f49 100644 --- a/cameraview/src/main/java/com/otaliastudios/cameraview/internal/utils/WorkerHandler.java +++ b/cameraview/src/main/java/com/otaliastudios/cameraview/internal/utils/WorkerHandler.java @@ -14,6 +14,7 @@ import androidx.annotation.NonNull; import java.lang.ref.WeakReference; import java.util.concurrent.Callable; import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.CountDownLatch; import java.util.concurrent.Executor; /** @@ -100,6 +101,20 @@ public class WorkerHandler { WorkerHandler.this.run(command); } }; + + // HandlerThreads/Handlers sometimes have a significant warmup time. + // We want to spend this time here so when this object is built, it + // is fully operational. + final CountDownLatch latch = new CountDownLatch(1); + post(new Runnable() { + @Override + public void run() { + latch.countDown(); + } + }); + try { + latch.await(); + } catch (InterruptedException ignore) {} } /** @@ -219,7 +234,6 @@ public class WorkerHandler { * interrupt it, so the next {@link #get(String)} call will remove it. * In any case, we only store weak references. */ - @SuppressWarnings("WeakerAccess") public void destroy() { HandlerThread thread = getThread(); if (thread.isAlive()) { diff --git a/cameraview/src/main/java/com/otaliastudios/cameraview/video/encoding/AudioConfig.java b/cameraview/src/main/java/com/otaliastudios/cameraview/video/encoding/AudioConfig.java index c1f2bf89..c677c401 100644 --- a/cameraview/src/main/java/com/otaliastudios/cameraview/video/encoding/AudioConfig.java +++ b/cameraview/src/main/java/com/otaliastudios/cameraview/video/encoding/AudioConfig.java @@ -62,7 +62,7 @@ public class AudioConfig { * @return the frame size */ int frameSize() { - return 1024 * channels; + return 2048 * channels; } /** @@ -91,6 +91,6 @@ public class AudioConfig { * @return the buffer pool max size */ int bufferPoolMaxSize() { - return 200; + return 500; } } diff --git a/cameraview/src/main/java/com/otaliastudios/cameraview/video/encoding/MediaEncoder.java b/cameraview/src/main/java/com/otaliastudios/cameraview/video/encoding/MediaEncoder.java index 05f2493c..5c9ac91e 100644 --- a/cameraview/src/main/java/com/otaliastudios/cameraview/video/encoding/MediaEncoder.java +++ b/cameraview/src/main/java/com/otaliastudios/cameraview/video/encoding/MediaEncoder.java @@ -180,6 +180,7 @@ public abstract class MediaEncoder { mBufferInfo = new MediaCodec.BufferInfo(); mMaxLengthMillis = maxLengthMillis; mWorker = WorkerHandler.get(mName); + mWorker.getThread().setPriority(Thread.MAX_PRIORITY); LOG.i(mName, "Prepare was called. Posting."); mWorker.post(new Runnable() { @Override diff --git a/cameraview/src/main/java/com/otaliastudios/cameraview/video/encoding/MediaEncoderEngine.java b/cameraview/src/main/java/com/otaliastudios/cameraview/video/encoding/MediaEncoderEngine.java index e97358b9..0ee9e60a 100644 --- a/cameraview/src/main/java/com/otaliastudios/cameraview/video/encoding/MediaEncoderEngine.java +++ b/cameraview/src/main/java/com/otaliastudios/cameraview/video/encoding/MediaEncoderEngine.java @@ -7,6 +7,7 @@ import android.os.Build; import android.text.format.DateFormat; import com.otaliastudios.cameraview.CameraLogger; +import com.otaliastudios.cameraview.internal.utils.WorkerHandler; import androidx.annotation.NonNull; import androidx.annotation.Nullable; @@ -97,17 +98,18 @@ public class MediaEncoderEngine { public final static int END_BY_MAX_DURATION = 1; public final static int END_BY_MAX_SIZE = 2; - private List mEncoders; + private final List mEncoders = new ArrayList<>(); private MediaMuxer mMediaMuxer; private int mStartedEncodersCount = 0; private int mStoppedEncodersCount = 0; private boolean mMediaMuxerStarted = false; @SuppressWarnings("FieldCanBeLocal") - private Controller mController; + private final Controller mController = new Controller(); + private final WorkerHandler mControllerThread = WorkerHandler.get("EncoderEngine"); + private final Object mControllerLock = new Object(); private Listener mListener; private int mEndReason = END_BY_USER; private int mPossibleEndReason; - private final Object mControllerLock = new Object(); /** * Creates a new engine for the given file, with the given encoders and max limits, @@ -127,8 +129,6 @@ public class MediaEncoderEngine { final long maxSize, @Nullable Listener listener) { mListener = listener; - mController = new Controller(); - mEncoders = new ArrayList<>(); mEncoders.add(videoEncoder); if (audioEncoder != null) { mEncoders.add(audioEncoder); @@ -219,10 +219,14 @@ public class MediaEncoderEngine { // went wrong, and we propagate that to the listener. try { mMediaMuxer.stop(); - mMediaMuxer.release(); } catch (Exception e) { error = e; } + try { + mMediaMuxer.release(); + } catch (Exception e) { + if (error == null) error = e; + } mMediaMuxer = null; } LOG.w("end:", "Dispatching end to listener - reason:", mEndReason, "error:", error); @@ -234,6 +238,7 @@ public class MediaEncoderEngine { mStartedEncodersCount = 0; mStoppedEncodersCount = 0; mMediaMuxerStarted = false; + mControllerThread.destroy(); LOG.i("end:", "Completed."); } @@ -282,11 +287,18 @@ public class MediaEncoderEngine { LOG.w("notifyStarted:", "Assigned track", track, "to format", format.getString(MediaFormat.KEY_MIME)); if (++mStartedEncodersCount == mEncoders.size()) { LOG.w("notifyStarted:", "All encoders have started. Starting muxer and dispatching onEncodingStart()."); - mMediaMuxer.start(); - mMediaMuxerStarted = true; - if (mListener != null) { - mListener.onEncodingStart(); - } + // Go out of this thread since it might be very important for the + // encoders and we don't want to perform expensive operations here. + mControllerThread.run(new Runnable() { + @Override + public void run() { + mMediaMuxer.start(); + mMediaMuxerStarted = true; + if (mListener != null) { + mListener.onEncodingStart(); + } + } + }); } return track; } @@ -323,10 +335,6 @@ public class MediaEncoderEngine { * large differences. */ public void write(@NonNull OutputBufferPool pool, @NonNull OutputBuffer buffer) { - if (!mMediaMuxerStarted) { - throw new IllegalStateException("Trying to write before muxer started"); - } - if (DEBUG_PERFORMANCE) { // When AUDIO = mono, this is called about twice the time. (200 vs 100 for 5 sec). Integer count = mDebugCount.get(buffer.trackIndex); @@ -343,7 +351,6 @@ public class MediaEncoderEngine { "track:", buffer.trackIndex, "presentation:", buffer.info.presentationTimeUs); } - mMediaMuxer.writeSampleData(buffer.trackIndex, buffer.data, buffer.info); pool.recycle(buffer); } @@ -361,7 +368,14 @@ public class MediaEncoderEngine { if (--mStartedEncodersCount == 0) { LOG.w("requestStop:", "All encoders have requested a stop. Stopping them."); mEndReason = mPossibleEndReason; - stop(); + // Go out of this thread since it might be very important for the + // encoders and we don't want to perform expensive operations here. + mControllerThread.run(new Runnable() { + @Override + public void run() { + stop(); + } + }); } } } @@ -375,7 +389,14 @@ public class MediaEncoderEngine { LOG.w("notifyStopped:", "Called for track", track); if (++mStoppedEncodersCount == mEncoders.size()) { LOG.w("requestStop:", "All encoders have been stopped. Stopping the muxer."); - end(); + // Go out of this thread since it might be very important for the + // encoders and we don't want to perform expensive operations here. + mControllerThread.run(new Runnable() { + @Override + public void run() { + end(); + } + }); } } } diff --git a/cameraview/src/main/java/com/otaliastudios/cameraview/video/encoding/TextureMediaEncoder.java b/cameraview/src/main/java/com/otaliastudios/cameraview/video/encoding/TextureMediaEncoder.java index faf681c4..c3ce7377 100644 --- a/cameraview/src/main/java/com/otaliastudios/cameraview/video/encoding/TextureMediaEncoder.java +++ b/cameraview/src/main/java/com/otaliastudios/cameraview/video/encoding/TextureMediaEncoder.java @@ -29,7 +29,7 @@ public class TextureMediaEncoder extends VideoMediaEncoder { private EglCore mEglCore; private EglWindowSurface mWindow; private EglViewport mViewport; - private Pool mFramePool = new Pool<>(Integer.MAX_VALUE, new Pool.Factory() { + private Pool mFramePool = new Pool<>(100, new Pool.Factory() { @Override public Frame create() { return new Frame(); @@ -115,14 +115,18 @@ public class TextureMediaEncoder extends VideoMediaEncoder { */ @Override protected boolean shouldRenderFrame(long timestampUs) { - if (!super.shouldRenderFrame(timestampUs)) return false; - if (mFrameNumber <= 5) return true; // Always render the first few frames, or muxer fails. - int events = getPendingEvents(FRAME_EVENT); - if (events > 2) { - LOG.w("Dropping frame because we already have too many pending events.", events); + if (!super.shouldRenderFrame(timestampUs)) { return false; + } else if (mFrameNumber <= 10) { + // Always render the first few frames, or muxer fails. + return true; + } else if (getPendingEvents(FRAME_EVENT) > 2) { + LOG.w("shouldRenderFrame - Dropping frame because we already have too many pending events:", + getPendingEvents(FRAME_EVENT)); + return false; + } else { + return true; } - return true; } @EncoderThread