From c63fe6e89d813dd4b3a99899d8901cfd1ebfcbfe Mon Sep 17 00:00:00 2001 From: Mattia Iavarone Date: Mon, 17 Sep 2018 22:06:56 -0300 Subject: [PATCH] Fix muxer issue --- .../cameraview/AudioMediaEncoder.java | 4 +- .../cameraview/MediaEncoder.java | 46 +++++++------------ .../cameraview/MediaEncoderEngine.java | 39 +++++++++++++++- .../cameraview/TextureMediaEncoder.java | 6 +-- .../cameraview/VideoMediaEncoder.java | 4 +- 5 files changed, 61 insertions(+), 38 deletions(-) diff --git a/cameraview/src/main/gles/com/otaliastudios/cameraview/AudioMediaEncoder.java b/cameraview/src/main/gles/com/otaliastudios/cameraview/AudioMediaEncoder.java index 717c6882..30acf2f7 100644 --- a/cameraview/src/main/gles/com/otaliastudios/cameraview/AudioMediaEncoder.java +++ b/cameraview/src/main/gles/com/otaliastudios/cameraview/AudioMediaEncoder.java @@ -18,8 +18,8 @@ class AudioMediaEncoder extends MediaEncoder { } @Override - void prepare(MediaMuxer muxer) { - super.prepare(muxer); + void prepare(MediaEncoderEngine.Controller controller) { + super.prepare(controller); } @Override diff --git a/cameraview/src/main/gles/com/otaliastudios/cameraview/MediaEncoder.java b/cameraview/src/main/gles/com/otaliastudios/cameraview/MediaEncoder.java index d7c39d5e..17203ef8 100644 --- a/cameraview/src/main/gles/com/otaliastudios/cameraview/MediaEncoder.java +++ b/cameraview/src/main/gles/com/otaliastudios/cameraview/MediaEncoder.java @@ -18,21 +18,20 @@ abstract class MediaEncoder { protected MediaCodec mMediaCodec; private MediaCodec.BufferInfo mBufferInfo; - private MediaMuxer mMuxer; + private MediaEncoderEngine.Controller mController; private int mTrackIndex; - private boolean mTrackStarted; - MediaEncoder() { - } - - void prepare(MediaMuxer muxer) { - mMuxer = muxer; + void prepare(MediaEncoderEngine.Controller controller) { + mController = controller; mBufferInfo = new MediaCodec.BufferInfo(); } abstract void start(); + abstract void notify(String event, Object data); + abstract void stop(); + abstract void release(); /** @@ -55,25 +54,19 @@ abstract class MediaEncoder { int encoderStatus = mMediaCodec.dequeueOutputBuffer(mBufferInfo, TIMEOUT_USEC); if (encoderStatus == MediaCodec.INFO_TRY_AGAIN_LATER) { // no output available yet - if (!endOfStream) { - break; // out of while - } + if (!endOfStream) break; // out of while + } else if (encoderStatus == MediaCodec.INFO_OUTPUT_BUFFERS_CHANGED) { // not expected for an encoder encoderOutputBuffers = mMediaCodec.getOutputBuffers(); + } else if (encoderStatus == MediaCodec.INFO_OUTPUT_FORMAT_CHANGED) { // should happen before receiving buffers, and should only happen once - if (mTrackStarted) { - throw new RuntimeException("format changed twice"); - } + if (mController.isStarted()) throw new RuntimeException("format changed twice"); MediaFormat newFormat = mMediaCodec.getOutputFormat(); // now that we have the Magic Goodies, start the muxer - mTrackIndex = mMuxer.addTrack(newFormat); - // TODO this is wrong. Look at the Github project: it is a muxer wrapper. - // If you have multiple encoders this breaks. - mMuxer.start(); - mTrackStarted = true; + mTrackIndex = mController.start(newFormat); } else if (encoderStatus < 0) { Log.w("VideoMediaEncoder", "unexpected result from encoder.dequeueOutputBuffer: " + encoderStatus); // let's ignore it @@ -83,21 +76,14 @@ abstract class MediaEncoder { throw new RuntimeException("encoderOutputBuffer " + encoderStatus + " was null"); } - if ((mBufferInfo.flags & MediaCodec.BUFFER_FLAG_CODEC_CONFIG) != 0) { - // The codec config data was pulled out and fed to the muxer when we got - // the INFO_OUTPUT_FORMAT_CHANGED status. Ignore it. - mBufferInfo.size = 0; - } - - if (mBufferInfo.size != 0) { - if (!mTrackStarted) { - throw new RuntimeException("muxer hasn't started"); - } - + // Codec config means that config data was pulled out and fed to the muxer when we got + // the INFO_OUTPUT_FORMAT_CHANGED status. Ignore it. + boolean isCodecConfig = (mBufferInfo.flags & MediaCodec.BUFFER_FLAG_CODEC_CONFIG) != 0; + if (!isCodecConfig && mController.isStarted() && mBufferInfo.size != 0) { // adjust the ByteBuffer values to match BufferInfo (not needed?) encodedData.position(mBufferInfo.offset); encodedData.limit(mBufferInfo.offset + mBufferInfo.size); - mMuxer.writeSampleData(mTrackIndex, encodedData, mBufferInfo); + mController.write(mTrackIndex, encodedData, mBufferInfo); } mMediaCodec.releaseOutputBuffer(encoderStatus, false); if ((mBufferInfo.flags & MediaCodec.BUFFER_FLAG_END_OF_STREAM) != 0) { diff --git a/cameraview/src/main/gles/com/otaliastudios/cameraview/MediaEncoderEngine.java b/cameraview/src/main/gles/com/otaliastudios/cameraview/MediaEncoderEngine.java index 9ef1679f..e51513e5 100644 --- a/cameraview/src/main/gles/com/otaliastudios/cameraview/MediaEncoderEngine.java +++ b/cameraview/src/main/gles/com/otaliastudios/cameraview/MediaEncoderEngine.java @@ -1,6 +1,8 @@ package com.otaliastudios.cameraview; import android.graphics.SurfaceTexture; +import android.media.MediaCodec; +import android.media.MediaFormat; import android.media.MediaMuxer; import android.os.Build; import android.support.annotation.NonNull; @@ -9,6 +11,7 @@ import android.support.annotation.RequiresApi; import java.io.File; import java.io.IOException; +import java.nio.ByteBuffer; import java.util.ArrayList; import java.util.List; @@ -18,9 +21,13 @@ class MediaEncoderEngine { private WorkerHandler mWorker; private ArrayList mEncoders; private MediaMuxer mMediaMuxer; + private int mMediaMuxerStartCount; + private boolean mMediaMuxerStarted; + private Controller mController; MediaEncoderEngine(@NonNull File file, @NonNull VideoMediaEncoder videoEncoder, @Nullable AudioMediaEncoder audioEncoder) { mWorker = WorkerHandler.get("EncoderEngine"); + mController = new Controller(); mEncoders = new ArrayList<>(); mEncoders.add(videoEncoder); if (audioEncoder != null) { @@ -31,16 +38,44 @@ class MediaEncoderEngine { } catch (IOException e) { throw new RuntimeException(e); } + mMediaMuxerStartCount = 0; + mMediaMuxerStarted = false; mWorker.post(new Runnable() { @Override public void run() { for (MediaEncoder encoder : mEncoders) { - encoder.prepare(mMediaMuxer); + encoder.prepare(mController); } } }); } + class Controller { + + int start(MediaFormat format) { + if (mMediaMuxerStarted) { + throw new IllegalStateException("Trying to start but muxer started already"); + } + int track = mMediaMuxer.addTrack(format); + mMediaMuxerStartCount++; + if (mMediaMuxerStartCount == mEncoders.size()) { + mMediaMuxer.start(); + } + return track; + } + + boolean isStarted() { + return mMediaMuxerStarted; + } + + void write(int track, ByteBuffer encodedData, MediaCodec.BufferInfo info) { + if (!mMediaMuxerStarted) { + throw new IllegalStateException("Trying to write before muxer started"); + } + mMediaMuxer.writeSampleData(track, encodedData, info); + } + } + void start() { mWorker.post(new Runnable() { @Override @@ -81,6 +116,8 @@ class MediaEncoderEngine { mMediaMuxer.release(); mMediaMuxer = null; } + mMediaMuxerStartCount = 0; + mMediaMuxerStarted = false; } }); } diff --git a/cameraview/src/main/gles/com/otaliastudios/cameraview/TextureMediaEncoder.java b/cameraview/src/main/gles/com/otaliastudios/cameraview/TextureMediaEncoder.java index 63d341f0..f7f6449d 100644 --- a/cameraview/src/main/gles/com/otaliastudios/cameraview/TextureMediaEncoder.java +++ b/cameraview/src/main/gles/com/otaliastudios/cameraview/TextureMediaEncoder.java @@ -18,7 +18,7 @@ import java.io.IOException; @RequiresApi(api = Build.VERSION_CODES.JELLY_BEAN_MR2) class TextureMediaEncoder extends VideoMediaEncoder { - public final static String FRAME_EVENT = "frame"; + final static String FRAME_EVENT = "frame"; static class Frame { float[] transform; @@ -49,8 +49,8 @@ class TextureMediaEncoder extends VideoMediaEncoder } @Override - void prepare(MediaMuxer muxer) { - super.prepare(muxer); + void prepare(MediaEncoderEngine.Controller controller) { + super.prepare(controller); mEglCore = new EglCore(mConfig.eglContext, EglCore.FLAG_RECORDABLE); mWindow = new EglWindowSurface(mEglCore, mSurface, true); mWindow.makeCurrent(); // drawing will happen on the InputWindowSurface, which diff --git a/cameraview/src/main/gles/com/otaliastudios/cameraview/VideoMediaEncoder.java b/cameraview/src/main/gles/com/otaliastudios/cameraview/VideoMediaEncoder.java index 948ee7f8..251a08d7 100644 --- a/cameraview/src/main/gles/com/otaliastudios/cameraview/VideoMediaEncoder.java +++ b/cameraview/src/main/gles/com/otaliastudios/cameraview/VideoMediaEncoder.java @@ -43,8 +43,8 @@ abstract class VideoMediaEncoder extends Med } @Override - void prepare(MediaMuxer muxer) { - super.prepare(muxer); + void prepare(MediaEncoderEngine.Controller controller) { + super.prepare(controller); MediaFormat format = MediaFormat.createVideoFormat(mConfig.mimeType, mConfig.width, mConfig.height); // Set some properties. Failing to specify some of these can cause the MediaCodec