From 91e12d900fb6eafd0c52db9817dcab27bb9df8ba Mon Sep 17 00:00:00 2001 From: Mattia Iavarone Date: Mon, 17 Sep 2018 23:53:48 -0300 Subject: [PATCH] Audio recording --- .../cameraview/AudioMediaEncoder.java | 90 +++++++++++++++++-- .../cameraview/EncoderThread.java | 3 + .../cameraview/MediaEncoder.java | 42 +++++++++ .../cameraview/MediaEncoderEngine.java | 7 +- .../otaliastudios/cameraview/CameraView.java | 2 +- .../cameraview/SnapshotVideoRecorder.java | 10 +-- .../cameraview/GlCameraPreview.java | 3 - .../cameraview/RendererThread.java | 3 + 8 files changed, 141 insertions(+), 19 deletions(-) create mode 100644 cameraview/src/main/gles/com/otaliastudios/cameraview/EncoderThread.java create mode 100644 cameraview/src/main/views/com/otaliastudios/cameraview/RendererThread.java diff --git a/cameraview/src/main/gles/com/otaliastudios/cameraview/AudioMediaEncoder.java b/cameraview/src/main/gles/com/otaliastudios/cameraview/AudioMediaEncoder.java index 810f911c..4df2746c 100644 --- a/cameraview/src/main/gles/com/otaliastudios/cameraview/AudioMediaEncoder.java +++ b/cameraview/src/main/gles/com/otaliastudios/cameraview/AudioMediaEncoder.java @@ -1,13 +1,30 @@ package com.otaliastudios.cameraview; +import android.media.AudioFormat; +import android.media.AudioRecord; +import android.media.MediaCodec; +import android.media.MediaCodecInfo; +import android.media.MediaFormat; import android.media.MediaMuxer; +import android.media.MediaRecorder; import android.os.Build; import android.support.annotation.NonNull; import android.support.annotation.RequiresApi; +import java.io.IOException; +import java.nio.ByteBuffer; + @RequiresApi(api = Build.VERSION_CODES.JELLY_BEAN_MR2) class AudioMediaEncoder extends MediaEncoder { + private static final String MIME_TYPE = "audio/mp4a-latm"; + private static final int SAMPLE_RATE = 44100; // 44.1[KHz] is only setting guaranteed to be available on all devices. + private static final int BIT_RATE = 64000; + public static final int SAMPLES_PER_FRAME = 1024; // AAC, bytes/frame/channel + public static final int FRAMES_PER_BUFFER = 25; // AAC, frame/buffer/sec + + private final Object mLock = new Object(); + private boolean mRequestStop = false; static class Config { Config() { } @@ -21,28 +38,91 @@ class AudioMediaEncoder extends MediaEncoder { @Override void prepare(MediaEncoderEngine.Controller controller) { super.prepare(controller); + final MediaFormat audioFormat = MediaFormat.createAudioFormat(MIME_TYPE, SAMPLE_RATE, 1); + audioFormat.setInteger(MediaFormat.KEY_AAC_PROFILE, MediaCodecInfo.CodecProfileLevel.AACObjectLC); + audioFormat.setInteger(MediaFormat.KEY_CHANNEL_MASK, AudioFormat.CHANNEL_IN_MONO); + audioFormat.setInteger(MediaFormat.KEY_BIT_RATE, BIT_RATE); + audioFormat.setInteger(MediaFormat.KEY_CHANNEL_COUNT, 1); + try { + mMediaCodec = MediaCodec.createEncoderByType(MIME_TYPE); + } catch (IOException e) { + throw new RuntimeException(e); + } + mMediaCodec.configure(audioFormat, null, null, MediaCodec.CONFIGURE_FLAG_ENCODE); + mMediaCodec.start(); } @EncoderThread @Override void start() { - + mRequestStop = false; + new AudioThread().start(); } @EncoderThread @Override - void notify(String event, Object data) { - - } + void notify(String event, Object data) { } @EncoderThread @Override void stop() { - + mRequestStop = true; + try { + synchronized (mLock) { + mLock.wait(); + } + } catch (InterruptedException e) { + // Ignore + } } @Override void release() { super.release(); } + + class AudioThread extends Thread { + + private AudioRecord mAudioRecord; + + AudioThread() { + final int minBufferSize = AudioRecord.getMinBufferSize( + SAMPLE_RATE, AudioFormat.CHANNEL_IN_MONO, + AudioFormat.ENCODING_PCM_16BIT); + int bufferSize = SAMPLES_PER_FRAME * FRAMES_PER_BUFFER; + if (bufferSize < minBufferSize) { + bufferSize = ((minBufferSize / SAMPLES_PER_FRAME) + 1) * SAMPLES_PER_FRAME * 2; + } + mAudioRecord = new AudioRecord( + MediaRecorder.AudioSource.CAMCORDER, SAMPLE_RATE, + AudioFormat.CHANNEL_IN_MONO, AudioFormat.ENCODING_PCM_16BIT, bufferSize); + } + + @Override + public void run() { + super.run(); + mAudioRecord.startRecording(); + final ByteBuffer buffer = ByteBuffer.allocateDirect(SAMPLES_PER_FRAME); + int readBytes; + while (!mRequestStop) { + buffer.clear(); + readBytes = mAudioRecord.read(buffer, SAMPLES_PER_FRAME); + if (readBytes > 0) { + // set audio data to encoder + buffer.position(readBytes); + buffer.flip(); + encode(buffer, readBytes, getPresentationTime()); + drain(false); + } + } + // This will signal the endOfStream. + // Can't use drain(true); it is only available when writing to the codec InputSurface. + encode(null, 0, getPresentationTime()); + drain(false); + mAudioRecord.stop(); + synchronized (mLock) { + mLock.notify(); + } + } + } } diff --git a/cameraview/src/main/gles/com/otaliastudios/cameraview/EncoderThread.java b/cameraview/src/main/gles/com/otaliastudios/cameraview/EncoderThread.java new file mode 100644 index 00000000..3ccfcc51 --- /dev/null +++ b/cameraview/src/main/gles/com/otaliastudios/cameraview/EncoderThread.java @@ -0,0 +1,3 @@ +package com.otaliastudios.cameraview; + +@interface EncoderThread {} diff --git a/cameraview/src/main/gles/com/otaliastudios/cameraview/MediaEncoder.java b/cameraview/src/main/gles/com/otaliastudios/cameraview/MediaEncoder.java index bb5ca853..4d5b49f3 100644 --- a/cameraview/src/main/gles/com/otaliastudios/cameraview/MediaEncoder.java +++ b/cameraview/src/main/gles/com/otaliastudios/cameraview/MediaEncoder.java @@ -72,6 +72,35 @@ abstract class MediaEncoder { } } + /** + * Encode data into the {@link #mMediaCodec}. + */ + protected void encode(final ByteBuffer buffer, final int length, final long presentationTimeUs) { + final ByteBuffer[] inputBuffers = mMediaCodec.getInputBuffers(); + while (true) { // TODO: stop if stop() is called! + final int inputBufferIndex = mMediaCodec.dequeueInputBuffer(TIMEOUT_USEC); + if (inputBufferIndex >= 0) { + final ByteBuffer inputBuffer = inputBuffers[inputBufferIndex]; + inputBuffer.clear(); + if (buffer != null) { + inputBuffer.put(buffer); + } + if (length <= 0) { // send EOS + mMediaCodec.queueInputBuffer(inputBufferIndex, 0, 0, + presentationTimeUs, MediaCodec.BUFFER_FLAG_END_OF_STREAM); + } else { + mMediaCodec.queueInputBuffer(inputBufferIndex, 0, length, + presentationTimeUs, 0); + } + break; + } else if (inputBufferIndex == MediaCodec.INFO_TRY_AGAIN_LATER) { + // wait for MediaCodec encoder is ready to encode + // nothing to do here because MediaCodec#dequeueInputBuffer(TIMEOUT_USEC) + // will wait for maximum TIMEOUT_USEC(10msec) on each call + } + } + } + /** * Extracts all pending data that was written and encoded into {@link #mMediaCodec}, * and forwards it to the muxer. @@ -120,6 +149,7 @@ abstract class MediaEncoder { encodedData.position(mBufferInfo.offset); encodedData.limit(mBufferInfo.offset + mBufferInfo.size); mController.write(mTrackIndex, encodedData, mBufferInfo); + mPreviousTime = mBufferInfo.presentationTimeUs; } mMediaCodec.releaseOutputBuffer(encoderStatus, false); if ((mBufferInfo.flags & MediaCodec.BUFFER_FLAG_END_OF_STREAM) != 0) { @@ -131,4 +161,16 @@ abstract class MediaEncoder { } } } + + private long mPreviousTime = 0; + + protected long getPresentationTime() { + long result = System.nanoTime() / 1000L; + // presentationTimeUs should be monotonic + // otherwise muxer fail to write + if (result < mPreviousTime) { + result = (mPreviousTime - result) + result; + } + return result; + } } diff --git a/cameraview/src/main/gles/com/otaliastudios/cameraview/MediaEncoderEngine.java b/cameraview/src/main/gles/com/otaliastudios/cameraview/MediaEncoderEngine.java index 0da3e63d..5f75a0e3 100644 --- a/cameraview/src/main/gles/com/otaliastudios/cameraview/MediaEncoderEngine.java +++ b/cameraview/src/main/gles/com/otaliastudios/cameraview/MediaEncoderEngine.java @@ -8,6 +8,7 @@ import android.os.Build; import android.support.annotation.NonNull; import android.support.annotation.Nullable; import android.support.annotation.RequiresApi; +import android.util.Log; import java.io.File; import java.io.IOException; @@ -15,8 +16,6 @@ import java.nio.ByteBuffer; import java.util.ArrayList; import java.util.List; -@interface EncoderThread {} - @RequiresApi(api = Build.VERSION_CODES.JELLY_BEAN_MR2) class MediaEncoderEngine { @@ -115,9 +114,7 @@ class MediaEncoderEngine { if (mMediaMuxer != null) { // stop() throws an exception if you haven't fed it any data. // We can just swallow I think. - try { - mMediaMuxer.stop(); - } catch (Exception e) {}; + mMediaMuxer.stop(); mMediaMuxer.release(); mMediaMuxer = null; } diff --git a/cameraview/src/main/java/com/otaliastudios/cameraview/CameraView.java b/cameraview/src/main/java/com/otaliastudios/cameraview/CameraView.java index 3294876d..e8321383 100644 --- a/cameraview/src/main/java/com/otaliastudios/cameraview/CameraView.java +++ b/cameraview/src/main/java/com/otaliastudios/cameraview/CameraView.java @@ -616,7 +616,7 @@ public class CameraView extends FrameLayout implements LifecycleObserver { Context c = getContext(); boolean needsCamera = true; - boolean needsAudio = mode == Mode.VIDEO && audio == Audio.ON; + boolean needsAudio = audio == Audio.ON; needsCamera = needsCamera && c.checkSelfPermission(Manifest.permission.CAMERA) != PackageManager.PERMISSION_GRANTED; needsAudio = needsAudio && c.checkSelfPermission(Manifest.permission.RECORD_AUDIO) != PackageManager.PERMISSION_GRANTED; diff --git a/cameraview/src/main/java/com/otaliastudios/cameraview/SnapshotVideoRecorder.java b/cameraview/src/main/java/com/otaliastudios/cameraview/SnapshotVideoRecorder.java index b4022cb0..a368e92a 100644 --- a/cameraview/src/main/java/com/otaliastudios/cameraview/SnapshotVideoRecorder.java +++ b/cameraview/src/main/java/com/otaliastudios/cameraview/SnapshotVideoRecorder.java @@ -8,10 +8,6 @@ import android.support.annotation.RequiresApi; /** * A {@link VideoRecorder} that uses {@link android.media.MediaCodec} APIs. - * - * TODO rotation support. Currently we pass the wrong size - * TODO audio support - * TODO when cropping is huge, the first frame of the video result, noticeably, has no transformation applied. Don't know why. */ @RequiresApi(api = Build.VERSION_CODES.JELLY_BEAN_MR2) class SnapshotVideoRecorder extends VideoRecorder implements GlCameraPreview.RendererFrameCallback { @@ -87,7 +83,11 @@ class SnapshotVideoRecorder extends VideoRecorder implements GlCameraPreview.Ren EGL14.eglGetCurrentContext() ); TextureMediaEncoder videoEncoder = new TextureMediaEncoder(config); - mEncoderEngine = new MediaEncoderEngine(mResult.file, videoEncoder, null); + AudioMediaEncoder audioEncoder = null; + if (mResult.audio == Audio.ON) { + audioEncoder = new AudioMediaEncoder(new AudioMediaEncoder.Config()); + } + mEncoderEngine = new MediaEncoderEngine(mResult.file, videoEncoder, audioEncoder); mEncoderEngine.start(); mResult.rotation = 0; // We will rotate the result instead. mCurrentState = STATE_RECORDING; diff --git a/cameraview/src/main/views/com/otaliastudios/cameraview/GlCameraPreview.java b/cameraview/src/main/views/com/otaliastudios/cameraview/GlCameraPreview.java index 3765bec3..c68f5976 100644 --- a/cameraview/src/main/views/com/otaliastudios/cameraview/GlCameraPreview.java +++ b/cameraview/src/main/views/com/otaliastudios/cameraview/GlCameraPreview.java @@ -21,9 +21,6 @@ import javax.microedition.khronos.egl.EGLDisplay; import javax.microedition.khronos.egl.EGLSurface; import javax.microedition.khronos.opengles.GL10; - -@interface RendererThread {} - /** * - The android camera will stream image to the given {@link SurfaceTexture}. * diff --git a/cameraview/src/main/views/com/otaliastudios/cameraview/RendererThread.java b/cameraview/src/main/views/com/otaliastudios/cameraview/RendererThread.java new file mode 100644 index 00000000..adb78e91 --- /dev/null +++ b/cameraview/src/main/views/com/otaliastudios/cameraview/RendererThread.java @@ -0,0 +1,3 @@ +package com.otaliastudios.cameraview; + +@interface RendererThread {}