Audio recording

cameraview/src/main/gles/com/otaliastudios/cameraview/AudioMediaEncoder.java

@@ -1,13 +1,30 @@
 package com.otaliastudios.cameraview;
 
+import android.media.AudioFormat;
+import android.media.AudioRecord;
+import android.media.MediaCodec;
+import android.media.MediaCodecInfo;
+import android.media.MediaFormat;
 import android.media.MediaMuxer;
+import android.media.MediaRecorder;
 import android.os.Build;
 import android.support.annotation.NonNull;
 import android.support.annotation.RequiresApi;
 
+import java.io.IOException;
+import java.nio.ByteBuffer;
+
 @RequiresApi(api = Build.VERSION_CODES.JELLY_BEAN_MR2)
 class AudioMediaEncoder extends MediaEncoder {
 
+    private static final String MIME_TYPE = "audio/mp4a-latm";
+    private static final int SAMPLE_RATE = 44100; // 44.1[KHz] is only setting guaranteed to be available on all devices.
+    private static final int BIT_RATE = 64000;
+    public static final int SAMPLES_PER_FRAME = 1024; // AAC, bytes/frame/channel
+    public static final int FRAMES_PER_BUFFER = 25; // AAC, frame/buffer/sec
+
+    private final Object mLock = new Object();
+    private boolean mRequestStop = false;
 
     static class Config {
         Config() { }
@@ -21,28 +38,91 @@ class AudioMediaEncoder extends MediaEncoder {
     @Override
     void prepare(MediaEncoderEngine.Controller controller) {
         super.prepare(controller);
+        final MediaFormat audioFormat = MediaFormat.createAudioFormat(MIME_TYPE, SAMPLE_RATE, 1);
+        audioFormat.setInteger(MediaFormat.KEY_AAC_PROFILE, MediaCodecInfo.CodecProfileLevel.AACObjectLC);
+        audioFormat.setInteger(MediaFormat.KEY_CHANNEL_MASK, AudioFormat.CHANNEL_IN_MONO);
+        audioFormat.setInteger(MediaFormat.KEY_BIT_RATE, BIT_RATE);
+        audioFormat.setInteger(MediaFormat.KEY_CHANNEL_COUNT, 1);
+        try {
+            mMediaCodec = MediaCodec.createEncoderByType(MIME_TYPE);
+        } catch (IOException e) {
+            throw new RuntimeException(e);
+        }
+        mMediaCodec.configure(audioFormat, null, null, MediaCodec.CONFIGURE_FLAG_ENCODE);
+        mMediaCodec.start();
     }
 
     @EncoderThread
     @Override
     void start() {
-
+        mRequestStop = false;
+        new AudioThread().start();
     }
 
     @EncoderThread
     @Override
-    void notify(String event, Object data) {
+    void notify(String event, Object data) { }
-
-    }
 
     @EncoderThread
     @Override
     void stop() {
-
+        mRequestStop = true;
+        try {
+            synchronized (mLock) {
+                mLock.wait();
+            }
+        } catch (InterruptedException e) {
+            // Ignore
+        }
     }
 
     @Override
     void release() {
         super.release();
     }
+
+    class AudioThread extends Thread {
+
+        private AudioRecord mAudioRecord;
+
+        AudioThread() {
+            final int minBufferSize = AudioRecord.getMinBufferSize(
+                    SAMPLE_RATE, AudioFormat.CHANNEL_IN_MONO,
+                    AudioFormat.ENCODING_PCM_16BIT);
+            int bufferSize = SAMPLES_PER_FRAME * FRAMES_PER_BUFFER;
+            if (bufferSize < minBufferSize) {
+                bufferSize = ((minBufferSize / SAMPLES_PER_FRAME) + 1) * SAMPLES_PER_FRAME * 2;
+            }
+            mAudioRecord = new AudioRecord(
+                    MediaRecorder.AudioSource.CAMCORDER, SAMPLE_RATE,
+                    AudioFormat.CHANNEL_IN_MONO, AudioFormat.ENCODING_PCM_16BIT, bufferSize);
+        }
+
+        @Override
+        public void run() {
+            super.run();
+            mAudioRecord.startRecording();
+            final ByteBuffer buffer = ByteBuffer.allocateDirect(SAMPLES_PER_FRAME);
+            int readBytes;
+            while (!mRequestStop) {
+                buffer.clear();
+                readBytes = mAudioRecord.read(buffer, SAMPLES_PER_FRAME);
+                if (readBytes > 0) {
+                    // set audio data to encoder
+                    buffer.position(readBytes);
+                    buffer.flip();
+                    encode(buffer, readBytes, getPresentationTime());
+                    drain(false);
+                }
+            }
+            // This will signal the endOfStream.
+            // Can't use drain(true); it is only available when writing to the codec InputSurface.
+            encode(null, 0, getPresentationTime());
+            drain(false);
+            mAudioRecord.stop();
+            synchronized (mLock) {
+                mLock.notify();
+            }
+        }
+    }
 }

cameraview/src/main/gles/com/otaliastudios/cameraview/EncoderThread.java

@@ -0,0 +1,3 @@
+package com.otaliastudios.cameraview;
+
+@interface EncoderThread {}

cameraview/src/main/gles/com/otaliastudios/cameraview/MediaEncoder.java

@@ -72,6 +72,35 @@ abstract class MediaEncoder {
         }
     }
 
+    /**
+     * Encode data into the {@link #mMediaCodec}.
+     */
+    protected void encode(final ByteBuffer buffer, final int length, final long presentationTimeUs) {
+        final ByteBuffer[] inputBuffers = mMediaCodec.getInputBuffers();
+        while (true) { // TODO: stop if stop() is called!
+            final int inputBufferIndex = mMediaCodec.dequeueInputBuffer(TIMEOUT_USEC);
+            if (inputBufferIndex >= 0) {
+                final ByteBuffer inputBuffer = inputBuffers[inputBufferIndex];
+                inputBuffer.clear();
+                if (buffer != null) {
+                    inputBuffer.put(buffer);
+                }
+                if (length <= 0) { // send EOS
+                    mMediaCodec.queueInputBuffer(inputBufferIndex, 0, 0,
+                            presentationTimeUs, MediaCodec.BUFFER_FLAG_END_OF_STREAM);
+                } else {
+                    mMediaCodec.queueInputBuffer(inputBufferIndex, 0, length,
+                            presentationTimeUs, 0);
+                }
+                break;
+            } else if (inputBufferIndex == MediaCodec.INFO_TRY_AGAIN_LATER) {
+                // wait for MediaCodec encoder is ready to encode
+                // nothing to do here because MediaCodec#dequeueInputBuffer(TIMEOUT_USEC)
+                // will wait for maximum TIMEOUT_USEC(10msec) on each call
+            }
+        }
+    }
+
     /**
      * Extracts all pending data that was written and encoded into {@link #mMediaCodec},
      * and forwards it to the muxer.
@@ -120,6 +149,7 @@ abstract class MediaEncoder {
                     encodedData.position(mBufferInfo.offset);
                     encodedData.limit(mBufferInfo.offset + mBufferInfo.size);
                     mController.write(mTrackIndex, encodedData, mBufferInfo);
+                    mPreviousTime = mBufferInfo.presentationTimeUs;
                 }
                 mMediaCodec.releaseOutputBuffer(encoderStatus, false);
                 if ((mBufferInfo.flags & MediaCodec.BUFFER_FLAG_END_OF_STREAM) != 0) {
@@ -131,4 +161,16 @@ abstract class MediaEncoder {
             }
         }
     }
+
+    private long mPreviousTime = 0;
+
+    protected long getPresentationTime() {
+        long result = System.nanoTime() / 1000L;
+        // presentationTimeUs should be monotonic
+        // otherwise muxer fail to write
+        if (result < mPreviousTime) {
+            result = (mPreviousTime - result) + result;
+        }
+        return result;
+    }
 }

cameraview/src/main/gles/com/otaliastudios/cameraview/MediaEncoderEngine.java

@@ -8,6 +8,7 @@ import android.os.Build;
 import android.support.annotation.NonNull;
 import android.support.annotation.Nullable;
 import android.support.annotation.RequiresApi;
+import android.util.Log;
 
 import java.io.File;
 import java.io.IOException;
@@ -15,8 +16,6 @@ import java.nio.ByteBuffer;
 import java.util.ArrayList;
 import java.util.List;
 
-@interface EncoderThread {}
-
 @RequiresApi(api = Build.VERSION_CODES.JELLY_BEAN_MR2)
 class MediaEncoderEngine {
 
@@ -115,9 +114,7 @@ class MediaEncoderEngine {
                 if (mMediaMuxer != null) {
                     // stop() throws an exception if you haven't fed it any data.
                     // We can just swallow I think.
-                    try {
                     mMediaMuxer.stop();
-                    } catch (Exception e) {};
                     mMediaMuxer.release();
                     mMediaMuxer = null;
                 }

cameraview/src/main/java/com/otaliastudios/cameraview/CameraView.java

@@ -616,7 +616,7 @@ public class CameraView extends FrameLayout implements LifecycleObserver {
 
         Context c = getContext();
         boolean needsCamera = true;
-        boolean needsAudio = mode == Mode.VIDEO && audio == Audio.ON;
+        boolean needsAudio = audio == Audio.ON;
 
         needsCamera = needsCamera && c.checkSelfPermission(Manifest.permission.CAMERA) != PackageManager.PERMISSION_GRANTED;
         needsAudio = needsAudio && c.checkSelfPermission(Manifest.permission.RECORD_AUDIO) != PackageManager.PERMISSION_GRANTED;

cameraview/src/main/java/com/otaliastudios/cameraview/SnapshotVideoRecorder.java

@@ -8,10 +8,6 @@ import android.support.annotation.RequiresApi;
 
 /**
  * A {@link VideoRecorder} that uses {@link android.media.MediaCodec} APIs.
- *
- * TODO rotation support. Currently we pass the wrong size
- * TODO audio support
- * TODO when cropping is huge, the first frame of the video result, noticeably, has no transformation applied. Don't know why.
  */
 @RequiresApi(api = Build.VERSION_CODES.JELLY_BEAN_MR2)
 class SnapshotVideoRecorder extends VideoRecorder implements GlCameraPreview.RendererFrameCallback {
@@ -87,7 +83,11 @@ class SnapshotVideoRecorder extends VideoRecorder implements GlCameraPreview.Ren
                     EGL14.eglGetCurrentContext()
             );
             TextureMediaEncoder videoEncoder = new TextureMediaEncoder(config);
-            mEncoderEngine = new MediaEncoderEngine(mResult.file, videoEncoder, null);
+            AudioMediaEncoder audioEncoder = null;
+            if (mResult.audio == Audio.ON) {
+                audioEncoder = new AudioMediaEncoder(new AudioMediaEncoder.Config());
+            }
+            mEncoderEngine = new MediaEncoderEngine(mResult.file, videoEncoder, audioEncoder);
             mEncoderEngine.start();
             mResult.rotation = 0; // We will rotate the result instead.
             mCurrentState = STATE_RECORDING;

cameraview/src/main/views/com/otaliastudios/cameraview/GlCameraPreview.java

@@ -21,9 +21,6 @@ import javax.microedition.khronos.egl.EGLDisplay;
 import javax.microedition.khronos.egl.EGLSurface;
 import javax.microedition.khronos.opengles.GL10;
 
-
-@interface RendererThread {}
-
 /**
  * - The android camera will stream image to the given {@link SurfaceTexture}.
  *

cameraview/src/main/views/com/otaliastudios/cameraview/RendererThread.java

@@ -0,0 +1,3 @@
+package com.otaliastudios.cameraview;
+
+@interface RendererThread {}