Small changes in AudioMediaEncoder

pull/502/head
Mattia Iavarone 6 years ago
parent 04e23b861f
commit 4823d5e5ed
  1. 2
      cameraview/src/androidTest/java/com/otaliastudios/cameraview/BaseTest.java
  2. 13
      cameraview/src/androidTest/java/com/otaliastudios/cameraview/engine/CameraIntegrationTest.java
  3. 88
      cameraview/src/main/java/com/otaliastudios/cameraview/video/encoding/AudioMediaEncoder.java
  4. 8
      cameraview/src/main/java/com/otaliastudios/cameraview/video/encoding/MediaEncoder.java
  5. 12
      cameraview/src/main/java/com/otaliastudios/cameraview/video/encoding/MediaEncoderEngine.java

@ -33,8 +33,6 @@ import static org.mockito.Mockito.mock;
public class BaseTest { public class BaseTest {
public static CameraLogger LOG = CameraLogger.create("Test");
private static KeyguardManager.KeyguardLock keyguardLock; private static KeyguardManager.KeyguardLock keyguardLock;
private static PowerManager.WakeLock wakeLock; private static PowerManager.WakeLock wakeLock;

@ -10,6 +10,7 @@ import android.os.Build;
import com.otaliastudios.cameraview.BaseTest; import com.otaliastudios.cameraview.BaseTest;
import com.otaliastudios.cameraview.CameraException; import com.otaliastudios.cameraview.CameraException;
import com.otaliastudios.cameraview.CameraListener; import com.otaliastudios.cameraview.CameraListener;
import com.otaliastudios.cameraview.CameraLogger;
import com.otaliastudios.cameraview.CameraOptions; import com.otaliastudios.cameraview.CameraOptions;
import com.otaliastudios.cameraview.CameraUtils; import com.otaliastudios.cameraview.CameraUtils;
import com.otaliastudios.cameraview.CameraView; import com.otaliastudios.cameraview.CameraView;
@ -62,7 +63,9 @@ import static org.mockito.Mockito.when;
public abstract class CameraIntegrationTest extends BaseTest { public abstract class CameraIntegrationTest extends BaseTest {
private final static long DELAY = 9000; private final static CameraLogger LOG = CameraLogger.create(CameraIntegrationTest.class.getSimpleName());
private final static long DELAY = 8000;
private final static long VIDEO_DELAY = 16000;
@Rule @Rule
public ActivityTestRule<TestActivity> rule = new ActivityTestRule<>(TestActivity.class); public ActivityTestRule<TestActivity> rule = new ActivityTestRule<>(TestActivity.class);
@ -82,6 +85,7 @@ public abstract class CameraIntegrationTest extends BaseTest {
@Before @Before
public void setUp() { public void setUp() {
LOG.e("Test started. Setting up camera.");
WorkerHandler.destroy(); WorkerHandler.destroy();
ui(new Runnable() { ui(new Runnable() {
@ -120,7 +124,7 @@ public abstract class CameraIntegrationTest extends BaseTest {
@After @After
public void tearDown() { public void tearDown() {
camera.stopVideo(); LOG.e("Test ended. Tearing down camera.");
camera.destroy(); camera.destroy();
WorkerHandler.destroy(); WorkerHandler.destroy();
} }
@ -141,7 +145,6 @@ public abstract class CameraIntegrationTest extends BaseTest {
assertNotNull("Can open", result); assertNotNull("Can open", result);
// Extra wait for the bind state. // Extra wait for the bind state.
// TODO fix this and other while {} in this class in a more elegant way. // TODO fix this and other while {} in this class in a more elegant way.
//noinspection StatementWithEmptyBody
while (controller.getBindState() != CameraEngine.STATE_STARTED) {} while (controller.getBindState() != CameraEngine.STATE_STARTED) {}
} else { } else {
assertNull("Should not open", result); assertNull("Should not open", result);
@ -172,7 +175,7 @@ public abstract class CameraIntegrationTest extends BaseTest {
return argument.getReason() == CameraException.REASON_VIDEO_FAILED; return argument.getReason() == CameraException.REASON_VIDEO_FAILED;
} }
})); }));
VideoResult result = video.await(DELAY); VideoResult result = video.await(VIDEO_DELAY);
if (expectSuccess) { if (expectSuccess) {
assertNotNull("Should end video", result); assertNotNull("Should end video", result);
} else { } else {
@ -624,7 +627,7 @@ public abstract class CameraIntegrationTest extends BaseTest {
camera.takePicture(); camera.takePicture();
boolean did = latch.await(4, TimeUnit.SECONDS); boolean did = latch.await(4, TimeUnit.SECONDS);
assertFalse(did); assertFalse(did);
assertEquals(latch.getCount(), 1); assertEquals(1, latch.getCount());
} }
@SuppressWarnings("StatementWithEmptyBody") @SuppressWarnings("StatementWithEmptyBody")

@ -44,17 +44,27 @@ public class AudioMediaEncoder extends MediaEncoder {
private static final int SAMPLE_SIZE = 2; // byte/sample/channel private static final int SAMPLE_SIZE = 2; // byte/sample/channel
private static final int BYTE_RATE_PER_CHANNEL = SAMPLING_FREQUENCY * SAMPLE_SIZE; // byte/sec/channel private static final int BYTE_RATE_PER_CHANNEL = SAMPLING_FREQUENCY * SAMPLE_SIZE; // byte/sec/channel
private static final int BYTE_RATE = BYTE_RATE_PER_CHANNEL * CHANNELS_COUNT; // byte/sec private static final int BYTE_RATE = BYTE_RATE_PER_CHANNEL * CHANNELS_COUNT; // byte/sec
@SuppressWarnings("unused")
static final int BIT_RATE = BYTE_RATE * 8; // bit/sec private static final int BIT_RATE = BYTE_RATE * 8; // bit/sec
// We call FRAME here the chunk of data that we want to read at each loop cycle // We call FRAME here the chunk of data that we want to read at each loop cycle
private static final int FRAME_SIZE_PER_CHANNEL = 1024; // bytes/frame/channel [AAC constant] private static final int FRAME_SIZE_PER_CHANNEL = 1024; // bytes/frame/channel [AAC constant]
private static final int FRAME_SIZE = FRAME_SIZE_PER_CHANNEL * CHANNELS_COUNT; // bytes/frame private static final int FRAME_SIZE = FRAME_SIZE_PER_CHANNEL * CHANNELS_COUNT; // bytes/frame
// We allocate buffers of 1KB each, which is not so much. I would say that allocating // We allocate buffers of 1KB each, which is not so much. This value indicates the maximum
// at most 200 of them is a reasonable value. With the current setup, in device tests, // number of these buffers that we can allocate at a given instant.
// we manage to use 50 at most. // This value is the number of runnables that the encoder thread is allowed to be 'behind'
private static final int BUFFER_POOL_MAX_SIZE = 200; // the recorder thread. It's not safe to have it very large or we can end encoding A LOT AFTER
// the actual recording. It's better to reduce this and skip recording at all.
private static final int BUFFER_POOL_MAX_SIZE = 60;
private static long bytesToUs(int bytes) {
return (1000000L * bytes) / BYTE_RATE;
}
private static long bytesToUs(long bytes) {
return (1000000L * bytes) / BYTE_RATE;
}
private boolean mRequestStop = false; private boolean mRequestStop = false;
private AudioEncodingHandler mEncoder; private AudioEncodingHandler mEncoder;
@ -169,21 +179,21 @@ public class AudioMediaEncoder extends MediaEncoder {
private void read(boolean endOfStream) { private void read(boolean endOfStream) {
mCurrentBuffer = mByteBufferPool.get(); mCurrentBuffer = mByteBufferPool.get();
if (mCurrentBuffer == null) { if (mCurrentBuffer == null) {
LOG.e("read thread - Skipping audio frame, encoding is too slow."); LOG.e("read thread - eos:", endOfStream, "- Skipping audio frame, encoding is too slow.");
// TODO should fix the next presentation time here. // Should fix the next presentation time here, but
} else { } else {
mCurrentBuffer.clear(); mCurrentBuffer.clear();
mReadBytes = mAudioRecord.read(mCurrentBuffer, FRAME_SIZE); mReadBytes = mAudioRecord.read(mCurrentBuffer, FRAME_SIZE);
LOG.v("read thread - Read new audio frame. Bytes:", mReadBytes); LOG.i("read thread - eos:", endOfStream, "- Read new audio frame. Bytes:", mReadBytes);
if (mReadBytes > 0) { // Good read: increase PTS. if (mReadBytes > 0) { // Good read: increase PTS.
mLastTimeUs = increaseTime(mReadBytes); mLastTimeUs = increaseTime(mReadBytes);
LOG.v("read thread - Increasing PTS to", mLastTimeUs); LOG.i("read thread - eos:", endOfStream, "- Frame PTS:", mLastTimeUs);
mCurrentBuffer.limit(mReadBytes); mCurrentBuffer.limit(mReadBytes);
onBuffer(endOfStream); onBuffer(endOfStream);
} else if (mReadBytes == AudioRecord.ERROR_INVALID_OPERATION) { } else if (mReadBytes == AudioRecord.ERROR_INVALID_OPERATION) {
LOG.e("read thread - Got AudioRecord.ERROR_INVALID_OPERATION"); LOG.e("read thread - eos:", endOfStream, "- Got AudioRecord.ERROR_INVALID_OPERATION");
} else if (mReadBytes == AudioRecord.ERROR_BAD_VALUE) { } else if (mReadBytes == AudioRecord.ERROR_BAD_VALUE) {
LOG.e("read thread - Got AudioRecord.ERROR_BAD_VALUE"); LOG.e("read thread - eos:", endOfStream, "- Got AudioRecord.ERROR_BAD_VALUE");
} }
} }
} }
@ -198,14 +208,6 @@ public class AudioMediaEncoder extends MediaEncoder {
mEncoder.sendInputBuffer(mCurrentBuffer, mLastTimeUs, endOfStream); mEncoder.sendInputBuffer(mCurrentBuffer, mLastTimeUs, endOfStream);
} }
private long bytesToUs(int bytes) {
return (1000000L * bytes) / BYTE_RATE;
}
private long bytesToUs(long bytes) {
return (1000000L * bytes) / BYTE_RATE;
}
private long increaseTime(int readBytes) { private long increaseTime(int readBytes) {
return increaseTime3(readBytes); return increaseTime3(readBytes);
} }
@ -238,6 +240,10 @@ public class AudioMediaEncoder extends MediaEncoder {
/** /**
* This method looks like an improvement over {@link #increaseTime1(int)} as it * This method looks like an improvement over {@link #increaseTime1(int)} as it
* accounts for the current time as well. Adapted & improved. from Kickflip. * accounts for the current time as well. Adapted & improved. from Kickflip.
*
* This creates regular timestamps unless we accumulate a lot of delay (greater than
* twice the buffer duration), in which case it creates a gap and starts again trying
* to be regular from the new point.
*/ */
private long increaseTime3(int readBytes) { private long increaseTime3(int readBytes) {
long bufferDurationUs = bytesToUs(readBytes); long bufferDurationUs = bytesToUs(readBytes);
@ -295,10 +301,11 @@ public class AudioMediaEncoder extends MediaEncoder {
super.handleMessage(msg); super.handleMessage(msg);
boolean endOfStream = msg.what == 1; boolean endOfStream = msg.what == 1;
long timestamp = (((long) msg.arg1) << 32) | (((long) msg.arg2) & 0xffffffffL); long timestamp = (((long) msg.arg1) << 32) | (((long) msg.arg2) & 0xffffffffL);
LOG.v("encoding thread - got buffer. timestamp:", timestamp, "eos:", endOfStream); LOG.i("encoding thread - got buffer. timestamp:", timestamp, "eos:", endOfStream);
ByteBuffer buffer = (ByteBuffer) msg.obj; ByteBuffer buffer = (ByteBuffer) msg.obj;
int readBytes = buffer.remaining(); int readBytes = buffer.remaining();
InputBuffer inputBuffer = mInputBufferPool.get(); InputBuffer inputBuffer = mInputBufferPool.get();
//noinspection ConstantConditions
inputBuffer.source = buffer; inputBuffer.source = buffer;
inputBuffer.timestamp = timestamp; inputBuffer.timestamp = timestamp;
inputBuffer.length = readBytes; inputBuffer.length = readBytes;
@ -308,7 +315,7 @@ public class AudioMediaEncoder extends MediaEncoder {
} }
private void performPendingOps(boolean force) { private void performPendingOps(boolean force) {
LOG.v("encoding thread - performing", mPendingOps.size(), "pending operations."); LOG.i("encoding thread - performing", mPendingOps.size(), "pending operations. force:", force);
InputBuffer buffer; InputBuffer buffer;
while ((buffer = mPendingOps.peek()) != null) { while ((buffer = mPendingOps.peek()) != null) {
if (force) { if (force) {
@ -323,20 +330,43 @@ public class AudioMediaEncoder extends MediaEncoder {
} }
private void performPendingOp(InputBuffer buffer) { private void performPendingOp(InputBuffer buffer) {
LOG.v("encoding thread - performing pending operation for timestamp:", buffer.timestamp); LOG.i("encoding thread - performing pending operation for timestamp:", buffer.timestamp, "- encoding.");
buffer.data.put(buffer.source); buffer.data.put(buffer.source); // TODO this copy is prob. the worst part here for performance
mByteBufferPool.recycle(buffer.source); mByteBufferPool.recycle(buffer.source);
mPendingOps.remove(buffer); mPendingOps.remove(buffer);
LOG.v("encoding thread - performing pending operation for timestamp:", buffer.timestamp, "- encoding.");
encodeInputBuffer(buffer); encodeInputBuffer(buffer);
boolean eos = buffer.isEndOfStream; boolean eos = buffer.isEndOfStream;
mInputBufferPool.recycle(buffer); mInputBufferPool.recycle(buffer);
LOG.v("encoding thread - performing pending operation for timestamp:", buffer.timestamp, "- draining."); if (eos) mInputBufferPool.clear();
LOG.i("encoding thread - performing pending operation for timestamp:", buffer.timestamp, "- draining.");
// NOTE: can consider calling this drainOutput on yet another thread, which would let us
// use an even smaller BUFFER_POOL_MAX_SIZE without losing audio frames. But this way
// we can accumulate delay on this new thread without noticing (no pool getting empty).
if (true) {
drainOutput(eos); drainOutput(eos);
if (eos) { if (eos) WorkerHandler.get("AudioEncodingHandler").getThread().interrupt();
mInputBufferPool.clear(); } else {
WorkerHandler.get("AudioEncodingHandler").getThread().interrupt(); // Testing the option above.
WorkerHandler.get("AudioEncodingDrainer").remove(drainRunnable);
WorkerHandler.get("AudioEncodingDrainer").remove(drainRunnableEos);
WorkerHandler.get("AudioEncodingDrainer").post(eos ? drainRunnableEos : drainRunnable);
}
}
private final Runnable drainRunnable = new Runnable() {
@Override
public void run() {
drainOutput(false);
} }
};
private final Runnable drainRunnableEos = new Runnable() {
@Override
public void run() {
drainOutput(true);
WorkerHandler.get("AudioEncodingHandler").getThread().interrupt();
WorkerHandler.get("AudioEncodingDrainer").getThread().interrupt();
} }
};
} }
} }

@ -108,11 +108,11 @@ abstract class MediaEncoder {
* @param data object * @param data object
*/ */
final void notify(final @NonNull String event, final @Nullable Object data) { final void notify(final @NonNull String event, final @Nullable Object data) {
LOG.i(getName(), "Notify was called. Posting."); LOG.v(getName(), "Notify was called. Posting.");
mWorker.post(new Runnable() { mWorker.post(new Runnable() {
@Override @Override
public void run() { public void run() {
LOG.i(getName(), "Notify was called. Executing."); LOG.v(getName(), "Notify was called. Executing.");
onEvent(event, data); onEvent(event, data);
} }
}); });
@ -321,9 +321,7 @@ abstract class MediaEncoder {
// TODO fix the mBufferInfo being the same, then implement delayed writing in Controller // TODO fix the mBufferInfo being the same, then implement delayed writing in Controller
// and remove the isStarted() check here. // and remove the isStarted() check here.
OutputBuffer buffer = mOutputBufferPool.get(); OutputBuffer buffer = mOutputBufferPool.get();
if (buffer == null) { //noinspection ConstantConditions
throw new IllegalStateException("buffer is null!");
}
buffer.info = mBufferInfo; buffer.info = mBufferInfo;
buffer.trackIndex = mTrackIndex; buffer.trackIndex = mTrackIndex;
buffer.data = encodedData; buffer.data = encodedData;

@ -149,7 +149,7 @@ public class MediaEncoderEngine {
/** /**
* Asks encoders to stop. This is not sync, of course we will ask for encoders * Asks encoders to stop. This is not sync, of course we will ask for encoders
* to call {@link Controller#notifyReleased(int)} before actually stop the muxer. * to call {@link Controller#notifyReleased(int)} before actually stop the muxer.
* When all encoders request a release, {@link #release()} is called to do cleanup * When all encoders request a release, {@link #end()} is called to do cleanup
* and notify the listener. * and notify the listener.
*/ */
public final void stop() { public final void stop() {
@ -163,8 +163,8 @@ public class MediaEncoderEngine {
* Called after all encoders have requested a release using {@link Controller#notifyReleased(int)}. * Called after all encoders have requested a release using {@link Controller#notifyReleased(int)}.
* At this point we will do cleanup and notify the listener. * At this point we will do cleanup and notify the listener.
*/ */
private void release() { private void end() {
LOG.i("release:", "Releasing muxer after all encoders have been released."); LOG.i("end:", "Releasing muxer after all encoders have been released.");
Exception error = null; Exception error = null;
if (mMediaMuxer != null) { if (mMediaMuxer != null) {
// stop() throws an exception if you haven't fed it any data. // stop() throws an exception if you haven't fed it any data.
@ -178,7 +178,7 @@ public class MediaEncoderEngine {
} }
mMediaMuxer = null; mMediaMuxer = null;
} }
LOG.w("release:", "Dispatching end to listener - reason:", mEndReason, "error:", error); LOG.w("end:", "Dispatching end to listener - reason:", mEndReason, "error:", error);
if (mListener != null) { if (mListener != null) {
mListener.onEncodingEnd(mEndReason, error); mListener.onEncodingEnd(mEndReason, error);
mListener = null; mListener = null;
@ -187,7 +187,7 @@ public class MediaEncoderEngine {
mStartedEncodersCount = 0; mStartedEncodersCount = 0;
mReleasedEncodersCount = 0; mReleasedEncodersCount = 0;
mMediaMuxerStarted = false; mMediaMuxerStarted = false;
LOG.i("release:", "Completed."); LOG.i("end:", "Completed.");
} }
/** /**
@ -299,7 +299,7 @@ public class MediaEncoderEngine {
LOG.w("notifyReleased:", "Called for track", track); LOG.w("notifyReleased:", "Called for track", track);
if (++mReleasedEncodersCount == mEncoders.size()) { if (++mReleasedEncodersCount == mEncoders.size()) {
LOG.w("requestStop:", "All encoders have been released. Stopping the muxer."); LOG.w("requestStop:", "All encoders have been released. Stopping the muxer.");
release(); end();
} }
} }
} }

Loading…
Cancel
Save