From 08ce79466f5640b7d25053c9bc566411a4972977 Mon Sep 17 00:00:00 2001 From: Mattia Iavarone Date: Thu, 24 Jan 2019 22:59:38 +0100 Subject: [PATCH] Timestamp changes --- .../cameraview/AudioMediaEncoder.java | 17 +++++++++-- .../cameraview/EglBaseSurface.java | 1 + .../com/otaliastudios/cameraview/EglCore.java | 1 + .../cameraview/MediaEncoder.java | 28 +++++++------------ .../cameraview/MediaEncoderEngine.java | 9 +++++- .../cameraview/TextureMediaEncoder.java | 9 ++++-- 6 files changed, 42 insertions(+), 23 deletions(-) diff --git a/cameraview/src/main/gles/com/otaliastudios/cameraview/AudioMediaEncoder.java b/cameraview/src/main/gles/com/otaliastudios/cameraview/AudioMediaEncoder.java index 58cae799..83e6ae9c 100644 --- a/cameraview/src/main/gles/com/otaliastudios/cameraview/AudioMediaEncoder.java +++ b/cameraview/src/main/gles/com/otaliastudios/cameraview/AudioMediaEncoder.java @@ -115,13 +115,13 @@ class AudioMediaEncoder extends MediaEncoder { // set audio data to encoder buffer.position(readBytes); buffer.flip(); - encode(buffer, readBytes, getPresentationTime()); + encode(buffer, readBytes, getInputPresentationTimeUs()); drain(false); } } // This will signal the endOfStream. // Can't use drain(true); it is only available when writing to the codec InputSurface. - encode(null, 0, getPresentationTime()); + encode(null, 0, getInputPresentationTimeUs()); drain(false); mAudioRecord.stop(); mAudioRecord.release(); @@ -136,4 +136,17 @@ class AudioMediaEncoder extends MediaEncoder { int getBitRate() { return mConfig.bitRate; } + + private long mLastInputPresentationTimeUs = Long.MIN_VALUE; + + // presentationTimeUs should be monotonic + // otherwise muxer fail to write + private long getInputPresentationTimeUs() { + long result = System.nanoTime() / 1000L; + if (result < mLastInputPresentationTimeUs) { + result = (mLastInputPresentationTimeUs - result) + result; + } + mLastInputPresentationTimeUs = result; + return result; + } } diff --git a/cameraview/src/main/gles/com/otaliastudios/cameraview/EglBaseSurface.java b/cameraview/src/main/gles/com/otaliastudios/cameraview/EglBaseSurface.java index ea842618..4e3393e2 100644 --- a/cameraview/src/main/gles/com/otaliastudios/cameraview/EglBaseSurface.java +++ b/cameraview/src/main/gles/com/otaliastudios/cameraview/EglBaseSurface.java @@ -151,6 +151,7 @@ class EglBaseSurface extends EglElement { /** * Sends the presentation time stamp to EGL. + * https://www.khronos.org/registry/EGL/extensions/ANDROID/EGL_ANDROID_presentation_time.txt * * @param nsecs Timestamp, in nanoseconds. */ diff --git a/cameraview/src/main/gles/com/otaliastudios/cameraview/EglCore.java b/cameraview/src/main/gles/com/otaliastudios/cameraview/EglCore.java index e0691b19..2483978d 100644 --- a/cameraview/src/main/gles/com/otaliastudios/cameraview/EglCore.java +++ b/cameraview/src/main/gles/com/otaliastudios/cameraview/EglCore.java @@ -314,6 +314,7 @@ final class EglCore { /** * Sends the presentation time stamp to EGL. Time is expressed in nanoseconds. + * https://www.khronos.org/registry/EGL/extensions/ANDROID/EGL_ANDROID_presentation_time.txt */ public void setPresentationTime(EGLSurface eglSurface, long nsecs) { EGLExt.eglPresentationTimeANDROID(mEGLDisplay, eglSurface, nsecs); diff --git a/cameraview/src/main/gles/com/otaliastudios/cameraview/MediaEncoder.java b/cameraview/src/main/gles/com/otaliastudios/cameraview/MediaEncoder.java index b1b0e8bc..9c3a60c5 100644 --- a/cameraview/src/main/gles/com/otaliastudios/cameraview/MediaEncoder.java +++ b/cameraview/src/main/gles/com/otaliastudios/cameraview/MediaEncoder.java @@ -155,17 +155,19 @@ abstract class MediaEncoder { // adjust the ByteBuffer values to match BufferInfo (not needed?) encodedData.position(mBufferInfo.offset); encodedData.limit(mBufferInfo.offset + mBufferInfo.size); - mController.write(mTrackIndex, encodedData, mBufferInfo); - mLastPresentationTime = mBufferInfo.presentationTimeUs; - if (mStartPresentationTime == 0) { - mStartPresentationTime = mLastPresentationTime; + if (mStartPresentationTimeUs == Long.MIN_VALUE) { + mStartPresentationTimeUs = mBufferInfo.presentationTimeUs; } + mBufferInfo.presentationTimeUs = mBufferInfo.presentationTimeUs - mStartPresentationTimeUs; + if (mBufferInfo.presentationTimeUs == 0) mBufferInfo.presentationTimeUs = 1; // Read somewhere in Grafika that 0 should not be used + mController.write(mTrackIndex, encodedData, mBufferInfo); + mLastPresentationTimeUs = mBufferInfo.presentationTimeUs; } mMediaCodec.releaseOutputBuffer(encoderStatus, false); if (!mMaxLengthReached) { - if (mLastPresentationTime / 1000 - mStartPresentationTime / 1000 > mMaxLengthMillis) { + if (mLastPresentationTimeUs - mStartPresentationTimeUs > mMaxLengthMillis * 1000) { mMaxLengthReached = true; - // Log.e("MediaEncoder", this.getClass().getSimpleName() + " requested stop at " + (mLastPresentationTime * 1000 * 1000)); + // Log.e("MediaEncoder", this.getClass().getSimpleName() + " requested stop at " + (mLastPresentationTimeUs * 1000 * 1000)); mController.requestStop(); break; } @@ -178,18 +180,8 @@ abstract class MediaEncoder { } } - private long mStartPresentationTime = 0; - private long mLastPresentationTime = 0; - - long getPresentationTime() { - long result = System.nanoTime() / 1000L; - // presentationTimeUs should be monotonic - // otherwise muxer fail to write - if (result < mLastPresentationTime) { - result = (mLastPresentationTime - result) + result; - } - return result; - } + private long mStartPresentationTimeUs = Long.MIN_VALUE; + private long mLastPresentationTimeUs = 0; abstract int getBitRate(); } diff --git a/cameraview/src/main/gles/com/otaliastudios/cameraview/MediaEncoderEngine.java b/cameraview/src/main/gles/com/otaliastudios/cameraview/MediaEncoderEngine.java index efb97e4c..c305db60 100644 --- a/cameraview/src/main/gles/com/otaliastudios/cameraview/MediaEncoderEngine.java +++ b/cameraview/src/main/gles/com/otaliastudios/cameraview/MediaEncoderEngine.java @@ -12,11 +12,13 @@ import java.io.File; import java.io.IOException; import java.nio.ByteBuffer; import java.util.ArrayList; +import java.util.Calendar; +import java.util.Date; @RequiresApi(api = Build.VERSION_CODES.JELLY_BEAN_MR2) class MediaEncoderEngine { - private final static String TAG = MediaEncoder.class.getSimpleName(); + private final static String TAG = MediaEncoderEngine.class.getSimpleName(); private final static CameraLogger LOG = CameraLogger.create(TAG); @SuppressWarnings("WeakerAccess") @@ -93,6 +95,7 @@ class MediaEncoderEngine { throw new IllegalStateException("Trying to start but muxer started already"); } int track = mMediaMuxer.addTrack(format); + LOG.w("Controller:", "Assigned track", track, "to format", format.getString(MediaFormat.KEY_MIME)); mMediaMuxerStartCount++; if (mMediaMuxerStartCount == mEncoders.size()) { mMediaMuxer.start(); @@ -113,6 +116,10 @@ class MediaEncoderEngine { if (!mMediaMuxerStarted) { throw new IllegalStateException("Trying to write before muxer started"); } + Calendar calendar = Calendar.getInstance(); + calendar.setTimeInMillis(info.presentationTimeUs / 1000); + LOG.e("Writing for track", track, ". Presentation:", + calendar.get(Calendar.MINUTE) + ":" + calendar.get(Calendar.SECOND) + ":" + calendar.get(Calendar.MILLISECOND)); mMediaMuxer.writeSampleData(track, encodedData, info); } diff --git a/cameraview/src/main/gles/com/otaliastudios/cameraview/TextureMediaEncoder.java b/cameraview/src/main/gles/com/otaliastudios/cameraview/TextureMediaEncoder.java index 5e72a878..9016657d 100644 --- a/cameraview/src/main/gles/com/otaliastudios/cameraview/TextureMediaEncoder.java +++ b/cameraview/src/main/gles/com/otaliastudios/cameraview/TextureMediaEncoder.java @@ -17,6 +17,9 @@ class TextureMediaEncoder extends VideoMediaEncoder static class Frame { float[] transform; + + // Nanoseconds, in no meaningful time-base. Should be for offsets only. + // Typically coming from SurfaceTexture.getTimestamp(). long timestamp; } static class Config extends VideoMediaEncoder.Config { @@ -99,14 +102,16 @@ class TextureMediaEncoder extends VideoMediaEncoder if (mFrameNum < 0) return; mFrameNum++; + // No idea what this does. Seems to leave the timestamp unaffected, but + // will leave it here just in case. int arg1 = (int) (frame.timestamp >> 32); int arg2 = (int) frame.timestamp; long timestamp = (((long) arg1) << 32) | (((long) arg2) & 0xffffffffL); - float[] transform = frame.transform; + LOG.v("Incoming frame timestamp:", timestamp); // We must scale this matrix like GlCameraPreview does, because it might have some cropping. // Scaling takes place with respect to the (0, 0, 0) point, so we must apply a Translation to compensate. - + float[] transform = frame.transform; float scaleX = mConfig.scaleX; float scaleY = mConfig.scaleY; float scaleTranslX = (1F - scaleX) / 2F;