From b063d592f5ce70c1f8ea2a32030b7e9f45d4df94 Mon Sep 17 00:00:00 2001 From: Mattia Iavarone Date: Wed, 11 Sep 2019 15:49:49 +0200 Subject: [PATCH] Add timestampUs in Filter.draw signature --- .../cameraview/filter/BaseFilterTest.java | 9 ++------- .../cameraview/filter/MultiFilterTest.java | 17 ++++++++--------- .../cameraview/overlay/OverlayDrawerTest.java | 3 ++- .../cameraview/filter/BaseFilter.java | 18 ++++++++++++------ .../cameraview/filter/Filter.java | 3 ++- .../cameraview/filter/MultiFilter.java | 6 +++--- .../cameraview/internal/egl/EglViewport.java | 4 ++-- .../cameraview/overlay/OverlayDrawer.java | 8 +++++--- .../picture/SnapshotGlPictureRecorder.java | 7 ++++--- .../cameraview/preview/GlCameraPreview.java | 3 ++- .../video/encoding/TextureMediaEncoder.java | 4 ++-- 11 files changed, 44 insertions(+), 38 deletions(-) diff --git a/cameraview/src/androidTest/java/com/otaliastudios/cameraview/filter/BaseFilterTest.java b/cameraview/src/androidTest/java/com/otaliastudios/cameraview/filter/BaseFilterTest.java index d6614f52..e67367c4 100644 --- a/cameraview/src/androidTest/java/com/otaliastudios/cameraview/filter/BaseFilterTest.java +++ b/cameraview/src/androidTest/java/com/otaliastudios/cameraview/filter/BaseFilterTest.java @@ -8,21 +8,16 @@ import androidx.test.ext.junit.runners.AndroidJUnit4; import androidx.test.filters.SmallTest; import com.otaliastudios.cameraview.BaseEglTest; -import com.otaliastudios.cameraview.BaseTest; import com.otaliastudios.cameraview.internal.GlUtils; import com.otaliastudios.cameraview.internal.egl.EglViewport; -import org.junit.Before; import org.junit.Test; import org.junit.runner.RunWith; -import java.util.HashSet; - import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertNotNull; import static org.junit.Assert.assertTrue; import static org.mockito.ArgumentMatchers.any; -import static org.mockito.ArgumentMatchers.anyInt; import static org.mockito.Mockito.never; import static org.mockito.Mockito.spy; import static org.mockito.Mockito.times; @@ -108,7 +103,7 @@ public class BaseFilterTest extends BaseEglTest { public void testDraw_whenInvalid() { filter = spy(new TestFilter()); float[] matrix = new float[16]; - filter.draw(matrix); + filter.draw(0L, matrix); verify(filter, never()).onPreDraw(matrix); verify(filter, never()).onDraw(); verify(filter, never()).onPostDraw(); @@ -122,7 +117,7 @@ public class BaseFilterTest extends BaseEglTest { int texture = viewport.createTexture(); float[] matrix = new float[16]; - viewport.drawFrame(texture, matrix); + viewport.drawFrame(0L, texture, matrix); verify(filter, times(1)).onPreDraw(matrix); verify(filter, times(1)).onDraw(); verify(filter, times(1)).onPostDraw(); diff --git a/cameraview/src/androidTest/java/com/otaliastudios/cameraview/filter/MultiFilterTest.java b/cameraview/src/androidTest/java/com/otaliastudios/cameraview/filter/MultiFilterTest.java index 40c4957b..09591425 100644 --- a/cameraview/src/androidTest/java/com/otaliastudios/cameraview/filter/MultiFilterTest.java +++ b/cameraview/src/androidTest/java/com/otaliastudios/cameraview/filter/MultiFilterTest.java @@ -3,7 +3,6 @@ package com.otaliastudios.cameraview.filter; import android.opengl.GLES20; -import androidx.annotation.NonNull; import androidx.test.ext.junit.runners.AndroidJUnit4; import androidx.test.filters.SmallTest; @@ -30,13 +29,13 @@ import static org.junit.Assert.assertNull; import static org.junit.Assert.assertTrue; import static org.mockito.ArgumentMatchers.any; import static org.mockito.ArgumentMatchers.anyInt; +import static org.mockito.ArgumentMatchers.eq; import static org.mockito.Mockito.atLeastOnce; import static org.mockito.Mockito.doAnswer; import static org.mockito.Mockito.never; import static org.mockito.Mockito.spy; import static org.mockito.Mockito.times; import static org.mockito.Mockito.verify; -import static org.mockito.Mockito.when; @RunWith(AndroidJUnit4.class) @@ -146,7 +145,7 @@ public class MultiFilterTest extends BaseEglTest { EglViewport viewport = new EglViewport(multiFilter); int texture = viewport.createTexture(); float[] matrix = new float[16]; - viewport.drawFrame(texture, matrix); + viewport.drawFrame(0L, texture, matrix); viewport.release(); // The child should have experienced the whole lifecycle. @@ -154,7 +153,7 @@ public class MultiFilterTest extends BaseEglTest { verify(filter, atLeastOnce()).getFragmentShader(); verify(filter, atLeastOnce()).setSize(anyInt(), anyInt()); verify(filter, times(1)).onCreate(anyInt()); - verify(filter, times(1)).draw(matrix); + verify(filter, times(1)).draw(0L, matrix); verify(filter, times(1)).onDestroy(); } @@ -181,7 +180,7 @@ public class MultiFilterTest extends BaseEglTest { assertTrue(result[0] != 0); return null; } - }).when(filter1).draw(matrix); + }).when(filter1).draw(0L, matrix); // Note: second filter is drawn with the identity matrix! doAnswer(new Answer() { @@ -198,16 +197,16 @@ public class MultiFilterTest extends BaseEglTest { return null; } - }).when(filter2).draw(any(float[].class)); + }).when(filter2).draw(eq(0L), any(float[].class)); EglViewport viewport = new EglViewport(multiFilter); int texture = viewport.createTexture(); - viewport.drawFrame(texture, matrix); + viewport.drawFrame(0L, texture, matrix); viewport.release(); // Verify that both are drawn. - verify(filter1, times(1)).draw(matrix); - verify(filter2, times(1)).draw(any(float[].class)); + verify(filter1, times(1)).draw(0L, matrix); + verify(filter2, times(1)).draw(eq(0L), any(float[].class)); } } diff --git a/cameraview/src/androidTest/java/com/otaliastudios/cameraview/overlay/OverlayDrawerTest.java b/cameraview/src/androidTest/java/com/otaliastudios/cameraview/overlay/OverlayDrawerTest.java index 81d69666..b763c83e 100644 --- a/cameraview/src/androidTest/java/com/otaliastudios/cameraview/overlay/OverlayDrawerTest.java +++ b/cameraview/src/androidTest/java/com/otaliastudios/cameraview/overlay/OverlayDrawerTest.java @@ -81,8 +81,9 @@ public class OverlayDrawerTest extends BaseEglTest { OverlayDrawer drawer = new OverlayDrawer(mock(Overlay.class), new Size(WIDTH, HEIGHT)); drawer.mViewport = spy(drawer.mViewport); drawer.draw(Overlay.Target.PICTURE_SNAPSHOT); - drawer.render(); + drawer.render(0L); verify(drawer.mViewport, times(1)).drawFrame( + 0L, drawer.mTextureId, drawer.getTransform() ); diff --git a/cameraview/src/main/java/com/otaliastudios/cameraview/filter/BaseFilter.java b/cameraview/src/main/java/com/otaliastudios/cameraview/filter/BaseFilter.java index b69f4392..b72b129a 100644 --- a/cameraview/src/main/java/com/otaliastudios/cameraview/filter/BaseFilter.java +++ b/cameraview/src/main/java/com/otaliastudios/cameraview/filter/BaseFilter.java @@ -128,6 +128,7 @@ public abstract class BaseFilter implements Filter { fragmentTextureCoordinateName); } + @SuppressWarnings("WeakerAccess") @NonNull protected String createDefaultFragmentShader() { return createDefaultFragmentShader(fragmentTextureCoordinateName); @@ -167,9 +168,10 @@ public abstract class BaseFilter implements Filter { } @Override - public void draw(float[] transformMatrix) { + public void draw(long timestampUs, float[] transformMatrix) { if (programHandle == -1) { - LOG.w("Filter.draw() called after destroying the filter. This can happen rarely because of threading."); + LOG.w("Filter.draw() called after destroying the filter. " + + "This can happen rarely because of threading."); } else { onPreDraw(transformMatrix); onDraw(); @@ -179,25 +181,29 @@ public abstract class BaseFilter implements Filter { protected void onPreDraw(float[] transformMatrix) { // Copy the model / view / projection matrix over. - GLES20.glUniformMatrix4fv(vertexModelViewProjectionMatrixLocation, 1, false, GlUtils.IDENTITY_MATRIX, 0); + GLES20.glUniformMatrix4fv(vertexModelViewProjectionMatrixLocation, 1, + false, GlUtils.IDENTITY_MATRIX, 0); GlUtils.checkError("glUniformMatrix4fv"); // Copy the texture transformation matrix over. - GLES20.glUniformMatrix4fv(vertexTranformMatrixLocation, 1, false, transformMatrix, 0); + GLES20.glUniformMatrix4fv(vertexTranformMatrixLocation, 1, false, + transformMatrix, 0); GlUtils.checkError("glUniformMatrix4fv"); // Enable the "aPosition" vertex attribute. // Connect vertexBuffer to "aPosition". GLES20.glEnableVertexAttribArray(vertexPositionLocation); GlUtils.checkError("glEnableVertexAttribArray: " + vertexPositionLocation); - GLES20.glVertexAttribPointer(vertexPositionLocation, 2, GLES20.GL_FLOAT, false, 8, vertexPosition); + GLES20.glVertexAttribPointer(vertexPositionLocation, 2, GLES20.GL_FLOAT, + false, 8, vertexPosition); GlUtils.checkError("glVertexAttribPointer"); // Enable the "aTextureCoord" vertex attribute. // Connect texBuffer to "aTextureCoord". GLES20.glEnableVertexAttribArray(vertexTextureCoordinateLocation); GlUtils.checkError("glEnableVertexAttribArray"); - GLES20.glVertexAttribPointer(vertexTextureCoordinateLocation, 2, GLES20.GL_FLOAT, false, 8, textureCoordinates); + GLES20.glVertexAttribPointer(vertexTextureCoordinateLocation, 2, GLES20.GL_FLOAT, + false, 8, textureCoordinates); GlUtils.checkError("glVertexAttribPointer"); } diff --git a/cameraview/src/main/java/com/otaliastudios/cameraview/filter/Filter.java b/cameraview/src/main/java/com/otaliastudios/cameraview/filter/Filter.java index 7a03f9c0..b86fee50 100644 --- a/cameraview/src/main/java/com/otaliastudios/cameraview/filter/Filter.java +++ b/cameraview/src/main/java/com/otaliastudios/cameraview/filter/Filter.java @@ -67,9 +67,10 @@ public interface Filter { * Called to render the actual texture. The given transformation matrix * should be applied. * + * @param timestampUs timestamp in microseconds * @param transformMatrix matrix */ - void draw(float[] transformMatrix); + void draw(long timestampUs, float[] transformMatrix); /** * Called anytime the output size changes. diff --git a/cameraview/src/main/java/com/otaliastudios/cameraview/filter/MultiFilter.java b/cameraview/src/main/java/com/otaliastudios/cameraview/filter/MultiFilter.java index 440b84cb..23b62ab0 100644 --- a/cameraview/src/main/java/com/otaliastudios/cameraview/filter/MultiFilter.java +++ b/cameraview/src/main/java/com/otaliastudios/cameraview/filter/MultiFilter.java @@ -230,7 +230,7 @@ public class MultiFilter implements Filter, OneParameterFilter, TwoParameterFilt } @Override - public void draw(float[] transformMatrix) { + public void draw(long timestampUs, float[] transformMatrix) { synchronized (lock) { for (int i = 0; i < filters.size(); i++) { boolean isFirst = i == 0; @@ -258,9 +258,9 @@ public class MultiFilter implements Filter, OneParameterFilter, TwoParameterFilt // The first filter should apply all the transformations. Then, // since they are applied, we should use a no-op matrix. if (isFirst) { - filter.draw(transformMatrix); + filter.draw(timestampUs, transformMatrix); } else { - filter.draw(GlUtils.IDENTITY_MATRIX); + filter.draw(timestampUs, GlUtils.IDENTITY_MATRIX); } // Set the input for the next cycle: diff --git a/cameraview/src/main/java/com/otaliastudios/cameraview/internal/egl/EglViewport.java b/cameraview/src/main/java/com/otaliastudios/cameraview/internal/egl/EglViewport.java index ad43de64..e3121a47 100644 --- a/cameraview/src/main/java/com/otaliastudios/cameraview/internal/egl/EglViewport.java +++ b/cameraview/src/main/java/com/otaliastudios/cameraview/internal/egl/EglViewport.java @@ -72,7 +72,7 @@ public class EglViewport { mPendingFilter = filter; } - public void drawFrame(int textureId, float[] textureMatrix) { + public void drawFrame(long timestampUs, int textureId, float[] textureMatrix) { if (mPendingFilter != null) { release(); mFilter = mPendingFilter; @@ -89,7 +89,7 @@ public class EglViewport { GLES20.glBindTexture(mTextureTarget, textureId); // Draw. - mFilter.draw(textureMatrix); + mFilter.draw(timestampUs, textureMatrix); // Release. GLES20.glBindTexture(mTextureTarget, 0); diff --git a/cameraview/src/main/java/com/otaliastudios/cameraview/overlay/OverlayDrawer.java b/cameraview/src/main/java/com/otaliastudios/cameraview/overlay/OverlayDrawer.java index ab9b1c2a..03b87222 100644 --- a/cameraview/src/main/java/com/otaliastudios/cameraview/overlay/OverlayDrawer.java +++ b/cameraview/src/main/java/com/otaliastudios/cameraview/overlay/OverlayDrawer.java @@ -24,7 +24,7 @@ import java.nio.Buffer; * * - Provides a {@link Canvas} to be passed to the Overlay * - Lets the overlay draw there: {@link #draw(Overlay.Target)} - * - Renders this into the current EGL window: {@link #render()} + * - Renders this into the current EGL window: {@link #render(long)} * - Applies the {@link Issue514Workaround} the correct way * * In the future we might want to use a different approach than {@link EglViewport}, @@ -93,8 +93,10 @@ public class OverlayDrawer { * Renders the drawn content in the current EGL surface, assuming there is one. * Should be called after {@link #draw(Overlay.Target)} and any {@link #getTransform()} * modification. + * + * @param timestampUs frame timestamp */ - public void render() { + public void render(long timestampUs) { // Enable blending // Reference http://www.learnopengles.com/android-lesson-five-an-introduction-to-blending/ GLES20.glDisable(GLES20.GL_CULL_FACE); @@ -103,7 +105,7 @@ public class OverlayDrawer { GLES20.glBlendFunc(GLES20.GL_SRC_ALPHA, GLES20.GL_ONE_MINUS_SRC_ALPHA); synchronized (mIssue514WorkaroundLock) { - mViewport.drawFrame(mTextureId, mTransform); + mViewport.drawFrame(timestampUs, mTextureId, mTransform); } } diff --git a/cameraview/src/main/java/com/otaliastudios/cameraview/picture/SnapshotGlPictureRecorder.java b/cameraview/src/main/java/com/otaliastudios/cameraview/picture/SnapshotGlPictureRecorder.java index b4d019e8..fc2b1a6e 100644 --- a/cameraview/src/main/java/com/otaliastudios/cameraview/picture/SnapshotGlPictureRecorder.java +++ b/cameraview/src/main/java/com/otaliastudios/cameraview/picture/SnapshotGlPictureRecorder.java @@ -223,9 +223,10 @@ public class SnapshotGlPictureRecorder extends PictureRecorder { } // 5. Draw and save - LOG.i("takeFrame:", "timestamp:", surfaceTexture.getTimestamp()); - mViewport.drawFrame(mTextureId, mTransform); - if (mHasOverlay) mOverlayDrawer.render(); + long timestampUs = surfaceTexture.getTimestamp() / 1000L; + LOG.i("takeFrame:", "timestampUs:", timestampUs); + mViewport.drawFrame(timestampUs, mTextureId, mTransform); + if (mHasOverlay) mOverlayDrawer.render(timestampUs); mResult.format = PictureResult.FORMAT_JPEG; mResult.data = eglSurface.saveFrameTo(Bitmap.CompressFormat.JPEG); diff --git a/cameraview/src/main/java/com/otaliastudios/cameraview/preview/GlCameraPreview.java b/cameraview/src/main/java/com/otaliastudios/cameraview/preview/GlCameraPreview.java index 9afc1f92..0f49afc4 100644 --- a/cameraview/src/main/java/com/otaliastudios/cameraview/preview/GlCameraPreview.java +++ b/cameraview/src/main/java/com/otaliastudios/cameraview/preview/GlCameraPreview.java @@ -216,7 +216,8 @@ public class GlCameraPreview extends FilterCameraPreview { Matrix.rotateM(mConfig.overlayDrawer.getTransform(), 0, mConfig.overlayRotation, 0, 0, 1); Matrix.translateM(mConfig.overlayDrawer.getTransform(), 0, -0.5F, -0.5F, 0); } - mViewport.drawFrame(mConfig.textureId, transform); + mViewport.drawFrame(frame.timestampUs(), mConfig.textureId, transform); if (mConfig.hasOverlay()) { - mConfig.overlayDrawer.render(); + mConfig.overlayDrawer.render(frame.timestampUs()); } mWindow.setPresentationTime(frame.timestampNanos); mWindow.swapBuffers();