Add timestampUs in Filter.draw signature

pull/588/head
Mattia Iavarone 6 years ago
parent f3703ca9c4
commit b063d592f5
  1. 9
      cameraview/src/androidTest/java/com/otaliastudios/cameraview/filter/BaseFilterTest.java
  2. 17
      cameraview/src/androidTest/java/com/otaliastudios/cameraview/filter/MultiFilterTest.java
  3. 3
      cameraview/src/androidTest/java/com/otaliastudios/cameraview/overlay/OverlayDrawerTest.java
  4. 18
      cameraview/src/main/java/com/otaliastudios/cameraview/filter/BaseFilter.java
  5. 3
      cameraview/src/main/java/com/otaliastudios/cameraview/filter/Filter.java
  6. 6
      cameraview/src/main/java/com/otaliastudios/cameraview/filter/MultiFilter.java
  7. 4
      cameraview/src/main/java/com/otaliastudios/cameraview/internal/egl/EglViewport.java
  8. 8
      cameraview/src/main/java/com/otaliastudios/cameraview/overlay/OverlayDrawer.java
  9. 7
      cameraview/src/main/java/com/otaliastudios/cameraview/picture/SnapshotGlPictureRecorder.java
  10. 3
      cameraview/src/main/java/com/otaliastudios/cameraview/preview/GlCameraPreview.java
  11. 4
      cameraview/src/main/java/com/otaliastudios/cameraview/video/encoding/TextureMediaEncoder.java

@ -8,21 +8,16 @@ import androidx.test.ext.junit.runners.AndroidJUnit4;
import androidx.test.filters.SmallTest; import androidx.test.filters.SmallTest;
import com.otaliastudios.cameraview.BaseEglTest; import com.otaliastudios.cameraview.BaseEglTest;
import com.otaliastudios.cameraview.BaseTest;
import com.otaliastudios.cameraview.internal.GlUtils; import com.otaliastudios.cameraview.internal.GlUtils;
import com.otaliastudios.cameraview.internal.egl.EglViewport; import com.otaliastudios.cameraview.internal.egl.EglViewport;
import org.junit.Before;
import org.junit.Test; import org.junit.Test;
import org.junit.runner.RunWith; import org.junit.runner.RunWith;
import java.util.HashSet;
import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertNotNull; import static org.junit.Assert.assertNotNull;
import static org.junit.Assert.assertTrue; import static org.junit.Assert.assertTrue;
import static org.mockito.ArgumentMatchers.any; import static org.mockito.ArgumentMatchers.any;
import static org.mockito.ArgumentMatchers.anyInt;
import static org.mockito.Mockito.never; import static org.mockito.Mockito.never;
import static org.mockito.Mockito.spy; import static org.mockito.Mockito.spy;
import static org.mockito.Mockito.times; import static org.mockito.Mockito.times;
@ -108,7 +103,7 @@ public class BaseFilterTest extends BaseEglTest {
public void testDraw_whenInvalid() { public void testDraw_whenInvalid() {
filter = spy(new TestFilter()); filter = spy(new TestFilter());
float[] matrix = new float[16]; float[] matrix = new float[16];
filter.draw(matrix); filter.draw(0L, matrix);
verify(filter, never()).onPreDraw(matrix); verify(filter, never()).onPreDraw(matrix);
verify(filter, never()).onDraw(); verify(filter, never()).onDraw();
verify(filter, never()).onPostDraw(); verify(filter, never()).onPostDraw();
@ -122,7 +117,7 @@ public class BaseFilterTest extends BaseEglTest {
int texture = viewport.createTexture(); int texture = viewport.createTexture();
float[] matrix = new float[16]; float[] matrix = new float[16];
viewport.drawFrame(texture, matrix); viewport.drawFrame(0L, texture, matrix);
verify(filter, times(1)).onPreDraw(matrix); verify(filter, times(1)).onPreDraw(matrix);
verify(filter, times(1)).onDraw(); verify(filter, times(1)).onDraw();
verify(filter, times(1)).onPostDraw(); verify(filter, times(1)).onPostDraw();

@ -3,7 +3,6 @@ package com.otaliastudios.cameraview.filter;
import android.opengl.GLES20; import android.opengl.GLES20;
import androidx.annotation.NonNull;
import androidx.test.ext.junit.runners.AndroidJUnit4; import androidx.test.ext.junit.runners.AndroidJUnit4;
import androidx.test.filters.SmallTest; import androidx.test.filters.SmallTest;
@ -30,13 +29,13 @@ import static org.junit.Assert.assertNull;
import static org.junit.Assert.assertTrue; import static org.junit.Assert.assertTrue;
import static org.mockito.ArgumentMatchers.any; import static org.mockito.ArgumentMatchers.any;
import static org.mockito.ArgumentMatchers.anyInt; import static org.mockito.ArgumentMatchers.anyInt;
import static org.mockito.ArgumentMatchers.eq;
import static org.mockito.Mockito.atLeastOnce; import static org.mockito.Mockito.atLeastOnce;
import static org.mockito.Mockito.doAnswer; import static org.mockito.Mockito.doAnswer;
import static org.mockito.Mockito.never; import static org.mockito.Mockito.never;
import static org.mockito.Mockito.spy; import static org.mockito.Mockito.spy;
import static org.mockito.Mockito.times; import static org.mockito.Mockito.times;
import static org.mockito.Mockito.verify; import static org.mockito.Mockito.verify;
import static org.mockito.Mockito.when;
@RunWith(AndroidJUnit4.class) @RunWith(AndroidJUnit4.class)
@ -146,7 +145,7 @@ public class MultiFilterTest extends BaseEglTest {
EglViewport viewport = new EglViewport(multiFilter); EglViewport viewport = new EglViewport(multiFilter);
int texture = viewport.createTexture(); int texture = viewport.createTexture();
float[] matrix = new float[16]; float[] matrix = new float[16];
viewport.drawFrame(texture, matrix); viewport.drawFrame(0L, texture, matrix);
viewport.release(); viewport.release();
// The child should have experienced the whole lifecycle. // The child should have experienced the whole lifecycle.
@ -154,7 +153,7 @@ public class MultiFilterTest extends BaseEglTest {
verify(filter, atLeastOnce()).getFragmentShader(); verify(filter, atLeastOnce()).getFragmentShader();
verify(filter, atLeastOnce()).setSize(anyInt(), anyInt()); verify(filter, atLeastOnce()).setSize(anyInt(), anyInt());
verify(filter, times(1)).onCreate(anyInt()); verify(filter, times(1)).onCreate(anyInt());
verify(filter, times(1)).draw(matrix); verify(filter, times(1)).draw(0L, matrix);
verify(filter, times(1)).onDestroy(); verify(filter, times(1)).onDestroy();
} }
@ -181,7 +180,7 @@ public class MultiFilterTest extends BaseEglTest {
assertTrue(result[0] != 0); assertTrue(result[0] != 0);
return null; return null;
} }
}).when(filter1).draw(matrix); }).when(filter1).draw(0L, matrix);
// Note: second filter is drawn with the identity matrix! // Note: second filter is drawn with the identity matrix!
doAnswer(new Answer() { doAnswer(new Answer() {
@ -198,16 +197,16 @@ public class MultiFilterTest extends BaseEglTest {
return null; return null;
} }
}).when(filter2).draw(any(float[].class)); }).when(filter2).draw(eq(0L), any(float[].class));
EglViewport viewport = new EglViewport(multiFilter); EglViewport viewport = new EglViewport(multiFilter);
int texture = viewport.createTexture(); int texture = viewport.createTexture();
viewport.drawFrame(texture, matrix); viewport.drawFrame(0L, texture, matrix);
viewport.release(); viewport.release();
// Verify that both are drawn. // Verify that both are drawn.
verify(filter1, times(1)).draw(matrix); verify(filter1, times(1)).draw(0L, matrix);
verify(filter2, times(1)).draw(any(float[].class)); verify(filter2, times(1)).draw(eq(0L), any(float[].class));
} }
} }

@ -81,8 +81,9 @@ public class OverlayDrawerTest extends BaseEglTest {
OverlayDrawer drawer = new OverlayDrawer(mock(Overlay.class), new Size(WIDTH, HEIGHT)); OverlayDrawer drawer = new OverlayDrawer(mock(Overlay.class), new Size(WIDTH, HEIGHT));
drawer.mViewport = spy(drawer.mViewport); drawer.mViewport = spy(drawer.mViewport);
drawer.draw(Overlay.Target.PICTURE_SNAPSHOT); drawer.draw(Overlay.Target.PICTURE_SNAPSHOT);
drawer.render(); drawer.render(0L);
verify(drawer.mViewport, times(1)).drawFrame( verify(drawer.mViewport, times(1)).drawFrame(
0L,
drawer.mTextureId, drawer.mTextureId,
drawer.getTransform() drawer.getTransform()
); );

@ -128,6 +128,7 @@ public abstract class BaseFilter implements Filter {
fragmentTextureCoordinateName); fragmentTextureCoordinateName);
} }
@SuppressWarnings("WeakerAccess")
@NonNull @NonNull
protected String createDefaultFragmentShader() { protected String createDefaultFragmentShader() {
return createDefaultFragmentShader(fragmentTextureCoordinateName); return createDefaultFragmentShader(fragmentTextureCoordinateName);
@ -167,9 +168,10 @@ public abstract class BaseFilter implements Filter {
} }
@Override @Override
public void draw(float[] transformMatrix) { public void draw(long timestampUs, float[] transformMatrix) {
if (programHandle == -1) { if (programHandle == -1) {
LOG.w("Filter.draw() called after destroying the filter. This can happen rarely because of threading."); LOG.w("Filter.draw() called after destroying the filter. " +
"This can happen rarely because of threading.");
} else { } else {
onPreDraw(transformMatrix); onPreDraw(transformMatrix);
onDraw(); onDraw();
@ -179,25 +181,29 @@ public abstract class BaseFilter implements Filter {
protected void onPreDraw(float[] transformMatrix) { protected void onPreDraw(float[] transformMatrix) {
// Copy the model / view / projection matrix over. // Copy the model / view / projection matrix over.
GLES20.glUniformMatrix4fv(vertexModelViewProjectionMatrixLocation, 1, false, GlUtils.IDENTITY_MATRIX, 0); GLES20.glUniformMatrix4fv(vertexModelViewProjectionMatrixLocation, 1,
false, GlUtils.IDENTITY_MATRIX, 0);
GlUtils.checkError("glUniformMatrix4fv"); GlUtils.checkError("glUniformMatrix4fv");
// Copy the texture transformation matrix over. // Copy the texture transformation matrix over.
GLES20.glUniformMatrix4fv(vertexTranformMatrixLocation, 1, false, transformMatrix, 0); GLES20.glUniformMatrix4fv(vertexTranformMatrixLocation, 1, false,
transformMatrix, 0);
GlUtils.checkError("glUniformMatrix4fv"); GlUtils.checkError("glUniformMatrix4fv");
// Enable the "aPosition" vertex attribute. // Enable the "aPosition" vertex attribute.
// Connect vertexBuffer to "aPosition". // Connect vertexBuffer to "aPosition".
GLES20.glEnableVertexAttribArray(vertexPositionLocation); GLES20.glEnableVertexAttribArray(vertexPositionLocation);
GlUtils.checkError("glEnableVertexAttribArray: " + vertexPositionLocation); GlUtils.checkError("glEnableVertexAttribArray: " + vertexPositionLocation);
GLES20.glVertexAttribPointer(vertexPositionLocation, 2, GLES20.GL_FLOAT, false, 8, vertexPosition); GLES20.glVertexAttribPointer(vertexPositionLocation, 2, GLES20.GL_FLOAT,
false, 8, vertexPosition);
GlUtils.checkError("glVertexAttribPointer"); GlUtils.checkError("glVertexAttribPointer");
// Enable the "aTextureCoord" vertex attribute. // Enable the "aTextureCoord" vertex attribute.
// Connect texBuffer to "aTextureCoord". // Connect texBuffer to "aTextureCoord".
GLES20.glEnableVertexAttribArray(vertexTextureCoordinateLocation); GLES20.glEnableVertexAttribArray(vertexTextureCoordinateLocation);
GlUtils.checkError("glEnableVertexAttribArray"); GlUtils.checkError("glEnableVertexAttribArray");
GLES20.glVertexAttribPointer(vertexTextureCoordinateLocation, 2, GLES20.GL_FLOAT, false, 8, textureCoordinates); GLES20.glVertexAttribPointer(vertexTextureCoordinateLocation, 2, GLES20.GL_FLOAT,
false, 8, textureCoordinates);
GlUtils.checkError("glVertexAttribPointer"); GlUtils.checkError("glVertexAttribPointer");
} }

@ -67,9 +67,10 @@ public interface Filter {
* Called to render the actual texture. The given transformation matrix * Called to render the actual texture. The given transformation matrix
* should be applied. * should be applied.
* *
* @param timestampUs timestamp in microseconds
* @param transformMatrix matrix * @param transformMatrix matrix
*/ */
void draw(float[] transformMatrix); void draw(long timestampUs, float[] transformMatrix);
/** /**
* Called anytime the output size changes. * Called anytime the output size changes.

@ -230,7 +230,7 @@ public class MultiFilter implements Filter, OneParameterFilter, TwoParameterFilt
} }
@Override @Override
public void draw(float[] transformMatrix) { public void draw(long timestampUs, float[] transformMatrix) {
synchronized (lock) { synchronized (lock) {
for (int i = 0; i < filters.size(); i++) { for (int i = 0; i < filters.size(); i++) {
boolean isFirst = i == 0; boolean isFirst = i == 0;
@ -258,9 +258,9 @@ public class MultiFilter implements Filter, OneParameterFilter, TwoParameterFilt
// The first filter should apply all the transformations. Then, // The first filter should apply all the transformations. Then,
// since they are applied, we should use a no-op matrix. // since they are applied, we should use a no-op matrix.
if (isFirst) { if (isFirst) {
filter.draw(transformMatrix); filter.draw(timestampUs, transformMatrix);
} else { } else {
filter.draw(GlUtils.IDENTITY_MATRIX); filter.draw(timestampUs, GlUtils.IDENTITY_MATRIX);
} }
// Set the input for the next cycle: // Set the input for the next cycle:

@ -72,7 +72,7 @@ public class EglViewport {
mPendingFilter = filter; mPendingFilter = filter;
} }
public void drawFrame(int textureId, float[] textureMatrix) { public void drawFrame(long timestampUs, int textureId, float[] textureMatrix) {
if (mPendingFilter != null) { if (mPendingFilter != null) {
release(); release();
mFilter = mPendingFilter; mFilter = mPendingFilter;
@ -89,7 +89,7 @@ public class EglViewport {
GLES20.glBindTexture(mTextureTarget, textureId); GLES20.glBindTexture(mTextureTarget, textureId);
// Draw. // Draw.
mFilter.draw(textureMatrix); mFilter.draw(timestampUs, textureMatrix);
// Release. // Release.
GLES20.glBindTexture(mTextureTarget, 0); GLES20.glBindTexture(mTextureTarget, 0);

@ -24,7 +24,7 @@ import java.nio.Buffer;
* *
* - Provides a {@link Canvas} to be passed to the Overlay * - Provides a {@link Canvas} to be passed to the Overlay
* - Lets the overlay draw there: {@link #draw(Overlay.Target)} * - Lets the overlay draw there: {@link #draw(Overlay.Target)}
* - Renders this into the current EGL window: {@link #render()} * - Renders this into the current EGL window: {@link #render(long)}
* - Applies the {@link Issue514Workaround} the correct way * - Applies the {@link Issue514Workaround} the correct way
* *
* In the future we might want to use a different approach than {@link EglViewport}, * In the future we might want to use a different approach than {@link EglViewport},
@ -93,8 +93,10 @@ public class OverlayDrawer {
* Renders the drawn content in the current EGL surface, assuming there is one. * Renders the drawn content in the current EGL surface, assuming there is one.
* Should be called after {@link #draw(Overlay.Target)} and any {@link #getTransform()} * Should be called after {@link #draw(Overlay.Target)} and any {@link #getTransform()}
* modification. * modification.
*
* @param timestampUs frame timestamp
*/ */
public void render() { public void render(long timestampUs) {
// Enable blending // Enable blending
// Reference http://www.learnopengles.com/android-lesson-five-an-introduction-to-blending/ // Reference http://www.learnopengles.com/android-lesson-five-an-introduction-to-blending/
GLES20.glDisable(GLES20.GL_CULL_FACE); GLES20.glDisable(GLES20.GL_CULL_FACE);
@ -103,7 +105,7 @@ public class OverlayDrawer {
GLES20.glBlendFunc(GLES20.GL_SRC_ALPHA, GLES20.GL_ONE_MINUS_SRC_ALPHA); GLES20.glBlendFunc(GLES20.GL_SRC_ALPHA, GLES20.GL_ONE_MINUS_SRC_ALPHA);
synchronized (mIssue514WorkaroundLock) { synchronized (mIssue514WorkaroundLock) {
mViewport.drawFrame(mTextureId, mTransform); mViewport.drawFrame(timestampUs, mTextureId, mTransform);
} }
} }

@ -223,9 +223,10 @@ public class SnapshotGlPictureRecorder extends PictureRecorder {
} }
// 5. Draw and save // 5. Draw and save
LOG.i("takeFrame:", "timestamp:", surfaceTexture.getTimestamp()); long timestampUs = surfaceTexture.getTimestamp() / 1000L;
mViewport.drawFrame(mTextureId, mTransform); LOG.i("takeFrame:", "timestampUs:", timestampUs);
if (mHasOverlay) mOverlayDrawer.render(); mViewport.drawFrame(timestampUs, mTextureId, mTransform);
if (mHasOverlay) mOverlayDrawer.render(timestampUs);
mResult.format = PictureResult.FORMAT_JPEG; mResult.format = PictureResult.FORMAT_JPEG;
mResult.data = eglSurface.saveFrameTo(Bitmap.CompressFormat.JPEG); mResult.data = eglSurface.saveFrameTo(Bitmap.CompressFormat.JPEG);

@ -216,7 +216,8 @@ public class GlCameraPreview extends FilterCameraPreview<GLSurfaceView, SurfaceT
Matrix.translateM(mTransformMatrix, 0, translX, translY, 0); Matrix.translateM(mTransformMatrix, 0, translX, translY, 0);
Matrix.scaleM(mTransformMatrix, 0, mCropScaleX, mCropScaleY, 1); Matrix.scaleM(mTransformMatrix, 0, mCropScaleX, mCropScaleY, 1);
} }
mOutputViewport.drawFrame(mOutputTextureId, mTransformMatrix); mOutputViewport.drawFrame(mInputSurfaceTexture.getTimestamp() / 1000L,
mOutputTextureId, mTransformMatrix);
synchronized (mRendererFrameCallbacks) { synchronized (mRendererFrameCallbacks) {
// Need to synchronize when iterating the Collections.synchronizedSet // Need to synchronize when iterating the Collections.synchronizedSet
for (RendererFrameCallback callback : mRendererFrameCallbacks) { for (RendererFrameCallback callback : mRendererFrameCallbacks) {

@ -213,9 +213,9 @@ public class TextureMediaEncoder extends VideoMediaEncoder<TextureConfig> {
Matrix.rotateM(mConfig.overlayDrawer.getTransform(), 0, mConfig.overlayRotation, 0, 0, 1); Matrix.rotateM(mConfig.overlayDrawer.getTransform(), 0, mConfig.overlayRotation, 0, 0, 1);
Matrix.translateM(mConfig.overlayDrawer.getTransform(), 0, -0.5F, -0.5F, 0); Matrix.translateM(mConfig.overlayDrawer.getTransform(), 0, -0.5F, -0.5F, 0);
} }
mViewport.drawFrame(mConfig.textureId, transform); mViewport.drawFrame(frame.timestampUs(), mConfig.textureId, transform);
if (mConfig.hasOverlay()) { if (mConfig.hasOverlay()) {
mConfig.overlayDrawer.render(); mConfig.overlayDrawer.render(frame.timestampUs());
} }
mWindow.setPresentationTime(frame.timestampNanos); mWindow.setPresentationTime(frame.timestampNanos);
mWindow.swapBuffers(); mWindow.swapBuffers();

Loading…
Cancel
Save