From d333348ce6d1c8e6aede4fa3f1ad9d9130241e7f Mon Sep 17 00:00:00 2001 From: Mattia Iavarone Date: Wed, 21 Aug 2019 18:23:10 +0200 Subject: [PATCH] MultiFilter to merge filters (#559) * Ensure copied filters have right size * Create MultiFilter base implementation * Working implementation with replaceAll hack * Implement OneParameterFilter and TwoParameterFilter in MultiFilter * Add docs * Add tests * Fix rotation issues * Fix tests and cleanup * Small improvements --- .../cameraview/filter/MultiFilterTest.java | 213 ++++++++++++ .../cameraview/filter/MultiFilter.java | 325 ++++++++++++++++++ .../cameraview/internal/egl/EglViewport.java | 16 +- .../picture/SnapshotGlPictureRecorder.java | 20 +- .../cameraview/preview/GlCameraPreview.java | 32 +- .../preview/RendererFrameCallback.java | 9 +- .../video/SnapshotVideoRecorder.java | 8 +- .../video/encoding/TextureMediaEncoder.java | 153 +++++---- .../cameraview/demo/CameraActivity.java | 9 + docs/_posts/2019-08-06-filters.md | 21 ++ 10 files changed, 703 insertions(+), 103 deletions(-) create mode 100644 cameraview/src/androidTest/java/com/otaliastudios/cameraview/filter/MultiFilterTest.java create mode 100644 cameraview/src/main/java/com/otaliastudios/cameraview/filter/MultiFilter.java diff --git a/cameraview/src/androidTest/java/com/otaliastudios/cameraview/filter/MultiFilterTest.java b/cameraview/src/androidTest/java/com/otaliastudios/cameraview/filter/MultiFilterTest.java new file mode 100644 index 00000000..40c4957b --- /dev/null +++ b/cameraview/src/androidTest/java/com/otaliastudios/cameraview/filter/MultiFilterTest.java @@ -0,0 +1,213 @@ +package com.otaliastudios.cameraview.filter; + + +import android.opengl.GLES20; + +import androidx.annotation.NonNull; +import androidx.test.ext.junit.runners.AndroidJUnit4; +import androidx.test.filters.SmallTest; + +import com.otaliastudios.cameraview.BaseEglTest; +import com.otaliastudios.cameraview.filters.AutoFixFilter; +import com.otaliastudios.cameraview.filters.BrightnessFilter; +import com.otaliastudios.cameraview.filters.DuotoneFilter; +import com.otaliastudios.cameraview.filters.VignetteFilter; +import com.otaliastudios.cameraview.internal.GlUtils; +import com.otaliastudios.cameraview.internal.egl.EglViewport; + +import org.junit.Test; +import org.junit.runner.RunWith; +import org.mockito.invocation.InvocationOnMock; +import org.mockito.stubbing.Answer; + +import java.util.ArrayList; +import java.util.List; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertFalse; +import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertNull; +import static org.junit.Assert.assertTrue; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.anyInt; +import static org.mockito.Mockito.atLeastOnce; +import static org.mockito.Mockito.doAnswer; +import static org.mockito.Mockito.never; +import static org.mockito.Mockito.spy; +import static org.mockito.Mockito.times; +import static org.mockito.Mockito.verify; +import static org.mockito.Mockito.when; + + +@RunWith(AndroidJUnit4.class) +@SmallTest +public class MultiFilterTest extends BaseEglTest { + + @Test + public void testConstructor1() { + MultiFilter multiFilter = new MultiFilter( + new DuotoneFilter(), + new AutoFixFilter() + ); + assertEquals(2, multiFilter.filters.size()); + } + + @Test + public void testConstructor2() { + List filters = new ArrayList<>(); + filters.add(new DuotoneFilter()); + filters.add(new AutoFixFilter()); + MultiFilter multiFilter = new MultiFilter(filters); + assertEquals(2, multiFilter.filters.size()); + } + + @Test + public void testAddFilter() { + MultiFilter multiFilter = new MultiFilter(); + assertEquals(0, multiFilter.filters.size()); + multiFilter.addFilter(new DuotoneFilter()); + assertEquals(1, multiFilter.filters.size()); + multiFilter.addFilter(new AutoFixFilter()); + assertEquals(2, multiFilter.filters.size()); + } + + @Test + public void testAddFilter_multi() { + MultiFilter multiFilter = new MultiFilter(new DuotoneFilter()); + assertEquals(1, multiFilter.filters.size()); + MultiFilter other = new MultiFilter( + new AutoFixFilter(), + new BrightnessFilter(), + new VignetteFilter()); + multiFilter.addFilter(other); + assertEquals(4, multiFilter.filters.size()); + } + + @Test + public void testSetSize() { + DuotoneFilter filter = new DuotoneFilter(); + MultiFilter multiFilter = new MultiFilter(filter); + MultiFilter.State state = multiFilter.states.get(filter); + assertNotNull(state); + assertNull(state.size); + multiFilter.setSize(WIDTH, HEIGHT); + assertNotNull(state.size); + } + + @Test + public void testCopy() { + DuotoneFilter filter = spy(new DuotoneFilter()); + MultiFilter multiFilter = new MultiFilter(filter); + MultiFilter multiFilterCopy = (MultiFilter) multiFilter.copy(); + assertEquals(1, multiFilterCopy.filters.size()); + verify(filter, times(1)).onCopy(); + } + + @Test + public void testParameter1() { + DuotoneFilter filter = new DuotoneFilter(); + MultiFilter multiFilter = new MultiFilter(filter); + float desired = 0.21582F; // whatever + multiFilter.setParameter1(desired); + assertEquals(desired, multiFilter.getParameter1(), 0.001F); + assertEquals(desired, filter.getParameter1(), 0.001F); + } + + @Test + public void testParameter2() { + DuotoneFilter filter = new DuotoneFilter(); + MultiFilter multiFilter = new MultiFilter(filter); + float desired = 0.21582F; // whatever + multiFilter.setParameter2(desired); + assertEquals(desired, multiFilter.getParameter2(), 0.001F); + assertEquals(desired, filter.getParameter2(), 0.001F); + } + + @Test + public void testOnCreate_isLazy() { + DuotoneFilter filter = spy(new DuotoneFilter()); + MultiFilter multiFilter = new MultiFilter(filter); + + int program = GlUtils.createProgram(multiFilter.getVertexShader(), + multiFilter.getFragmentShader()); + multiFilter.onCreate(program); + verify(filter, never()).onCreate(anyInt()); + + multiFilter.onDestroy(); + GLES20.glDeleteProgram(program); + verify(filter, never()).onDestroy(); + } + + @Test + public void testDraw_simple() { + DuotoneFilter filter = spy(new DuotoneFilter()); + MultiFilter multiFilter = new MultiFilter(filter); + multiFilter.setSize(WIDTH, HEIGHT); + EglViewport viewport = new EglViewport(multiFilter); + int texture = viewport.createTexture(); + float[] matrix = new float[16]; + viewport.drawFrame(texture, matrix); + viewport.release(); + + // The child should have experienced the whole lifecycle. + verify(filter, atLeastOnce()).getVertexShader(); + verify(filter, atLeastOnce()).getFragmentShader(); + verify(filter, atLeastOnce()).setSize(anyInt(), anyInt()); + verify(filter, times(1)).onCreate(anyInt()); + verify(filter, times(1)).draw(matrix); + verify(filter, times(1)).onDestroy(); + } + + @Test + public void testDraw_multi() { + // Want to test that when filter1 is drawn, the current framebuffer is a + // non-default one. When filter2 is drawn, the current framebuffer is 0. + final DuotoneFilter filter1 = spy(new DuotoneFilter()); + final DuotoneFilter filter2 = spy(new DuotoneFilter()); + final MultiFilter multiFilter = new MultiFilter(filter1, filter2); + multiFilter.setSize(WIDTH, HEIGHT); + float[] matrix = new float[16]; + final int[] result = new int[1]; + + doAnswer(new Answer() { + @Override + public Object answer(InvocationOnMock invocation) { + MultiFilter.State state = multiFilter.states.get(filter1); + assertNotNull(state); + assertTrue(state.isCreated); + assertTrue(state.isFramebufferCreated); + + GLES20.glGetIntegerv(GLES20.GL_FRAMEBUFFER_BINDING, result, 0); + assertTrue(result[0] != 0); + return null; + } + }).when(filter1).draw(matrix); + + // Note: second filter is drawn with the identity matrix! + doAnswer(new Answer() { + @Override + public Object answer(InvocationOnMock invocation) { + // The last filter has no FBO / texture. + MultiFilter.State state = multiFilter.states.get(filter2); + assertNotNull(state); + assertTrue(state.isCreated); + assertFalse(state.isFramebufferCreated); + + GLES20.glGetIntegerv(GLES20.GL_FRAMEBUFFER_BINDING, result, 0); + assertEquals(0, result[0]); + return null; + + } + }).when(filter2).draw(any(float[].class)); + + EglViewport viewport = new EglViewport(multiFilter); + int texture = viewport.createTexture(); + viewport.drawFrame(texture, matrix); + viewport.release(); + + // Verify that both are drawn. + verify(filter1, times(1)).draw(matrix); + verify(filter2, times(1)).draw(any(float[].class)); + } + +} diff --git a/cameraview/src/main/java/com/otaliastudios/cameraview/filter/MultiFilter.java b/cameraview/src/main/java/com/otaliastudios/cameraview/filter/MultiFilter.java new file mode 100644 index 00000000..440b84cb --- /dev/null +++ b/cameraview/src/main/java/com/otaliastudios/cameraview/filter/MultiFilter.java @@ -0,0 +1,325 @@ +package com.otaliastudios.cameraview.filter; + +import android.opengl.GLES11Ext; +import android.opengl.GLES20; + +import androidx.annotation.NonNull; +import androidx.annotation.VisibleForTesting; + +import com.otaliastudios.cameraview.internal.GlUtils; +import com.otaliastudios.cameraview.size.Size; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collection; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +/** + * A {@link MultiFilter} is a special {@link Filter} that can group one or more filters together. + * When this happens, filters are applied in sequence: + * - the first filter reads from input frames + * - the second filters reads the output of the first + * And so on, until the last filter which will read from the previous and write to the real output. + * + * New filters can be added at any time through {@link #addFilter(Filter)}, but currently they + * can not be removed because we can not easily ensure that they would be correctly released. + * + * The {@link MultiFilter} does also implement {@link OneParameterFilter} and {@link TwoParameterFilter}, + * dispatching all the parameter calls to child filters, assuming they support it. + * + * There are some important technical caveats when using {@link MultiFilter}: + * - each child filter requires the allocation of a GL framebuffer. Using a large number of filters + * will likely cause memory issues (e.g. https://stackoverflow.com/q/6354208/4288782). + * - some of the children need to write into {@link GLES20#GL_TEXTURE_2D} instead of + * {@link GLES11Ext#GL_TEXTURE_EXTERNAL_OES}! To achieve this, we replace samplerExternalOES + * with sampler2D in your fragment shader code. This might cause issues for some shaders. + */ +@SuppressWarnings("unused") +public class MultiFilter implements Filter, OneParameterFilter, TwoParameterFilter { + + @VisibleForTesting + static class State { + @VisibleForTesting boolean isCreated = false; + @VisibleForTesting boolean isFramebufferCreated = false; + @VisibleForTesting Size size = null; + + private int programHandle = -1; + private int framebufferId = -1; + private int textureId = -1; + } + + @VisibleForTesting final List filters = new ArrayList<>(); + @VisibleForTesting final Map states = new HashMap<>(); + private final Object lock = new Object(); + private Size size = null; + private float parameter1 = 0F; + private float parameter2 = 0F; + + /** + * Creates a new group with the given filters. + * @param filters children + */ + @SuppressWarnings("WeakerAccess") + public MultiFilter(@NonNull Filter... filters) { + this(Arrays.asList(filters)); + } + + /** + * Creates a new group with the given filters. + * @param filters children + */ + @SuppressWarnings("WeakerAccess") + public MultiFilter(@NonNull Collection filters) { + for (Filter filter : filters) { + addFilter(filter); + } + } + + /** + * Adds a new filter. It will be used in the next frame. + * If the filter is a {@link MultiFilter}, we'll use its children instead. + * + * @param filter a new filter + */ + @SuppressWarnings("WeakerAccess") + public void addFilter(@NonNull Filter filter) { + if (filter instanceof MultiFilter) { + MultiFilter multiFilter = (MultiFilter) filter; + for (Filter multiChild : multiFilter.filters) { + addFilter(multiChild); + } + return; + } + synchronized (lock) { + if (!filters.contains(filter)) { + filters.add(filter); + states.put(filter, new State()); + } + } + } + + // We don't offer a removeFilter method since that would cause issues + // with cleanup. Cleanup must happen on the GL thread so we'd have to wait + // for new rendering call (which might not even happen). + + private void maybeCreate(@NonNull Filter filter, boolean isFirst) { + State state = states.get(filter); + //noinspection ConstantConditions + if (!state.isCreated) { + state.isCreated = true; + String shader = filter.getFragmentShader(); + if (!isFirst) { + // The first shader actually reads from a OES texture, but the others + // will read from the 2d framebuffer texture. This is a dirty hack. + shader = shader.replace("samplerExternalOES ", "sampler2D "); + } + state.programHandle = GlUtils.createProgram(filter.getVertexShader(), shader); + filter.onCreate(state.programHandle); + } + } + + private void maybeDestroy(@NonNull Filter filter) { + State state = states.get(filter); + //noinspection ConstantConditions + if (state.isCreated) { + state.isCreated = false; + filter.onDestroy(); + GLES20.glDeleteProgram(state.programHandle); + state.programHandle = -1; + } + } + + private void maybeCreateFramebuffer(@NonNull Filter filter, boolean isLast) { + if (isLast) return; + State state = states.get(filter); + //noinspection ConstantConditions + if (!state.isFramebufferCreated) { + state.isFramebufferCreated = true; + + int[] framebufferArray = new int[1]; + int[] textureArray = new int[1]; + GLES20.glGenFramebuffers(1, framebufferArray, 0); + GLES20.glGenTextures(1, textureArray, 0); + state.framebufferId = framebufferArray[0]; + state.textureId = textureArray[0]; + + GLES20.glBindTexture(GLES20.GL_TEXTURE_2D, state.textureId); + GLES20.glTexImage2D(GLES20.GL_TEXTURE_2D, 0, GLES20.GL_RGBA, state.size.getWidth(), state.size.getHeight(), 0, GLES20.GL_RGBA, GLES20.GL_UNSIGNED_BYTE, null); + GLES20.glTexParameterf(GLES20.GL_TEXTURE_2D, GLES20.GL_TEXTURE_MAG_FILTER, GLES20.GL_LINEAR); + GLES20.glTexParameterf(GLES20.GL_TEXTURE_2D, GLES20.GL_TEXTURE_MIN_FILTER, GLES20.GL_LINEAR); + GLES20.glTexParameterf(GLES20.GL_TEXTURE_2D, GLES20.GL_TEXTURE_WRAP_S, GLES20.GL_CLAMP_TO_EDGE); + GLES20.glTexParameterf(GLES20.GL_TEXTURE_2D, GLES20.GL_TEXTURE_WRAP_T, GLES20.GL_CLAMP_TO_EDGE); + GLES20.glBindFramebuffer(GLES20.GL_FRAMEBUFFER, state.framebufferId); + GLES20.glFramebufferTexture2D(GLES20.GL_FRAMEBUFFER, + GLES20.GL_COLOR_ATTACHMENT0, + GLES20.GL_TEXTURE_2D, + state.textureId, + 0); + + int status = GLES20.glCheckFramebufferStatus(GLES20.GL_FRAMEBUFFER); + if (status != GLES20.GL_FRAMEBUFFER_COMPLETE) { + throw new RuntimeException("Invalid framebuffer generation. Error:" + status); + } + + GLES20.glBindTexture(GLES20.GL_TEXTURE_2D, 0); + GLES20.glBindFramebuffer(GLES20.GL_FRAMEBUFFER, 0); + } + } + + private void maybeDestroyFramebuffer(@NonNull Filter filter) { + State state = states.get(filter); + //noinspection ConstantConditions + if (state.isFramebufferCreated) { + state.isFramebufferCreated = false; + GLES20.glDeleteFramebuffers(1, new int[]{state.framebufferId}, 0); + state.framebufferId = -1; + GLES20.glDeleteTextures(1, new int[]{state.textureId}, 0); + state.textureId = -1; + } + } + + private void maybeSetSize(@NonNull Filter filter) { + State state = states.get(filter); + //noinspection ConstantConditions + if (size != null && !size.equals(state.size)) { + state.size = size; + filter.setSize(size.getWidth(), size.getHeight()); + } + } + + @NonNull + @Override + public String getVertexShader() { + // Whatever, we won't be using this. + return new NoFilter().getVertexShader(); + } + + @NonNull + @Override + public String getFragmentShader() { + // Whatever, we won't be using this. + return new NoFilter().getFragmentShader(); + } + + @Override + public void onCreate(int programHandle) { + // We'll create children during the draw() op, since some of them + // might have been added after this onCreate() is called. + } + + @Override + public void onDestroy() { + synchronized (lock) { + for (Filter filter : filters) { + maybeDestroyFramebuffer(filter); + maybeDestroy(filter); + } + } + } + + @Override + public void setSize(int width, int height) { + size = new Size(width, height); + synchronized (lock) { + for (Filter filter : filters) { + maybeSetSize(filter); + } + } + } + + @Override + public void draw(float[] transformMatrix) { + synchronized (lock) { + for (int i = 0; i < filters.size(); i++) { + boolean isFirst = i == 0; + boolean isLast = i == filters.size() - 1; + Filter filter = filters.get(i); + State state = states.get(filter); + maybeSetSize(filter); + maybeCreate(filter, isFirst); + maybeCreateFramebuffer(filter, isLast); + + //noinspection ConstantConditions + GLES20.glUseProgram(state.programHandle); + + // Define the output framebuffer. + // Each filter outputs into its own framebuffer object, except the + // last filter, which outputs into the default framebuffer. + if (!isLast) { + GLES20.glBindFramebuffer(GLES20.GL_FRAMEBUFFER, state.framebufferId); + GLES20.glClearColor(0, 0, 0, 0); + } else { + GLES20.glBindFramebuffer(GLES20.GL_FRAMEBUFFER, 0); + } + + // Perform the actual drawing. + // The first filter should apply all the transformations. Then, + // since they are applied, we should use a no-op matrix. + if (isFirst) { + filter.draw(transformMatrix); + } else { + filter.draw(GlUtils.IDENTITY_MATRIX); + } + + // Set the input for the next cycle: + // It is the framebuffer texture from this cycle. If this is the last + // filter, reset this value just to cleanup. + if (!isLast) { + GLES20.glBindTexture(GLES20.GL_TEXTURE_2D, state.textureId); + } else { + GLES20.glBindTexture(GLES20.GL_TEXTURE_2D, 0); + } + + GLES20.glUseProgram(0); + } + } + } + + @NonNull + @Override + public Filter copy() { + synchronized (lock) { + MultiFilter copy = new MultiFilter(); + for (Filter filter : filters) { + copy.addFilter(filter.copy()); + } + return copy; + } + } + + @Override + public void setParameter1(float parameter1) { + this.parameter1 = parameter1; + synchronized (lock) { + for (Filter filter : filters) { + if (filter instanceof OneParameterFilter) { + ((OneParameterFilter) filter).setParameter1(parameter1); + } + } + } + } + + @Override + public void setParameter2(float parameter2) { + this.parameter2 = parameter2; + synchronized (lock) { + for (Filter filter : filters) { + if (filter instanceof TwoParameterFilter) { + ((TwoParameterFilter) filter).setParameter2(parameter2); + } + } + } + } + + @Override + public float getParameter1() { + return parameter1; + } + + @Override + public float getParameter2() { + return parameter2; + } +} diff --git a/cameraview/src/main/java/com/otaliastudios/cameraview/internal/egl/EglViewport.java b/cameraview/src/main/java/com/otaliastudios/cameraview/internal/egl/EglViewport.java index c7d03a87..ad43de64 100644 --- a/cameraview/src/main/java/com/otaliastudios/cameraview/internal/egl/EglViewport.java +++ b/cameraview/src/main/java/com/otaliastudios/cameraview/internal/egl/EglViewport.java @@ -21,7 +21,7 @@ public class EglViewport { private int mTextureUnit; private Filter mFilter; - private boolean mFilterChanged = false; + private Filter mPendingFilter; public EglViewport() { this(new NoFilter()); @@ -35,7 +35,6 @@ public class EglViewport { } private void createProgram() { - release(); // Release old program if present. mProgramHandle = GlUtils.createProgram(mFilter.getVertexShader(), mFilter.getFragmentShader()); mFilter.onCreate(mProgramHandle); } @@ -67,15 +66,18 @@ public class EglViewport { return texId; } - public void setFilter(@NonNull Filter filter){ - this.mFilter = filter; - mFilterChanged = true; + public void setFilter(@NonNull Filter filter) { + // TODO see if this is needed. If setFilter is always called from the correct GL thread, + // we don't need to wait for a new draw call (which might not even happen). + mPendingFilter = filter; } public void drawFrame(int textureId, float[] textureMatrix) { - if (mFilterChanged) { + if (mPendingFilter != null) { + release(); + mFilter = mPendingFilter; + mPendingFilter = null; createProgram(); - mFilterChanged = false; } GlUtils.checkError("draw start"); diff --git a/cameraview/src/main/java/com/otaliastudios/cameraview/picture/SnapshotGlPictureRecorder.java b/cameraview/src/main/java/com/otaliastudios/cameraview/picture/SnapshotGlPictureRecorder.java index a9939f0f..5e42d13e 100644 --- a/cameraview/src/main/java/com/otaliastudios/cameraview/picture/SnapshotGlPictureRecorder.java +++ b/cameraview/src/main/java/com/otaliastudios/cameraview/picture/SnapshotGlPictureRecorder.java @@ -95,6 +95,12 @@ public class SnapshotGlPictureRecorder extends PictureRecorder { SnapshotGlPictureRecorder.this.onRendererTextureCreated(textureId); } + @RendererThread + @Override + public void onRendererFilterChanged(@NonNull Filter filter) { + SnapshotGlPictureRecorder.this.onRendererFilterChanged(filter); + } + @RendererThread @Override public void onRendererFrame(@NonNull SurfaceTexture surfaceTexture, final float scaleX, final float scaleY) { @@ -102,10 +108,6 @@ public class SnapshotGlPictureRecorder extends PictureRecorder { SnapshotGlPictureRecorder.this.onRendererFrame(surfaceTexture, scaleX, scaleY); } - @Override - public void onFilterChanged(@NonNull Filter filter) { - mViewport.setFilter(filter.copy()); - } }); } @@ -125,6 +127,12 @@ public class SnapshotGlPictureRecorder extends PictureRecorder { } } + @RendererThread + @TargetApi(Build.VERSION_CODES.KITKAT) + private void onRendererFilterChanged(@NonNull Filter filter) { + mViewport.setFilter(filter.copy()); + } + /** * The tricky part here is the EGL surface creation. * @@ -150,7 +158,9 @@ public class SnapshotGlPictureRecorder extends PictureRecorder { */ @RendererThread @TargetApi(Build.VERSION_CODES.KITKAT) - private void onRendererFrame(final @NonNull SurfaceTexture surfaceTexture, final float scaleX, final float scaleY) { + private void onRendererFrame(@SuppressWarnings("unused") @NonNull SurfaceTexture surfaceTexture, + final float scaleX, + final float scaleY) { // Get egl context from the RendererThread, which is the one in which we have created // the textureId and the overlayTextureId, managed by the GlSurfaceView. // Next operations can then be performed on different threads using this handle. diff --git a/cameraview/src/main/java/com/otaliastudios/cameraview/preview/GlCameraPreview.java b/cameraview/src/main/java/com/otaliastudios/cameraview/preview/GlCameraPreview.java index d65e23ef..b85785df 100644 --- a/cameraview/src/main/java/com/otaliastudios/cameraview/preview/GlCameraPreview.java +++ b/cameraview/src/main/java/com/otaliastudios/cameraview/preview/GlCameraPreview.java @@ -141,7 +141,9 @@ public class GlCameraPreview extends FilterCameraPreview { @EncoderThread @Override protected void onEvent(@NonNull String event, @Nullable Object data) { + switch (event) { + case FILTER_EVENT: + //noinspection ConstantConditions + onFilter((Filter) data); + break; + case FRAME_EVENT: + //noinspection ConstantConditions + onFrame((Frame) data); + break; + } + } - if (event.equals(FILTER_EVENT)) { - Filter filter = (Filter) data; - mViewport.setFilter(filter); - } else if (event.equals(FRAME_EVENT)) { - Frame frame = (Frame) data; - if (frame == null) { - throw new IllegalArgumentException("Got null frame for FRAME_EVENT."); - } - if (!shouldRenderFrame(frame.timestampUs())) { - mFramePool.recycle(frame); - return; - } + private void onFilter(@NonNull Filter filter) { + mViewport.setFilter(filter); + } - // Notify we're got the first frame and its absolute time. - if (mFrameNumber == 1) { - notifyFirstFrameMillis(frame.timestampMillis); - } + private void onFrame(@NonNull Frame frame) { + if (!shouldRenderFrame(frame.timestampUs())) { + mFramePool.recycle(frame); + return; + } - // Notify we have reached the max length value. - if (mFirstTimeUs == Long.MIN_VALUE) mFirstTimeUs = frame.timestampUs(); - if (!hasReachedMaxLength()) { - boolean didReachMaxLength = (frame.timestampUs() - mFirstTimeUs) > getMaxLengthUs(); - if (didReachMaxLength) { - LOG.w("onEvent -", - "frameNumber:", mFrameNumber, - "timestampUs:", frame.timestampUs(), - "firstTimeUs:", mFirstTimeUs, - "- reached max length! deltaUs:", frame.timestampUs() - mFirstTimeUs); - notifyMaxLengthReached(); - } - } + // Notify we're got the first frame and its absolute time. + if (mFrameNumber == 1) { + notifyFirstFrameMillis(frame.timestampMillis); + } - // First, drain any previous data. - LOG.i("onEvent -", - "frameNumber:", mFrameNumber, - "timestampUs:", frame.timestampUs(), - "- draining."); - drainOutput(false); - - // Then draw on the surface. - LOG.i("onEvent -", - "frameNumber:", mFrameNumber, - "timestampUs:", frame.timestampUs(), - "- rendering."); - - // 1. We must scale this matrix like GlCameraPreview does, because it might have some cropping. - // Scaling takes place with respect to the (0, 0, 0) point, so we must apply a Translation to compensate. - float[] transform = frame.transform; - float scaleX = mConfig.scaleX; - float scaleY = mConfig.scaleY; - float scaleTranslX = (1F - scaleX) / 2F; - float scaleTranslY = (1F - scaleY) / 2F; - Matrix.translateM(transform, 0, scaleTranslX, scaleTranslY, 0); - Matrix.scaleM(transform, 0, scaleX, scaleY, 1); - - // 2. We also must rotate this matrix. In GlCameraPreview it is not needed because it is a live - // stream, but the output video, must be correctly rotated based on the device rotation at the moment. - // Rotation also takes place with respect to the origin (the Z axis), so we must - // translate to origin, rotate, then back to where we were. - Matrix.translateM(transform, 0, 0.5F, 0.5F, 0); - Matrix.rotateM(transform, 0, mTransformRotation, 0, 0, 1); - Matrix.translateM(transform, 0, -0.5F, -0.5F, 0); - - // 3. Do the same for overlays with their own rotation. - if (mConfig.hasOverlay()) { - mConfig.overlayDrawer.draw(mConfig.overlayTarget); - Matrix.translateM(mConfig.overlayDrawer.getTransform(), 0, 0.5F, 0.5F, 0); - Matrix.rotateM(mConfig.overlayDrawer.getTransform(), 0, mConfig.overlayRotation, 0, 0, 1); - Matrix.translateM(mConfig.overlayDrawer.getTransform(), 0, -0.5F, -0.5F, 0); + // Notify we have reached the max length value. + if (mFirstTimeUs == Long.MIN_VALUE) mFirstTimeUs = frame.timestampUs(); + if (!hasReachedMaxLength()) { + boolean didReachMaxLength = (frame.timestampUs() - mFirstTimeUs) > getMaxLengthUs(); + if (didReachMaxLength) { + LOG.w("onEvent -", + "frameNumber:", mFrameNumber, + "timestampUs:", frame.timestampUs(), + "firstTimeUs:", mFirstTimeUs, + "- reached max length! deltaUs:", frame.timestampUs() - mFirstTimeUs); + notifyMaxLengthReached(); } - mViewport.drawFrame(mConfig.textureId, transform); - if (mConfig.hasOverlay()) { - mConfig.overlayDrawer.render(); - } - mWindow.setPresentationTime(frame.timestampNanos); - mWindow.swapBuffers(); - mFramePool.recycle(frame); } + + // First, drain any previous data. + LOG.i("onEvent -", + "frameNumber:", mFrameNumber, + "timestampUs:", frame.timestampUs(), + "- draining."); + drainOutput(false); + + // Then draw on the surface. + LOG.i("onEvent -", + "frameNumber:", mFrameNumber, + "timestampUs:", frame.timestampUs(), + "- rendering."); + + // 1. We must scale this matrix like GlCameraPreview does, because it might have some cropping. + // Scaling takes place with respect to the (0, 0, 0) point, so we must apply a Translation to compensate. + float[] transform = frame.transform; + float scaleX = mConfig.scaleX; + float scaleY = mConfig.scaleY; + float scaleTranslX = (1F - scaleX) / 2F; + float scaleTranslY = (1F - scaleY) / 2F; + Matrix.translateM(transform, 0, scaleTranslX, scaleTranslY, 0); + Matrix.scaleM(transform, 0, scaleX, scaleY, 1); + + // 2. We also must rotate this matrix. In GlCameraPreview it is not needed because it is a live + // stream, but the output video, must be correctly rotated based on the device rotation at the moment. + // Rotation also takes place with respect to the origin (the Z axis), so we must + // translate to origin, rotate, then back to where we were. + Matrix.translateM(transform, 0, 0.5F, 0.5F, 0); + Matrix.rotateM(transform, 0, mTransformRotation, 0, 0, 1); + Matrix.translateM(transform, 0, -0.5F, -0.5F, 0); + + // 3. Do the same for overlays with their own rotation. + if (mConfig.hasOverlay()) { + mConfig.overlayDrawer.draw(mConfig.overlayTarget); + Matrix.translateM(mConfig.overlayDrawer.getTransform(), 0, 0.5F, 0.5F, 0); + Matrix.rotateM(mConfig.overlayDrawer.getTransform(), 0, mConfig.overlayRotation, 0, 0, 1); + Matrix.translateM(mConfig.overlayDrawer.getTransform(), 0, -0.5F, -0.5F, 0); + } + mViewport.drawFrame(mConfig.textureId, transform); + if (mConfig.hasOverlay()) { + mConfig.overlayDrawer.render(); + } + mWindow.setPresentationTime(frame.timestampNanos); + mWindow.swapBuffers(); + mFramePool.recycle(frame); } @Override diff --git a/demo/src/main/java/com/otaliastudios/cameraview/demo/CameraActivity.java b/demo/src/main/java/com/otaliastudios/cameraview/demo/CameraActivity.java index 8b97aa4a..3d8c8078 100644 --- a/demo/src/main/java/com/otaliastudios/cameraview/demo/CameraActivity.java +++ b/demo/src/main/java/com/otaliastudios/cameraview/demo/CameraActivity.java @@ -5,6 +5,7 @@ import android.content.Intent; import android.content.pm.PackageManager; import android.graphics.Bitmap; import android.graphics.BitmapFactory; +import android.graphics.Color; import android.graphics.ImageFormat; import android.graphics.Rect; import android.graphics.YuvImage; @@ -28,7 +29,9 @@ import com.otaliastudios.cameraview.controls.Mode; import com.otaliastudios.cameraview.VideoResult; import com.otaliastudios.cameraview.controls.Preview; import com.otaliastudios.cameraview.filter.Filters; +import com.otaliastudios.cameraview.filter.MultiFilter; import com.otaliastudios.cameraview.filters.BrightnessFilter; +import com.otaliastudios.cameraview.filters.DuotoneFilter; import com.otaliastudios.cameraview.frame.Frame; import com.otaliastudios.cameraview.frame.FrameProcessor; @@ -334,6 +337,12 @@ public class CameraActivity extends AppCompatActivity implements View.OnClickLis Filters filter = mAllFilters[mCurrentFilter]; camera.setFilter(filter.newInstance()); message(filter.toString(), false); + + // To test MultiFilter: + // DuotoneFilter duotone = new DuotoneFilter(); + // duotone.setFirstColor(Color.RED); + // duotone.setSecondColor(Color.GREEN); + // camera.setFilter(new MultiFilter(duotone, filter.newInstance())); } @Override diff --git a/docs/_posts/2019-08-06-filters.md b/docs/_posts/2019-08-06-filters.md index e1a864e4..56c80a85 100644 --- a/docs/_posts/2019-08-06-filters.md +++ b/docs/_posts/2019-08-06-filters.md @@ -91,6 +91,27 @@ camera.mapGesture(Gesture.SCROLL_HORIZONTAL, GestureAction.FILTER_CONTROL_1); camera.mapGesture(Gesture.SCROLL_VERTICAL, GestureAction.FILTER_CONTROL_2); ``` +### MultiFilter + +CameraView provides a special filter called `MultiFilter` which can be used to group different filters +together, and apply them in sequence to the input frame, in order to process it more than once. + +```java +camera.setFilter(new MultiFilter(firstFilter, secondFilter, thirdFilter)); +``` + +You can even add new filters to the group at any time, using `MultiFilter.addFilter()`. +The `MultiFilter` will also try to dispatch [filter controls](#filters-controls) (e.g. from gestures) +to its children. + +There are some technical caveats when using a `MultiFilter`: + +- Using a large number of child filters can consume the available graphic memory +- For [advanced users](#advanced-usage), child filters need to read from `GLES20.GL_TEXTURE_2D` + instead of the typical `GLES11Ext.GL_TEXTURE_EXTERNAL_OES`. To achieve this, we get your fragment + shader String and replace any `"samplerExternalOES"` with `"sampler2D"`. This is a hack + that might cause issues with specific shader implementations. + ### Advanced usage Advanced users with OpenGL experience can create their own filters by implementing the `Filter` interface