Improvements for supporting a round preview and output

pull/360/head
Mattia Iavarone 6 years ago
parent 97c839af87
commit fd69768ac0
  1. 9
      cameraview/src/androidTest/java/com/otaliastudios/cameraview/IntegrationTest.java
  2. 9
      cameraview/src/androidTest/java/com/otaliastudios/cameraview/PictureResultTest.java
  3. 6
      cameraview/src/main/gles/com/otaliastudios/cameraview/EglBaseSurface.java
  4. 103
      cameraview/src/main/gles/com/otaliastudios/cameraview/EglViewport.java
  5. 2
      cameraview/src/main/gles/com/otaliastudios/cameraview/TextureMediaEncoder.java
  6. 4
      cameraview/src/main/java/com/otaliastudios/cameraview/CameraView.java
  7. 3
      cameraview/src/main/java/com/otaliastudios/cameraview/FullPictureRecorder.java
  8. 28
      cameraview/src/main/java/com/otaliastudios/cameraview/PictureResult.java
  9. 12
      cameraview/src/main/java/com/otaliastudios/cameraview/SnapshotPictureRecorder.java
  10. 3
      cameraview/src/main/res/layout/cameraview_gl_view.xml
  11. 41
      cameraview/src/main/views/com/otaliastudios/cameraview/GlCameraPreview.java
  12. 1
      demo/src/main/java/com/otaliastudios/cameraview/demo/CameraActivity.java

@ -12,7 +12,6 @@ import android.support.test.runner.AndroidJUnit4;
import org.junit.After;
import org.junit.Before;
import org.junit.BeforeClass;
import org.junit.Ignore;
import org.junit.Rule;
import org.junit.Test;
import org.junit.runner.RunWith;
@ -481,11 +480,11 @@ public class IntegrationTest extends BaseTest {
Size size = camera.getPictureSize();
camera.takePicture();
PictureResult result = waitForPicture(true);
Bitmap bitmap = CameraUtils.decodeBitmap(result.getJpeg(), Integer.MAX_VALUE, Integer.MAX_VALUE);
Bitmap bitmap = CameraUtils.decodeBitmap(result.getData(), Integer.MAX_VALUE, Integer.MAX_VALUE);
assertEquals(result.getSize(), size);
assertEquals(bitmap.getWidth(), size.getWidth());
assertEquals(bitmap.getHeight(), size.getHeight());
assertNotNull(result.getJpeg());
assertNotNull(result.getData());
assertNull(result.getLocation());
assertFalse(result.isSnapshot());
}
@ -526,11 +525,11 @@ public class IntegrationTest extends BaseTest {
camera.takePictureSnapshot();
PictureResult result = waitForPicture(true);
Bitmap bitmap = CameraUtils.decodeBitmap(result.getJpeg(), Integer.MAX_VALUE, Integer.MAX_VALUE);
Bitmap bitmap = CameraUtils.decodeBitmap(result.getData(), Integer.MAX_VALUE, Integer.MAX_VALUE);
assertEquals(result.getSize(), size);
assertEquals(bitmap.getWidth(), size.getWidth());
assertEquals(bitmap.getHeight(), size.getHeight());
assertNotNull(result.getJpeg());
assertNotNull(result.getData());
assertNull(result.getLocation());
assertTrue(result.isSnapshot());
}

@ -9,8 +9,6 @@ import org.junit.Test;
import org.junit.runner.RunWith;
import org.mockito.Mockito;
import java.io.File;
import static org.junit.Assert.assertEquals;
@ -22,21 +20,24 @@ public class PictureResultTest extends BaseTest {
@Test
public void testResult() {
int format = PictureResult.FORMAT_JPEG;
int rotation = 90;
Size size = new Size(20, 120);
byte[] jpeg = new byte[]{2, 4, 1, 5, 2};
Location location = Mockito.mock(Location.class);
boolean isSnapshot = true;
result. format = format;
result.rotation = rotation;
result.size = size;
result.jpeg = jpeg;
result.data = jpeg;
result.location = location;
result.isSnapshot = isSnapshot;
assertEquals(result.getFormat(), format);
assertEquals(result.getRotation(), rotation);
assertEquals(result.getSize(), size);
assertEquals(result.getJpeg(), jpeg);
assertEquals(result.getData(), jpeg);
assertEquals(result.getLocation(), location);
assertEquals(result.isSnapshot(), isSnapshot);
}

@ -199,11 +199,11 @@ class EglBaseSurface extends EglElement {
}
/**
* Saves the EGL surface to jpeg.
* Saves the EGL surface to given format.
* <p>
* Expects that this object's EGL surface is current.
*/
public byte[] saveFrameToJpeg() {
public byte[] saveFrameTo(Bitmap.CompressFormat compressFormat) {
if (!mEglCore.isCurrent(mEGLSurface)) {
throw new RuntimeException("Expected EGL context/surface is not current");
}
@ -231,7 +231,7 @@ class EglBaseSurface extends EglElement {
ByteArrayOutputStream bos = new ByteArrayOutputStream(buf.array().length);
Bitmap bmp = Bitmap.createBitmap(width, height, Bitmap.Config.ARGB_8888);
bmp.copyPixelsFromBuffer(buf);
bmp.compress(Bitmap.CompressFormat.JPEG, 90, bos);
bmp.compress(compressFormat, 90, bos);
bmp.recycle();
return bos.toByteArray();
}

@ -3,6 +3,7 @@ package com.otaliastudios.cameraview;
import android.opengl.GLES11Ext;
import android.opengl.GLES20;
import android.opengl.Matrix;
import java.nio.FloatBuffer;
@ -17,16 +18,41 @@ class EglViewport extends EglElement {
private static final String SIMPLE_VERTEX_SHADER =
"uniform mat4 uMVPMatrix;\n" +
"uniform mat4 uTexMatrix;\n" +
"uniform vec2 uTextureScale;\n" +
"attribute vec4 aPosition;\n" +
"attribute vec4 aTextureCoord;\n" +
"varying vec2 vTextureCoord;\n" +
"varying vec2 vTextureScale;\n" +
"void main() {\n" +
" gl_Position = uMVPMatrix * aPosition;\n" +
" vTextureCoord = (uTexMatrix * aTextureCoord).xy;\n" +
" vTextureScale = uTextureScale.xy;\n" +
"}\n";
// Circle fragment shader for use with external 2D textures
// We draw as transparent all the pixels outside the circle of center (0.5, 0.5)
// and radius 0.5. Works well! But we have:
// - black in the preview. Solved with canvas.clipPath and a custom GlSurfaceView.
// - black in picture snapshots, UNLESS we use PNG which is super slow
// - black in videos because they have no alpha channel
private static final String CIRCLE_FRAGMENT_SHADER =
"#extension GL_OES_EGL_image_external : require\n" +
"precision mediump float;\n" +
"varying vec2 vTextureCoord;\n" +
"varying vec2 vTextureScale;\n" +
"uniform samplerExternalOES sTexture;\n" +
"vec2 axis = vec2(0.5, 0.5);\n" +
"vec2 center = vec2(0.5, 0.5);\n" +
"vec4 none = vec4(0.0, 0.0, 0.0, 0.0);\n" +
"void main() {\n" +
" if (pow((vTextureCoord.x - center.x) / (axis.x * vTextureScale.x), 2.0) + pow((vTextureCoord.y - center.y) / (axis.y * vTextureScale.y), 2.0) > 1.0) {\n" +
" gl_FragColor = none;\n" +
" } else {\n" +
" gl_FragColor = texture2D(sTexture, vTextureCoord);\n" +
" }\n" +
"}\n";
// Simple fragment shader for use with external 2D textures
// (e.g. what we get from SurfaceTexture).
private static final String SIMPLE_FRAGMENT_SHADER =
"#extension GL_OES_EGL_image_external : require\n" +
"precision mediump float;\n" +
@ -56,12 +82,9 @@ class EglViewport extends EglElement {
};
// Stuff from Drawable2d.FULL_RECTANGLE
private final static int VERTEX_COUNT = FULL_RECTANGLE_COORDS.length / 2;
private FloatBuffer mVertexCoordinatesArray = floatBuffer(FULL_RECTANGLE_COORDS);
private FloatBuffer mTextureCoordinatesArray = floatBuffer(FULL_RECTANGLE_TEX_COORDS);
private int mVertexCount = FULL_RECTANGLE_COORDS.length / 2;
private final int mCoordinatesPerVertex = 2;
private final int mVertexStride = 8;
private final int mTextureStride = 8;
// Stuff from Texture2dProgram
private int mProgramHandle;
@ -71,6 +94,7 @@ class EglViewport extends EglElement {
private int muTexMatrixLocation;
private int maPositionLocation;
private int maTextureCoordLocation;
private int muTextureScaleLocation;
// private int muKernelLoc; // Used for filtering
// private int muTexOffsetLoc; // Used for filtering
@ -78,11 +102,13 @@ class EglViewport extends EglElement {
EglViewport() {
mTextureTarget = GLES11Ext.GL_TEXTURE_EXTERNAL_OES;
mProgramHandle = createProgram(SIMPLE_VERTEX_SHADER, SIMPLE_FRAGMENT_SHADER);
mProgramHandle = createProgram(SIMPLE_VERTEX_SHADER, CIRCLE_FRAGMENT_SHADER);
maPositionLocation = GLES20.glGetAttribLocation(mProgramHandle, "aPosition");
checkLocation(maPositionLocation, "aPosition");
maTextureCoordLocation = GLES20.glGetAttribLocation(mProgramHandle, "aTextureCoord");
checkLocation(maTextureCoordLocation, "aTextureCoord");
muTextureScaleLocation = GLES20.glGetUniformLocation(mProgramHandle, "uTextureScale");
checkLocation(muTextureScaleLocation, "uTextureScale");
muMVPMatrixLocation = GLES20.glGetUniformLocation(mProgramHandle, "uMVPMatrix");
checkLocation(muMVPMatrixLocation, "uMVPMatrix");
muTexMatrixLocation = GLES20.glGetUniformLocation(mProgramHandle, "uTexMatrix");
@ -110,14 +136,10 @@ class EglViewport extends EglElement {
GLES20.glBindTexture(mTextureTarget, texId);
check("glBindTexture " + texId);
GLES20.glTexParameterf(GLES11Ext.GL_TEXTURE_EXTERNAL_OES,
GLES20.GL_TEXTURE_MIN_FILTER, GLES20.GL_NEAREST);
GLES20.glTexParameterf(GLES11Ext.GL_TEXTURE_EXTERNAL_OES,
GLES20.GL_TEXTURE_MAG_FILTER, GLES20.GL_LINEAR);
GLES20.glTexParameteri(GLES11Ext.GL_TEXTURE_EXTERNAL_OES,
GLES20.GL_TEXTURE_WRAP_S, GLES20.GL_CLAMP_TO_EDGE);
GLES20.glTexParameteri(GLES11Ext.GL_TEXTURE_EXTERNAL_OES,
GLES20.GL_TEXTURE_WRAP_T, GLES20.GL_CLAMP_TO_EDGE);
GLES20.glTexParameterf(GLES11Ext.GL_TEXTURE_EXTERNAL_OES, GLES20.GL_TEXTURE_MIN_FILTER, GLES20.GL_NEAREST);
GLES20.glTexParameterf(GLES11Ext.GL_TEXTURE_EXTERNAL_OES, GLES20.GL_TEXTURE_MAG_FILTER, GLES20.GL_LINEAR);
GLES20.glTexParameteri(GLES11Ext.GL_TEXTURE_EXTERNAL_OES, GLES20.GL_TEXTURE_WRAP_S, GLES20.GL_CLAMP_TO_EDGE);
GLES20.glTexParameteri(GLES11Ext.GL_TEXTURE_EXTERNAL_OES, GLES20.GL_TEXTURE_WRAP_T, GLES20.GL_CLAMP_TO_EDGE);
check("glTexParameter");
return texId;
@ -125,16 +147,34 @@ class EglViewport extends EglElement {
void drawFrame(int textureId, float[] textureMatrix) {
drawFrame(textureId, textureMatrix,
IDENTITY_MATRIX, mVertexCoordinatesArray, 0,
mVertexCount, mCoordinatesPerVertex,
mVertexStride, mTextureCoordinatesArray,
mTextureStride);
mVertexCoordinatesArray,
mTextureCoordinatesArray, 1F, 1F);
}
void drawFrame(int textureId, float[] textureMatrix, float textureScaleX, float textureScaleY) {
drawFrame(textureId, textureMatrix,
mVertexCoordinatesArray,
mTextureCoordinatesArray, textureScaleX, textureScaleY);
}
private static final FloatBuffer TEMP = floatBuffer(new float[] { 1.0F, 1.0F });
/**
* The issue with the CIRCLE shader is that if the textureMatrix has a scale value,
* it fails miserably, not taking the scale into account.
* So what we can do here is
*
* - read textureMatrix scaleX and scaleY values. This is pretty much impossible to do from the matrix itself
* without making risky assumptions over the order of operations.
* https://www.opengl.org/discussion_boards/showthread.php/159215-Is-it-possible-to-extract-rotation-translation-scale-given-a-matrix
* So we prefer passing scaleX and scaleY here to the draw function.
* - pass these values to the vertex shader
* - pass them to the fragment shader
* - in the fragment shader, take this scale value into account
*/
private void drawFrame(int textureId, float[] textureMatrix,
float[] mvpMatrix, FloatBuffer vertexBuffer, int firstVertex,
int vertexCount, int coordsPerVertex, int vertexStride,
FloatBuffer texBuffer, int texStride) {
FloatBuffer vertexBuffer,
FloatBuffer texBuffer,
float textureScaleX, float textureScaleY) {
check("draw start");
// Select the program.
@ -146,33 +186,34 @@ class EglViewport extends EglElement {
GLES20.glBindTexture(mTextureTarget, textureId);
// Copy the model / view / projection matrix over.
GLES20.glUniformMatrix4fv(muMVPMatrixLocation, 1, false, mvpMatrix, 0);
GLES20.glUniformMatrix4fv(muMVPMatrixLocation, 1, false, IDENTITY_MATRIX, 0);
check("glUniformMatrix4fv");
// Copy the texture transformation matrix over.
GLES20.glUniformMatrix4fv(muTexMatrixLocation, 1, false, textureMatrix, 0);
check("glUniformMatrix4fv");
// Pass scale values.
GLES20.glUniform2f(muTextureScaleLocation, textureScaleX, textureScaleY);
check("glUniform2f");
// Enable the "aPosition" vertex attribute.
// Connect vertexBuffer to "aPosition".
GLES20.glEnableVertexAttribArray(maPositionLocation);
check("glEnableVertexAttribArray");
// Connect vertexBuffer to "aPosition".
GLES20.glVertexAttribPointer(maPositionLocation, coordsPerVertex,
GLES20.GL_FLOAT, false, vertexStride, vertexBuffer);
GLES20.glVertexAttribPointer(maPositionLocation, 2, GLES20.GL_FLOAT, false, 8, vertexBuffer);
check("glVertexAttribPointer");
// Enable the "aTextureCoord" vertex attribute.
// Connect texBuffer to "aTextureCoord".
GLES20.glEnableVertexAttribArray(maTextureCoordLocation);
check("glEnableVertexAttribArray");
// Connect texBuffer to "aTextureCoord".
GLES20.glVertexAttribPointer(maTextureCoordLocation, 2,
GLES20.GL_FLOAT, false, texStride, texBuffer);
GLES20.glVertexAttribPointer(maTextureCoordLocation, 2, GLES20.GL_FLOAT, false, 8, texBuffer);
check("glVertexAttribPointer");
// Draw the rect.
GLES20.glDrawArrays(GLES20.GL_TRIANGLE_STRIP, firstVertex, vertexCount);
GLES20.glDrawArrays(GLES20.GL_TRIANGLE_STRIP, 0, VERTEX_COUNT);
check("glDrawArrays");
// Done -- disable vertex array, texture, and program.

@ -123,7 +123,7 @@ class TextureMediaEncoder extends VideoMediaEncoder<TextureMediaEncoder.Config>
Matrix.translateM(transform, 0, -0.5F, -0.5F, 0);
drain(false);
mViewport.drawFrame(mConfig.textureId, transform);
mViewport.drawFrame(mConfig.textureId, transform, scaleX, scaleY);
mWindow.setPresentationTime(timestamp);
mWindow.swapBuffers();
}

@ -877,7 +877,7 @@ public class CameraView extends FrameLayout implements LifecycleObserver {
/**
* Set location coordinates to be found later in the jpeg EXIF header
* Set location coordinates to be found later in the EXIF header
*
* @param latitude current latitude
* @param longitude current longitude
@ -893,7 +893,7 @@ public class CameraView extends FrameLayout implements LifecycleObserver {
/**
* Set location values to be found later in the jpeg EXIF header
* Set location values to be found later in the EXIF header
*
* @param location current location
*/

@ -51,7 +51,8 @@ class FullPictureRecorder extends PictureRecorder {
} catch (IOException e) {
exifRotation = 0;
}
mResult.jpeg = data;
mResult.format = PictureResult.FORMAT_JPEG;
mResult.data = data;
mResult.rotation = exifRotation;
camera.startPreview(); // This is needed, read somewhere in the docs.
dispatchResult();

@ -10,11 +10,15 @@ import android.support.annotation.Nullable;
*/
public class PictureResult {
public final static int FORMAT_JPEG = 0;
// public final static int FORMAT_PNG = 1;
boolean isSnapshot;
Location location;
int rotation;
Size size;
byte[] jpeg;
byte[] data;
int format;
PictureResult() {}
@ -41,7 +45,7 @@ public class PictureResult {
/**
* Returns the clock-wise rotation that should be applied to the
* picture before displaying. If it is non-zero, it is also present
* in the jpeg EXIF metadata.
* in the EXIF metadata.
*
* @return the clock-wise rotation
*/
@ -60,15 +64,25 @@ public class PictureResult {
}
/**
* Returns the raw jpeg, ready to be saved to file.
* Returns the raw compressed, ready to be saved to file,
* in the given format.
*
* @return the jpeg stream
* @return the compressed data stream
*/
@NonNull
public byte[] getJpeg() {
return jpeg;
public byte[] getData() {
return data;
}
/**
* Returns the image format. At the moment this will always be
* {@link #FORMAT_JPEG}.
*
* @return the current format
*/
public int getFormat() {
return format;
}
/**
* Shorthand for {@link CameraUtils#decodeBitmap(byte[], int, int, BitmapCallback)}.
@ -80,7 +94,7 @@ public class PictureResult {
* @param callback a callback to be notified of image decoding
*/
public void asBitmap(int maxWidth, int maxHeight, BitmapCallback callback) {
CameraUtils.decodeBitmap(getJpeg(), maxWidth, maxHeight, rotation, callback);
CameraUtils.decodeBitmap(getData(), maxWidth, maxHeight, rotation, callback);
}
/**

@ -1,6 +1,7 @@
package com.otaliastudios.cameraview;
import android.annotation.SuppressLint;
import android.graphics.Bitmap;
import android.graphics.ImageFormat;
import android.graphics.Rect;
import android.graphics.SurfaceTexture;
@ -118,9 +119,13 @@ class SnapshotPictureRecorder extends PictureRecorder {
Matrix.rotateM(mTransform, 0, rotation, 0, 0, 1);
Matrix.translateM(mTransform, 0, -0.5F, -0.5F, 0);
viewport.drawFrame(mTextureId, mTransform);
viewport.drawFrame(mTextureId, mTransform, realScaleX, realScaleY);
// don't - surface.swapBuffers();
mResult.jpeg = surface.saveFrameToJpeg();
// TODO: need PNG when the preview is rounded, which is an experimental thing
// In all the other cases this must be JPEG!
// Although PNG is extremely slow.
mResult.data = surface.saveFrameTo(Bitmap.CompressFormat.JPEG);
mResult.format = PictureResult.FORMAT_JPEG;
mSurfaceTexture.releaseTexImage();
// EGL14.eglMakeCurrent(oldDisplay, oldSurface, oldSurface, eglContext);
@ -160,9 +165,10 @@ class SnapshotPictureRecorder extends PictureRecorder {
yuv.compressToJpeg(outputRect, 90, stream);
data = stream.toByteArray();
mResult.jpeg = data;
mResult.data = data;
mResult.size = new Size(outputRect.width(), outputRect.height());
mResult.rotation = 0;
mResult.format = PictureResult.FORMAT_JPEG;
dispatchResult();
}
});

@ -6,7 +6,8 @@
android:layout_gravity="center"
android:gravity="center">
<android.opengl.GLSurfaceView
<view
class="com.otaliastudios.cameraview.GlCameraPreview$ClippingSurfaceView"
android:id="@+id/gl_surface_view"
android:layout_width="match_parent"
android:layout_height="match_parent" />

@ -1,11 +1,17 @@
package com.otaliastudios.cameraview;
import android.content.Context;
import android.graphics.Canvas;
import android.graphics.Color;
import android.graphics.Path;
import android.graphics.PixelFormat;
import android.graphics.RectF;
import android.graphics.SurfaceTexture;
import android.opengl.GLSurfaceView;
import android.opengl.Matrix;
import android.support.annotation.NonNull;
import android.support.annotation.Nullable;
import android.util.AttributeSet;
import android.view.LayoutInflater;
import android.view.SurfaceHolder;
import android.view.View;
@ -68,15 +74,43 @@ class GlCameraPreview extends CameraPreview<GLSurfaceView, SurfaceTexture> imple
super(context, parent, callback);
}
static class ClippingSurfaceView extends GLSurfaceView {
private Path path = new Path();
private RectF rect = new RectF();
public ClippingSurfaceView(Context context) {
super(context);
}
public ClippingSurfaceView(Context context, AttributeSet attrs) {
super(context, attrs);
}
@Override
protected void dispatchDraw(Canvas canvas) {
if (false) {
if (getWidth() != rect.width() || getHeight() != rect.height()) {
rect.set(0, 0, getWidth(), getHeight());
path.rewind();
path.addOval(rect, Path.Direction.CW);
}
canvas.clipPath(path);
}
super.dispatchDraw(canvas);
}
}
@NonNull
@Override
protected GLSurfaceView onCreateView(Context context, ViewGroup parent) {
View root = LayoutInflater.from(context).inflate(R.layout.cameraview_gl_view, parent, false);
parent.addView(root, 0);
ViewGroup root = (ViewGroup) LayoutInflater.from(context).inflate(R.layout.cameraview_gl_view, parent, false);
GLSurfaceView glView = root.findViewById(R.id.gl_surface_view);
glView.setEGLContextClientVersion(2);
glView.setRenderer(this);
glView.setRenderMode(GLSurfaceView.RENDERMODE_WHEN_DIRTY);
// Tried these 2 to remove the black background, does not work.
// glView.getHolder().setFormat(PixelFormat.TRANSLUCENT);
// glView.setZOrderMediaOverlay(true);
glView.getHolder().addCallback(new SurfaceHolder.Callback() {
public void surfaceCreated(SurfaceHolder holder) {}
public void surfaceChanged(SurfaceHolder holder, int format, int width, int height) { }
@ -87,6 +121,7 @@ class GlCameraPreview extends CameraPreview<GLSurfaceView, SurfaceTexture> imple
mDispatched = false;
}
});
parent.addView(root, 0);
return glView;
}
@ -192,7 +227,7 @@ class GlCameraPreview extends CameraPreview<GLSurfaceView, SurfaceTexture> imple
Matrix.translateM(mTransformMatrix, 0, translX, translY, 0);
Matrix.scaleM(mTransformMatrix, 0, mScaleX, mScaleY, 1);
}
mOutputViewport.drawFrame(mOutputTextureId, mTransformMatrix);
mOutputViewport.drawFrame(mOutputTextureId, mTransformMatrix, mScaleX, mScaleY);
for (RendererFrameCallback callback : mRendererFrameCallbacks) {
callback.onRendererFrame(mInputSurfaceTexture, mScaleX, mScaleY);
}

@ -4,6 +4,7 @@ import android.content.Intent;
import android.content.pm.PackageManager;
import android.net.Uri;
import android.os.Bundle;
import android.os.Handler;
import android.support.annotation.NonNull;
import android.support.design.widget.BottomSheetBehavior;
import android.support.v7.app.AppCompatActivity;

Loading…
Cancel
Save