diff --git a/cameraview/src/androidTest/java/com/otaliastudios/cameraview/engine/MockCameraEngine.java b/cameraview/src/androidTest/java/com/otaliastudios/cameraview/engine/MockCameraEngine.java index 9b3d4566..3b320da7 100644 --- a/cameraview/src/androidTest/java/com/otaliastudios/cameraview/engine/MockCameraEngine.java +++ b/cameraview/src/androidTest/java/com/otaliastudios/cameraview/engine/MockCameraEngine.java @@ -128,7 +128,7 @@ public class MockCameraEngine extends CameraEngine { } @Override - protected void onTakePictureSnapshot(@NonNull PictureResult.Stub stub, @NonNull AspectRatio viewAspectRatio) { + protected void onTakePictureSnapshot(@NonNull PictureResult.Stub stub, @NonNull AspectRatio outputRatio) { } @@ -138,7 +138,7 @@ public class MockCameraEngine extends CameraEngine { } @Override - protected void onTakeVideoSnapshot(@NonNull VideoResult.Stub stub, @NonNull AspectRatio viewAspectRatio) { + protected void onTakeVideoSnapshot(@NonNull VideoResult.Stub stub, @NonNull AspectRatio outputRatio) { } diff --git a/cameraview/src/main/java/com/otaliastudios/cameraview/CameraView.java b/cameraview/src/main/java/com/otaliastudios/cameraview/CameraView.java index e54bfaab..94259dd2 100644 --- a/cameraview/src/main/java/com/otaliastudios/cameraview/CameraView.java +++ b/cameraview/src/main/java/com/otaliastudios/cameraview/CameraView.java @@ -425,10 +425,10 @@ public class CameraView extends FrameLayout implements LifecycleObserver { int height, width; if (freeWidth) { height = heightValue; - width = (int) (height / ratio); + width = Math.round(height / ratio); } else { width = widthValue; - height = (int) (width * ratio); + height = Math.round(width * ratio); } LOG.i("onMeasure:", "one dimension was free, we adapted it to fit the aspect ratio.", "(" + width + "x" + height + ")"); @@ -445,10 +445,10 @@ public class CameraView extends FrameLayout implements LifecycleObserver { int height, width; if (freeWidth) { height = heightValue; - width = Math.min((int) (height / ratio), widthValue); + width = Math.min(Math.round(height / ratio), widthValue); } else { width = widthValue; - height = Math.min((int) (width * ratio), heightValue); + height = Math.min(Math.round(width * ratio), heightValue); } LOG.i("onMeasure:", "one dimension was EXACTLY, another AT_MOST.", "We have TRIED to fit the aspect ratio, but it's not guaranteed.", @@ -465,10 +465,10 @@ public class CameraView extends FrameLayout implements LifecycleObserver { if (atMostRatio >= ratio) { // We must reduce height. width = widthValue; - height = (int) (width * ratio); + height = Math.round(width * ratio); } else { height = heightValue; - width = (int) (height / ratio); + width = Math.round(height / ratio); } LOG.i("onMeasure:", "both dimension were AT_MOST.", "We fit the preview aspect ratio.", @@ -1465,9 +1465,8 @@ public class CameraView extends FrameLayout implements LifecycleObserver { * @see #takePicture() */ public void takePictureSnapshot() { - if (getWidth() == 0 || getHeight() == 0) return; PictureResult.Stub stub = new PictureResult.Stub(); - mCameraEngine.takePictureSnapshot(stub, AspectRatio.of(getWidth(), getHeight())); + mCameraEngine.takePictureSnapshot(stub); } @@ -1499,9 +1498,8 @@ public class CameraView extends FrameLayout implements LifecycleObserver { * @param file a file where the video will be saved */ public void takeVideoSnapshot(@NonNull File file) { - if (getWidth() == 0 || getHeight() == 0) return; VideoResult.Stub stub = new VideoResult.Stub(); - mCameraEngine.takeVideoSnapshot(stub, file, AspectRatio.of(getWidth(), getHeight())); + mCameraEngine.takeVideoSnapshot(stub, file); mUiHandler.post(new Runnable() { @Override public void run() { diff --git a/cameraview/src/main/java/com/otaliastudios/cameraview/engine/Camera1Engine.java b/cameraview/src/main/java/com/otaliastudios/cameraview/engine/Camera1Engine.java index 5ee4a480..be9f9009 100644 --- a/cameraview/src/main/java/com/otaliastudios/cameraview/engine/Camera1Engine.java +++ b/cameraview/src/main/java/com/otaliastudios/cameraview/engine/Camera1Engine.java @@ -306,10 +306,9 @@ public class Camera1Engine extends CameraEngine implements @WorkerThread @Override - protected void onTakePictureSnapshot(@NonNull PictureResult.Stub stub, @NonNull AspectRatio viewAspectRatio) { + protected void onTakePictureSnapshot(@NonNull PictureResult.Stub stub, @NonNull AspectRatio outputRatio) { stub.size = getUncroppedSnapshotSize(Reference.OUTPUT); // Not the real size: it will be cropped to match the view ratio stub.rotation = getAngles().offset(Reference.SENSOR, Reference.OUTPUT, Axis.RELATIVE_TO_SENSOR); // Actually it will be rotated and set to 0. - AspectRatio outputRatio = getAngles().flip(Reference.OUTPUT, Reference.VIEW) ? viewAspectRatio.flip() : viewAspectRatio; if (mPreview instanceof GlCameraPreview && Build.VERSION.SDK_INT >= Build.VERSION_CODES.KITKAT) { mPictureRecorder = new SnapshotGlPictureRecorder(stub, this, (GlCameraPreview) mPreview, outputRatio, getOverlay()); @@ -343,7 +342,7 @@ public class Camera1Engine extends CameraEngine implements @SuppressLint("NewApi") @WorkerThread @Override - protected void onTakeVideoSnapshot(@NonNull VideoResult.Stub stub, @NonNull AspectRatio viewAspectRatio) { + protected void onTakeVideoSnapshot(@NonNull VideoResult.Stub stub, @NonNull AspectRatio outputRatio) { if (!(mPreview instanceof GlCameraPreview)) { throw new IllegalStateException("Video snapshots are only supported with GlCameraPreview."); } @@ -355,7 +354,6 @@ public class Camera1Engine extends CameraEngine implements if (outputSize == null) { throw new IllegalStateException("outputSize should not be null."); } - AspectRatio outputRatio = getAngles().flip(Reference.VIEW, Reference.OUTPUT) ? viewAspectRatio.flip() : viewAspectRatio; Rect outputCrop = CropHelper.computeCrop(outputSize, outputRatio); outputSize = new Size(outputCrop.width(), outputCrop.height()); stub.size = outputSize; diff --git a/cameraview/src/main/java/com/otaliastudios/cameraview/engine/Camera2Engine.java b/cameraview/src/main/java/com/otaliastudios/cameraview/engine/Camera2Engine.java index 24c36554..4839303c 100644 --- a/cameraview/src/main/java/com/otaliastudios/cameraview/engine/Camera2Engine.java +++ b/cameraview/src/main/java/com/otaliastudios/cameraview/engine/Camera2Engine.java @@ -610,10 +610,9 @@ public class Camera2Engine extends CameraEngine implements ImageReader.OnImageAv @WorkerThread @Override - protected void onTakePictureSnapshot(@NonNull PictureResult.Stub stub, @NonNull AspectRatio viewAspectRatio) { + protected void onTakePictureSnapshot(@NonNull PictureResult.Stub stub, @NonNull AspectRatio outputRatio) { stub.size = getUncroppedSnapshotSize(Reference.OUTPUT); // Not the real size: it will be cropped to match the view ratio stub.rotation = getAngles().offset(Reference.SENSOR, Reference.OUTPUT, Axis.RELATIVE_TO_SENSOR); // Actually it will be rotated and set to 0. - AspectRatio outputRatio = getAngles().flip(Reference.OUTPUT, Reference.VIEW) ? viewAspectRatio.flip() : viewAspectRatio; if (mPreview instanceof GlCameraPreview) { mPictureRecorder = new SnapshotGlPictureRecorder(stub, this, (GlCameraPreview) mPreview, outputRatio, getOverlay()); } else { @@ -695,7 +694,7 @@ public class Camera2Engine extends CameraEngine implements ImageReader.OnImageAv @WorkerThread @Override - protected void onTakeVideoSnapshot(@NonNull VideoResult.Stub stub, @NonNull AspectRatio viewAspectRatio) { + protected void onTakeVideoSnapshot(@NonNull VideoResult.Stub stub, @NonNull AspectRatio outputRatio) { if (!(mPreview instanceof GlCameraPreview)) { throw new IllegalStateException("Video snapshots are only supported with GlCameraPreview."); } @@ -704,7 +703,6 @@ public class Camera2Engine extends CameraEngine implements ImageReader.OnImageAv if (outputSize == null) { throw new IllegalStateException("outputSize should not be null."); } - AspectRatio outputRatio = getAngles().flip(Reference.VIEW, Reference.OUTPUT) ? viewAspectRatio.flip() : viewAspectRatio; Rect outputCrop = CropHelper.computeCrop(outputSize, outputRatio); outputSize = new Size(outputCrop.width(), outputCrop.height()); stub.size = outputSize; @@ -1257,8 +1255,8 @@ public class Camera2Engine extends CameraEngine implements ImageReader.OnImageAv private void onAutoFocusCapture(@NonNull CaptureResult result) { Integer afState = result.get(CaptureResult.CONTROL_AF_STATE); if (afState == null) { - LOG.e("onAutoFocusCapture", "afState is null! Assuming AF failed."); - afState = CaptureResult.CONTROL_AF_STATE_NOT_FOCUSED_LOCKED; + LOG.i("onAutoFocusCapture", "afState is null! This can happen for partial results. Waiting."); + return; } switch (afState) { case CaptureRequest.CONTROL_AF_STATE_FOCUSED_LOCKED: { diff --git a/cameraview/src/main/java/com/otaliastudios/cameraview/engine/CameraEngine.java b/cameraview/src/main/java/com/otaliastudios/cameraview/engine/CameraEngine.java index 2d2b1d60..a39907b2 100644 --- a/cameraview/src/main/java/com/otaliastudios/cameraview/engine/CameraEngine.java +++ b/cameraview/src/main/java/com/otaliastudios/cameraview/engine/CameraEngine.java @@ -1087,9 +1087,8 @@ public abstract class CameraEngine implements * The snapshot size is the {@link #getPreviewStreamSize(Reference)}, but cropped based on the * view/surface aspect ratio. * @param stub a picture stub - * @param viewAspectRatio the view aspect ratio */ - public final void takePictureSnapshot(final @NonNull PictureResult.Stub stub, @NonNull final AspectRatio viewAspectRatio) { + public final void takePictureSnapshot(final @NonNull PictureResult.Stub stub) { LOG.v("takePictureSnapshot", "scheduling"); mHandler.run(new Runnable() { @Override @@ -1101,7 +1100,9 @@ public abstract class CameraEngine implements stub.isSnapshot = true; stub.facing = mFacing; // Leave the other parameters to subclasses. - onTakePictureSnapshot(stub, viewAspectRatio); + //noinspection ConstantConditions + AspectRatio ratio = AspectRatio.of(getPreviewSurfaceSize(Reference.OUTPUT)); + onTakePictureSnapshot(stub, ratio); } }); } @@ -1155,9 +1156,8 @@ public abstract class CameraEngine implements /** * @param stub a video stub * @param file the output file - * @param viewAspectRatio the view aspect ratio */ - public final void takeVideoSnapshot(final @NonNull VideoResult.Stub stub, @NonNull final File file, @NonNull final AspectRatio viewAspectRatio) { + public final void takeVideoSnapshot(final @NonNull VideoResult.Stub stub, @NonNull final File file) { LOG.v("takeVideoSnapshot", "scheduling"); mHandler.run(new Runnable() { @Override @@ -1175,7 +1175,9 @@ public abstract class CameraEngine implements stub.audio = mAudio; stub.maxSize = mVideoMaxSize; stub.maxDuration = mVideoMaxDuration; - onTakeVideoSnapshot(stub, viewAspectRatio); + //noinspection ConstantConditions + AspectRatio ratio = AspectRatio.of(getPreviewSurfaceSize(Reference.OUTPUT)); + onTakeVideoSnapshot(stub, ratio); } }); } @@ -1220,10 +1222,10 @@ public abstract class CameraEngine implements protected abstract void onTakePicture(@NonNull PictureResult.Stub stub); @WorkerThread - protected abstract void onTakePictureSnapshot(@NonNull PictureResult.Stub stub, @NonNull AspectRatio viewAspectRatio); + protected abstract void onTakePictureSnapshot(@NonNull PictureResult.Stub stub, @NonNull AspectRatio outputRatio); @WorkerThread - protected abstract void onTakeVideoSnapshot(@NonNull VideoResult.Stub stub, @NonNull AspectRatio viewAspectRatio); + protected abstract void onTakeVideoSnapshot(@NonNull VideoResult.Stub stub, @NonNull AspectRatio outputRatio); @WorkerThread protected abstract void onTakeVideo(@NonNull VideoResult.Stub stub); diff --git a/cameraview/src/main/java/com/otaliastudios/cameraview/internal/egl/EglBaseSurface.java b/cameraview/src/main/java/com/otaliastudios/cameraview/internal/egl/EglBaseSurface.java index baff79e6..de2919fa 100644 --- a/cameraview/src/main/java/com/otaliastudios/cameraview/internal/egl/EglBaseSurface.java +++ b/cameraview/src/main/java/com/otaliastudios/cameraview/internal/egl/EglBaseSurface.java @@ -51,7 +51,7 @@ public class EglBaseSurface extends EglElement { private int mWidth = -1; private int mHeight = -1; - protected EglBaseSurface(EglCore eglCore) { + public EglBaseSurface(EglCore eglCore) { mEglCore = eglCore; } diff --git a/cameraview/src/main/java/com/otaliastudios/cameraview/internal/utils/CropHelper.java b/cameraview/src/main/java/com/otaliastudios/cameraview/internal/utils/CropHelper.java index 59586120..f7f71ee7 100644 --- a/cameraview/src/main/java/com/otaliastudios/cameraview/internal/utils/CropHelper.java +++ b/cameraview/src/main/java/com/otaliastudios/cameraview/internal/utils/CropHelper.java @@ -17,7 +17,7 @@ public class CropHelper { public static Rect computeCrop(@NonNull Size currentSize, @NonNull AspectRatio targetRatio) { int currentWidth = currentSize.getWidth(); int currentHeight = currentSize.getHeight(); - if (targetRatio.matches(currentSize)) { + if (targetRatio.matches(currentSize, 0.0005F)) { return new Rect(0, 0, currentWidth, currentHeight); } @@ -26,13 +26,13 @@ public class CropHelper { int x, y, width, height; if (currentRatio.toFloat() > targetRatio.toFloat()) { height = currentHeight; - width = (int) (height * targetRatio.toFloat()); + width = Math.round(height * targetRatio.toFloat()); y = 0; - x = (currentWidth - width) / 2; + x = Math.round((currentWidth - width) / 2F); } else { width = currentWidth; - height = (int) (width / targetRatio.toFloat()); - y = (currentHeight - height) / 2; + height = Math.round(width / targetRatio.toFloat()); + y = Math.round((currentHeight - height) / 2F); x = 0; } return new Rect(x, y, x + width, y + height); diff --git a/cameraview/src/main/java/com/otaliastudios/cameraview/picture/SnapshotGlPictureRecorder.java b/cameraview/src/main/java/com/otaliastudios/cameraview/picture/SnapshotGlPictureRecorder.java index d13ef3a9..9078fa89 100644 --- a/cameraview/src/main/java/com/otaliastudios/cameraview/picture/SnapshotGlPictureRecorder.java +++ b/cameraview/src/main/java/com/otaliastudios/cameraview/picture/SnapshotGlPictureRecorder.java @@ -14,6 +14,7 @@ import android.os.Build; import com.otaliastudios.cameraview.CameraLogger; import com.otaliastudios.cameraview.PictureResult; +import com.otaliastudios.cameraview.internal.egl.EglBaseSurface; import com.otaliastudios.cameraview.overlay.Overlay; import com.otaliastudios.cameraview.controls.Facing; import com.otaliastudios.cameraview.engine.CameraEngine; @@ -35,6 +36,23 @@ import androidx.annotation.Nullable; import android.view.Surface; +/** + * API 19. + * Records a picture snapshots from the {@link GlCameraPreview}. It works as follows: + * + * - We register a one time {@link RendererFrameCallback} on the preview + * - We get the textureId and the frame callback on the {@link RendererThread} + * - [Optional: we construct another textureId for overlays] + * - We take a handle of the EGL context from the {@link RendererThread} + * - We move to another thread, and create a new EGL surface for that EGL context. + * - We make this new surface current, and re-draw the textureId on it + * - [Optional: fill the overlayTextureId and draw it on the same surface] + * - We use glReadPixels (through {@link EglBaseSurface#saveFrameTo(Bitmap.CompressFormat)}) and save to file. + * + * We create a new EGL surface and redraw the frame because: + * 1. We want to go off the renderer thread as soon as possible + * 2. We have overlays to be drawn - we don't want to draw them on the preview surface, not even for a frame. + */ public class SnapshotGlPictureRecorder extends PictureRecorder { private static final String TAG = SnapshotGlPictureRecorder.class.getSimpleName(); @@ -47,6 +65,17 @@ public class SnapshotGlPictureRecorder extends PictureRecorder { private Overlay mOverlay; private boolean mHasOverlay; + private int mTextureId; + private SurfaceTexture mSurfaceTexture; + private float[] mTransform; + + private int mOverlayTextureId = 0; + private SurfaceTexture mOverlaySurfaceTexture; + private Surface mOverlaySurface; + private float[] mOverlayTransform; + + private EglViewport mViewport; + public SnapshotGlPictureRecorder( @NonNull PictureResult.Stub stub, @NonNull CameraEngine engine, @@ -66,142 +95,148 @@ public class SnapshotGlPictureRecorder extends PictureRecorder { public void take() { mPreview.addRendererFrameCallback(new RendererFrameCallback() { - int mTextureId; - SurfaceTexture mSurfaceTexture; - float[] mTransform; - - int mOverlayTextureId = 0; - SurfaceTexture mOverlaySurfaceTexture; - Surface mOverlaySurface; - float[] mOverlayTransform; - - EglViewport mViewport; - @RendererThread public void onRendererTextureCreated(int textureId) { - mTextureId = textureId; - mViewport = new EglViewport(); - mSurfaceTexture = new SurfaceTexture(mTextureId, true); - // Need to crop the size. - Rect crop = CropHelper.computeCrop(mResult.size, mOutputRatio); - mResult.size = new Size(crop.width(), crop.height()); - mSurfaceTexture.setDefaultBufferSize(mResult.size.getWidth(), mResult.size.getHeight()); - mTransform = new float[16]; - - if (mHasOverlay) { - mOverlayTextureId = mViewport.createTexture(); - mOverlaySurfaceTexture = new SurfaceTexture(mOverlayTextureId, true); - mOverlaySurfaceTexture.setDefaultBufferSize(mResult.size.getWidth(), mResult.size.getHeight()); - mOverlaySurface = new Surface(mOverlaySurfaceTexture); - mOverlayTransform = new float[16]; - } + SnapshotGlPictureRecorder.this.onRendererTextureCreated(textureId); } @RendererThread @Override public void onRendererFrame(@NonNull SurfaceTexture surfaceTexture, final float scaleX, final float scaleY) { mPreview.removeRendererFrameCallback(this); + SnapshotGlPictureRecorder.this.onRendererFrame(scaleX, scaleY); + } + }); + } - // This kinda work but has drawbacks: - // - output is upside down due to coordinates in GL: need to flip the byte[] someway - // - output is not rotated as we would like to: need to create a bitmap copy... - // - works only in the renderer thread, where it allocates the buffer and reads pixels. Bad! - /* - ByteBuffer buffer = ByteBuffer.allocateDirect(width * height * 4); - buffer.order(ByteOrder.LITTLE_ENDIAN); - GLES20.glReadPixels(0, 0, width, height, GLES20.GL_RGBA, GLES20.GL_UNSIGNED_BYTE, buffer); - buffer.rewind(); - ByteArrayOutputStream bos = new ByteArrayOutputStream(buffer.array().length); - Bitmap bitmap = Bitmap.createBitmap(width, height, Bitmap.Config.ARGB_8888); - bitmap.copyPixelsFromBuffer(buffer); - bitmap.compress(Bitmap.CompressFormat.JPEG, 90, bos); - bitmap.recycle(); */ - - // For this reason it is better to create a new surface, - // and draw the last frame again there. - final EGLContext eglContext = EGL14.eglGetCurrentContext(); - final EglCore core = new EglCore(eglContext, EglCore.FLAG_RECORDABLE); - // final EGLSurface oldSurface = EGL14.eglGetCurrentSurface(EGL14.EGL_DRAW); - // final EGLDisplay oldDisplay = EGL14.eglGetCurrentDisplay(); - WorkerHandler.execute(new Runnable() { - @Override - public void run() { - // 1. Get latest texture - EglWindowSurface surface = new EglWindowSurface(core, mSurfaceTexture); - surface.makeCurrent(); - mSurfaceTexture.updateTexImage(); - mSurfaceTexture.getTransformMatrix(mTransform); - - // 2. Apply scale and crop: - // scaleX and scaleY are in REF_VIEW, while our input appears to be in REF_SENSOR. - boolean flip = mEngine.getAngles().flip(Reference.VIEW, Reference.SENSOR); - float realScaleX = flip ? scaleY : scaleX; - float realScaleY = flip ? scaleX : scaleY; - float scaleTranslX = (1F - realScaleX) / 2F; - float scaleTranslY = (1F - realScaleY) / 2F; - Matrix.translateM(mTransform, 0, scaleTranslX, scaleTranslY, 0); - Matrix.scaleM(mTransform, 0, realScaleX, realScaleY, 1); - - // 3. Go back to 0,0 so that rotate and flip work well. - Matrix.translateM(mTransform, 0, 0.5F, 0.5F, 0); - - // 4. Apply rotation: - // Not sure why we need the minus here. - Matrix.rotateM(mTransform, 0, -mResult.rotation, 0, 0, 1); - mResult.rotation = 0; - - // 5. Flip horizontally for front camera: - if (mResult.facing == Facing.FRONT) { - Matrix.scaleM(mTransform, 0, -1, 1, 1); - } - - // 6. Go back to old position. - Matrix.translateM(mTransform, 0, -0.5F, -0.5F, 0); - - // 7. Do pretty much the same for overlays, though with - // some differences. - if (mHasOverlay) { - // 1. First we must draw on the texture and get latest image. - try { - final Canvas surfaceCanvas = mOverlaySurface.lockCanvas(null); - surfaceCanvas.drawColor(Color.TRANSPARENT, PorterDuff.Mode.CLEAR); - mOverlay.drawOn(Overlay.Target.PICTURE_SNAPSHOT, surfaceCanvas); - mOverlaySurface.unlockCanvasAndPost(surfaceCanvas); - } catch (Surface.OutOfResourcesException e) { - LOG.w("Got Surface.OutOfResourcesException while drawing picture overlays", e); - } - mOverlaySurfaceTexture.updateTexImage(); - mOverlaySurfaceTexture.getTransformMatrix(mOverlayTransform); - - // 2. Then we can apply the transformations. - int rotation = mEngine.getAngles().offset(Reference.VIEW, Reference.OUTPUT, Axis.ABSOLUTE); - Matrix.translateM(mOverlayTransform, 0, 0.5F, 0.5F, 0); - Matrix.rotateM(mOverlayTransform, 0, rotation, 0, 0, 1); - // No need to flip the x axis for front camera, but need to flip the y axis always. - Matrix.scaleM(mOverlayTransform, 0, 1, -1, 1); - Matrix.translateM(mOverlayTransform, 0, -0.5F, -0.5F, 0); - } - - // 8. Draw and save - mViewport.drawFrame(mTextureId, mTransform); - if (mHasOverlay) mViewport.drawFrame(mOverlayTextureId, mOverlayTransform); - // don't - surface.swapBuffers(); - mResult.data = surface.saveFrameTo(Bitmap.CompressFormat.JPEG); - mResult.format = PictureResult.FORMAT_JPEG; - - // 9. Cleanup - mSurfaceTexture.releaseTexImage(); - surface.release(); - mViewport.release(); - mSurfaceTexture.release(); - if (mHasOverlay) { - mOverlaySurface.release(); - mOverlaySurfaceTexture.release(); - } - core.release(); - dispatchResult(); + @RendererThread + @TargetApi(Build.VERSION_CODES.KITKAT) + private void onRendererTextureCreated(int textureId) { + mTextureId = textureId; + mViewport = new EglViewport(); + mSurfaceTexture = new SurfaceTexture(mTextureId, true); + // Need to crop the size. + Rect crop = CropHelper.computeCrop(mResult.size, mOutputRatio); + mResult.size = new Size(crop.width(), crop.height()); + mSurfaceTexture.setDefaultBufferSize(mResult.size.getWidth(), mResult.size.getHeight()); + mTransform = new float[16]; + + if (mHasOverlay) { + mOverlayTextureId = mViewport.createTexture(); + mOverlaySurfaceTexture = new SurfaceTexture(mOverlayTextureId, true); + mOverlaySurfaceTexture.setDefaultBufferSize(mResult.size.getWidth(), mResult.size.getHeight()); + mOverlaySurface = new Surface(mOverlaySurfaceTexture); + mOverlayTransform = new float[16]; + } + } + + /** + * The tricky part here is the EGL surface creation. + * + * We don't have a real output window for the EGL surface - we will use glReadPixels() + * and never call swapBuffers(), so what we draw is never published. + * + * 1. One option is to use a pbuffer EGL surface. This works, we just have to pass + * the correct width and height. However, it is significantly slower than the current + * solution. + * + * 2. Another option is to create the EGL surface out of a ImageReader.getSurface() + * and use the reader to create a JPEG. In this case, we would have to publish + * the frame with swapBuffers(). However, currently ImageReader does not support + * all formats, it's risky. This is an example error that we get: + * "RGBA override BLOB format buffer should have height == width" + * + * The third option, which we are using, is to create the EGL surface using whatever + * {@link Surface} or {@link SurfaceTexture} we have at hand. Since we never call + * swapBuffers(), the frame will not actually be rendered. This is the fastest. + * + * @param scaleX frame scale x in {@link Reference#VIEW} + * @param scaleY frame scale y in {@link Reference#VIEW} + */ + @RendererThread + @TargetApi(Build.VERSION_CODES.KITKAT) + private void onRendererFrame(final float scaleX, final float scaleY) { + // Get egl context from the RendererThread, which is the one in which we have created + // the textureId and the overlayTextureId, managed by the GlSurfaceView. + // Next operations can then be performed on different threads using this handle. + final EGLContext eglContext = EGL14.eglGetCurrentContext(); + final EglCore core = new EglCore(eglContext, EglCore.FLAG_RECORDABLE); + WorkerHandler.execute(new Runnable() { + @Override + public void run() { + // 0. Create an EGL surface + EglBaseSurface eglSurface = new EglWindowSurface(core, mSurfaceTexture); + eglSurface.makeCurrent(); + + // 1. Get latest texture + mSurfaceTexture.updateTexImage(); + mSurfaceTexture.getTransformMatrix(mTransform); + + // 2. Apply scale and crop + boolean flip = mEngine.getAngles().flip(Reference.VIEW, Reference.SENSOR); + float realScaleX = flip ? scaleY : scaleX; + float realScaleY = flip ? scaleX : scaleY; + float scaleTranslX = (1F - realScaleX) / 2F; + float scaleTranslY = (1F - realScaleY) / 2F; + Matrix.translateM(mTransform, 0, scaleTranslX, scaleTranslY, 0); + Matrix.scaleM(mTransform, 0, realScaleX, realScaleY, 1); + + // 3. Go back to 0,0 so that rotate and flip work well + Matrix.translateM(mTransform, 0, 0.5F, 0.5F, 0); + + // 4. Apply rotation (not sure why we need the minus here) + Matrix.rotateM(mTransform, 0, -mResult.rotation, 0, 0, 1); + mResult.rotation = 0; + + // 5. Flip horizontally for front camera + if (mResult.facing == Facing.FRONT) { + Matrix.scaleM(mTransform, 0, -1, 1, 1); + } + + // 6. Go back to old position + Matrix.translateM(mTransform, 0, -0.5F, -0.5F, 0); + + // 7. Do pretty much the same for overlays + if (mHasOverlay) { + // 1. First we must draw on the texture and get latest image + try { + final Canvas surfaceCanvas = mOverlaySurface.lockCanvas(null); + surfaceCanvas.drawColor(Color.TRANSPARENT, PorterDuff.Mode.CLEAR); + mOverlay.drawOn(Overlay.Target.PICTURE_SNAPSHOT, surfaceCanvas); + mOverlaySurface.unlockCanvasAndPost(surfaceCanvas); + } catch (Surface.OutOfResourcesException e) { + LOG.w("Got Surface.OutOfResourcesException while drawing picture overlays", e); } - }); + mOverlaySurfaceTexture.updateTexImage(); + mOverlaySurfaceTexture.getTransformMatrix(mOverlayTransform); + + // 2. Then we can apply the transformations + int rotation = mEngine.getAngles().offset(Reference.VIEW, Reference.OUTPUT, Axis.ABSOLUTE); + Matrix.translateM(mOverlayTransform, 0, 0.5F, 0.5F, 0); + Matrix.rotateM(mOverlayTransform, 0, rotation, 0, 0, 1); + // No need to flip the x axis for front camera, but need to flip the y axis always. + Matrix.scaleM(mOverlayTransform, 0, 1, -1, 1); + Matrix.translateM(mOverlayTransform, 0, -0.5F, -0.5F, 0); + } + + // 8. Draw and save + mViewport.drawFrame(mTextureId, mTransform); + if (mHasOverlay) mViewport.drawFrame(mOverlayTextureId, mOverlayTransform); + mResult.format = PictureResult.FORMAT_JPEG; + mResult.data = eglSurface.saveFrameTo(Bitmap.CompressFormat.JPEG); + + // 9. Cleanup + mSurfaceTexture.releaseTexImage(); + eglSurface.releaseEglSurface(); + mViewport.release(); + mSurfaceTexture.release(); + if (mHasOverlay) { + mOverlaySurfaceTexture.releaseTexImage(); + mOverlaySurface.release(); + mOverlaySurfaceTexture.release(); + } + core.release(); + dispatchResult(); } }); } diff --git a/cameraview/src/main/java/com/otaliastudios/cameraview/size/AspectRatio.java b/cameraview/src/main/java/com/otaliastudios/cameraview/size/AspectRatio.java index 71a47572..e13b73c1 100644 --- a/cameraview/src/main/java/com/otaliastudios/cameraview/size/AspectRatio.java +++ b/cameraview/src/main/java/com/otaliastudios/cameraview/size/AspectRatio.java @@ -19,7 +19,7 @@ public class AspectRatio implements Comparable { * @return a (possibly cached) aspect ratio */ @NonNull - public static AspectRatio of(Size size) { + public static AspectRatio of(@NonNull Size size) { return AspectRatio.of(size.getWidth(), size.getHeight()); } @@ -78,7 +78,6 @@ public class AspectRatio implements Comparable { return mY; } - @SuppressWarnings("WeakerAccess") public boolean matches(@NonNull Size size) { int gcd = gcd(size.getWidth(), size.getHeight()); int x = size.getWidth() / gcd; @@ -86,6 +85,10 @@ public class AspectRatio implements Comparable { return mX == x && mY == y; } + public boolean matches(@NonNull Size size, float tolerance) { + return Math.abs(toFloat() - (float) size.getWidth() / size.getHeight()) <= tolerance; + } + @Override public boolean equals(Object o) { if (o == null) { @@ -107,7 +110,6 @@ public class AspectRatio implements Comparable { return mX + ":" + mY; } - @SuppressWarnings("WeakerAccess") public float toFloat() { return (float) mX / mY; } diff --git a/cameraview/src/main/java/com/otaliastudios/cameraview/video/SnapshotVideoRecorder.java b/cameraview/src/main/java/com/otaliastudios/cameraview/video/SnapshotVideoRecorder.java index 34d7f5da..d56dba67 100644 --- a/cameraview/src/main/java/com/otaliastudios/cameraview/video/SnapshotVideoRecorder.java +++ b/cameraview/src/main/java/com/otaliastudios/cameraview/video/SnapshotVideoRecorder.java @@ -98,6 +98,7 @@ public class SnapshotVideoRecorder extends VideoRecorder implements RendererFram mOverlaySurfaceTexture = new SurfaceTexture(mOverlayTextureId); mOverlaySurfaceTexture.setDefaultBufferSize(mResult.size.getWidth(), mResult.size.getHeight()); mOverlaySurface = new Surface(mOverlaySurfaceTexture); + temp.release(true); } } @@ -135,6 +136,9 @@ public class SnapshotVideoRecorder extends VideoRecorder implements RendererFram videoConfig.textureId = mTextureId; videoConfig.scaleX = scaleX; videoConfig.scaleY = scaleY; + // Get egl context from the RendererThread, which is the one in which we have created + // the textureId and the overlayTextureId, managed by the GlSurfaceView. + // Next operations can then be performed on different threads using this handle. videoConfig.eglContext = EGL14.eglGetCurrentContext(); if (mHasOverlay) { videoConfig.overlayTextureId = mOverlayTextureId; diff --git a/cameraview/src/main/java/com/otaliastudios/cameraview/video/encoding/TextureMediaEncoder.java b/cameraview/src/main/java/com/otaliastudios/cameraview/video/encoding/TextureMediaEncoder.java index fcd78cd4..d7b8eb75 100644 --- a/cameraview/src/main/java/com/otaliastudios/cameraview/video/encoding/TextureMediaEncoder.java +++ b/cameraview/src/main/java/com/otaliastudios/cameraview/video/encoding/TextureMediaEncoder.java @@ -1,6 +1,7 @@ package com.otaliastudios.cameraview.video.encoding; import android.graphics.SurfaceTexture; +import android.media.ImageReader; import android.opengl.Matrix; import android.os.Build; @@ -94,8 +95,7 @@ public class TextureMediaEncoder extends VideoMediaEncoder { super.onPrepare(controller, maxLengthMillis); mEglCore = new EglCore(mConfig.eglContext, EglCore.FLAG_RECORDABLE); mWindow = new EglWindowSurface(mEglCore, mSurface, true); - mWindow.makeCurrent(); // drawing will happen on the InputWindowSurface, which - // is backed by mVideoEncoder.getInputSurface() + mWindow.makeCurrent(); mViewport = new EglViewport(); } diff --git a/cameraview/src/main/java/com/otaliastudios/cameraview/video/encoding/VideoMediaEncoder.java b/cameraview/src/main/java/com/otaliastudios/cameraview/video/encoding/VideoMediaEncoder.java index bc5e98f8..64061a37 100644 --- a/cameraview/src/main/java/com/otaliastudios/cameraview/video/encoding/VideoMediaEncoder.java +++ b/cameraview/src/main/java/com/otaliastudios/cameraview/video/encoding/VideoMediaEncoder.java @@ -53,16 +53,16 @@ abstract class VideoMediaEncoder extends MediaEncoder { protected void onPrepare(@NonNull MediaEncoderEngine.Controller controller, long maxLengthMillis) { MediaFormat format = MediaFormat.createVideoFormat(mConfig.mimeType, mConfig.width, mConfig.height); - // Set some properties. Failing to specify some of these can cause the MediaCodec - // configure() call to throw an unhelpful exception. + // Failing to specify some of these can cause the MediaCodec configure() call to throw an unhelpful exception. + // About COLOR_FormatSurface, see https://stackoverflow.com/q/28027858/4288782 + // This just means it is an opaque, implementation-specific format that the device GPU prefers. + // So as long as we use the GPU to draw, the format will match what the encoder expects. format.setInteger(MediaFormat.KEY_COLOR_FORMAT, MediaCodecInfo.CodecCapabilities.COLOR_FormatSurface); format.setInteger(MediaFormat.KEY_BIT_RATE, mConfig.bitRate); format.setInteger(MediaFormat.KEY_FRAME_RATE, mConfig.frameRate); - format.setInteger(MediaFormat.KEY_I_FRAME_INTERVAL, 2); + format.setInteger(MediaFormat.KEY_I_FRAME_INTERVAL, 5); format.setInteger("rotation-degrees", mConfig.rotation); - // Create a MediaCodec encoder, and configure it with our format. Get a Surface - // we can use for input and wrap it with a class that handles the EGL work. try { mMediaCodec = MediaCodec.createEncoderByType(mConfig.mimeType); } catch (IOException e) { diff --git a/demo/src/main/AndroidManifest.xml b/demo/src/main/AndroidManifest.xml index 18f0871a..00e973a9 100644 --- a/demo/src/main/AndroidManifest.xml +++ b/demo/src/main/AndroidManifest.xml @@ -1,5 +1,6 @@ @@ -8,9 +9,9 @@ android:allowBackup="false" android:icon="@mipmap/cameraview" android:label="@string/app_name" - android:roundIcon="@mipmap/cameraview" android:supportsRtl="true" - android:theme="@style/AppTheme"> + android:theme="@style/AppTheme" + tools:ignore="GoogleAppIndexingWarning"> image; + private static PictureResult picture; - public static void setPictureResult(@Nullable PictureResult im) { - image = im != null ? new WeakReference<>(im) : null; + public static void setPictureResult(@Nullable PictureResult pictureResult) { + picture = pictureResult; } @Override protected void onCreate(@Nullable Bundle savedInstanceState) { super.onCreate(savedInstanceState); setContentView(R.layout.activity_picture_preview); - final ImageView imageView = findViewById(R.id.image); - final MessageView captureResolution = findViewById(R.id.nativeCaptureResolution); - final MessageView captureLatency = findViewById(R.id.captureLatency); - final MessageView exifRotation = findViewById(R.id.exifRotation); - PictureResult result = image == null ? null : image.get(); + final PictureResult result = picture; if (result == null) { finish(); return; } + + final ImageView imageView = findViewById(R.id.image); + final MessageView captureResolution = findViewById(R.id.nativeCaptureResolution); + final MessageView captureLatency = findViewById(R.id.captureLatency); + final MessageView exifRotation = findViewById(R.id.exifRotation); + final long delay = getIntent().getLongExtra("delay", 0); AspectRatio ratio = AspectRatio.of(result.getSize()); captureLatency.setTitleAndMessage("Approx. latency", delay + " milliseconds"); diff --git a/demo/src/main/java/com/otaliastudios/cameraview/demo/VideoPreviewActivity.java b/demo/src/main/java/com/otaliastudios/cameraview/demo/VideoPreviewActivity.java index d7921671..1d26fcc9 100644 --- a/demo/src/main/java/com/otaliastudios/cameraview/demo/VideoPreviewActivity.java +++ b/demo/src/main/java/com/otaliastudios/cameraview/demo/VideoPreviewActivity.java @@ -13,23 +13,28 @@ import android.widget.MediaController; import android.widget.VideoView; import com.otaliastudios.cameraview.VideoResult; - -import java.lang.ref.WeakReference; +import com.otaliastudios.cameraview.size.AspectRatio; public class VideoPreviewActivity extends Activity { private VideoView videoView; - private static WeakReference videoResult; + private static VideoResult videoResult; public static void setVideoResult(@Nullable VideoResult result) { - videoResult = result != null ? new WeakReference<>(result) : null; + videoResult = result; } @Override protected void onCreate(@Nullable Bundle savedInstanceState) { super.onCreate(savedInstanceState); setContentView(R.layout.activity_video_preview); + final VideoResult result = videoResult; + if (result == null) { + finish(); + return; + } + videoView = findViewById(R.id.video); videoView.setOnClickListener(new View.OnClickListener() { @Override @@ -46,13 +51,8 @@ public class VideoPreviewActivity extends Activity { final MessageView videoBitRate = findViewById(R.id.videoBitRate); final MessageView videoFrameRate = findViewById(R.id.videoFrameRate); - final VideoResult result = videoResult == null ? null : videoResult.get(); - if (result == null) { - finish(); - return; - } - - actualResolution.setTitleAndMessage("Size", result.getSize() + ""); + AspectRatio ratio = AspectRatio.of(result.getSize()); + actualResolution.setTitleAndMessage("Size", result.getSize() + " (" + ratio + ")"); isSnapshot.setTitleAndMessage("Snapshot", result.isSnapshot() + ""); rotation.setTitleAndMessage("Rotation", result.getRotation() + ""); audio.setTitleAndMessage("Audio", result.getAudio().name()); @@ -85,8 +85,9 @@ public class VideoPreviewActivity extends Activity { } void playVideo() { - if (videoView.isPlaying()) return; - videoView.start(); + if (!videoView.isPlaying()) { + videoView.start(); + } } @Override diff --git a/demo/src/main/res/layout/activity_camera.xml b/demo/src/main/res/layout/activity_camera.xml index 1f94941c..714e8dfb 100644 --- a/demo/src/main/res/layout/activity_camera.xml +++ b/demo/src/main/res/layout/activity_camera.xml @@ -23,7 +23,7 @@ app:cameraGrid="off" app:cameraFlash="off" app:cameraAudio="on" - app:cameraFacing="front" + app:cameraFacing="back" app:cameraGestureTap="autoFocus" app:cameraGestureLongTap="none" app:cameraGesturePinch="zoom"