AuroraMiddleware/skia2
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

SkAR-Java: finger painting first implementation

Browse Source 
SkAR Java: drawing planes as paths


FREEZE.unindexed

Bug: skia:
Change-Id: I2a7a8534fc1c35c4b8d6054bc1d6e682ffabc47a
Reviewed-on: https://skia-review.googlesource.com/141827
Reviewed-by: Mike Reed <reed@google.com>
This commit is contained in:
ziadb 2018-07-17 15:30:55 -04:00 committed by Ziad Ben Hadj-Alouane
parent dc97fdc235
commit a44a8a16a8
4 changed files with 180 additions and 2 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
platform_tools/android/apps/skar_java/src/main/AndroidManifest.xml
View File

@ -46,5 +46,9 @@
<!-- This tag indicates that this application requires ARCore. This results in the Google Play
Store downloading and installing ARCore along with the application. -->
<meta-data android:name="com.google.ar.core" android:value="required" />
<meta-data
android:name="com.google.ar.core.min_apk_version"
tools:replace="android:value"
android:value="180226000" />
</application>
</manifest>

47
platform_tools/android/apps/skar_java/src/main/java/com/google/ar/core/examples/java/helloskar/DrawManager.java
View File

@ -1,12 +1,14 @@
package com.google.ar.core.examples.java.helloskar;
import android.content.Context;
import android.content.pm.ApplicationInfo;
import android.graphics.Bitmap;
import android.graphics.BitmapFactory;
import android.graphics.BitmapShader;
import android.graphics.Canvas;
import android.graphics.Color;
import android.graphics.ColorFilter;
import android.graphics.LinearGradient;
import android.graphics.Paint;
import android.graphics.Path;
import android.graphics.PointF;
@ -15,10 +17,13 @@ import android.graphics.PorterDuffColorFilter;
import android.graphics.RectF;
import android.graphics.Shader;
import android.opengl.Matrix;
import android.os.Build;
import com.google.ar.core.Plane;
import com.google.ar.core.PointCloud;
import com.google.ar.core.Pose;
import com.google.ar.core.TrackingState;
import com.google.skar.SkARFingerPainting;
import com.google.skar.SkARMatrix;
import com.google.skar.SkARUtil;
import java.io.IOException;
@ -40,6 +45,7 @@ public class DrawManager {
private ColorFilter lightFilter;
private BitmapShader planeShader;
public ArrayList<float[]> modelMatrices = new ArrayList<>();
public SkARFingerPainting fingerPainting = new SkARFingerPainting();
public void updateViewport(float width, float height) {
viewportWidth = width;
@ -58,6 +64,10 @@ public class DrawManager {
lightFilter = SkARUtil.createLightCorrectionColorFilter(colorCorr);
}
public void updateFingerPainting(PointF p) {
fingerPainting.addPoint(p);
}
// Sample function for drawing a circle
public void drawCircle(Canvas canvas) {
if (modelMatrices.isEmpty()) {
@ -129,6 +139,41 @@ public class DrawManager {
canvas.restore();
}
public void drawFingerPainting(Canvas canvas) {
if (fingerPainting.path.isEmpty()) {
return;
}
// Get finger painting model matrix
float[] m = fingerPainting.getModelMatrix();
float[] model = new float[16];
Matrix.setIdentityM(model, 0);
Matrix.translateM(model, 0, m[12], m[13], m[14]);
float[] initRot = SkARMatrix.createXYtoXZRotationMatrix();
// Matrix = mvpv
float[][] matrices = {initRot, model, viewMatrix, projectionMatrix, SkARMatrix.createViewportMatrix(viewportWidth, viewportHeight)};
android.graphics.Matrix mvpv = SkARMatrix.createMatrixFrom4x4(SkARMatrix.multiplyMatrices4x4(matrices));
// Set up paint
Paint p = new Paint();
p.setColor(Color.GREEN);
p.setStyle(Paint.Style.STROKE);
p.setStrokeWidth(10f);
p.setAlpha(120);
// Build destination path by transforming source path
Path pathDst = new Path();
fingerPainting.path.transform(mvpv, pathDst);
// Draw dest path
canvas.save();
canvas.setMatrix(new android.graphics.Matrix());
canvas.drawPath(pathDst, p);
canvas.restore();
}
// Sample function for drawing the AR point cloud
public void drawPointCloud(Canvas canvas, PointCloud cloud) {
FloatBuffer points = cloud.getPoints();
@ -224,7 +269,7 @@ public class DrawManager {
// Shader local matrix
android.graphics.Matrix lm = new android.graphics.Matrix();
lm.setScale(0.0005f, 0.0005f);
lm.setScale(0.00005f, 0.00005f);
lm.postConcat(mvpv);
planeShader.setLocalMatrix(lm);

75
platform_tools/android/apps/skar_java/src/main/java/com/google/ar/core/examples/java/helloskar/HelloSkARActivity.java
View File

@ -20,14 +20,17 @@ import android.animation.PropertyValuesHolder;
import android.animation.ValueAnimator;
import android.graphics.Canvas;
import android.graphics.Color;
import android.graphics.PointF;
import android.graphics.PorterDuff;
import android.opengl.GLES20;
import android.opengl.GLSurfaceView;
import android.opengl.Matrix;
import android.os.Bundle;
import android.support.v7.app.AppCompatActivity;
import android.util.Log;
import android.view.MotionEvent;
import android.view.SurfaceHolder;
import android.view.View;
import android.widget.Toast;
import com.google.ar.core.Anchor;
@ -54,6 +57,7 @@ import com.google.ar.core.exceptions.UnavailableArcoreNotInstalledException;
import com.google.ar.core.exceptions.UnavailableDeviceNotCompatibleException;
import com.google.ar.core.exceptions.UnavailableSdkTooOldException;
import com.google.ar.core.exceptions.UnavailableUserDeclinedInstallationException;
import com.google.skar.SkARMatrix;
import java.io.IOException;
import java.util.ArrayList;
@ -72,6 +76,8 @@ public class HelloSkARActivity extends AppCompatActivity implements GLSurfaceVie
//Simple SurfaceView used to draw 2D objects on top of the GLSurfaceView
private ARSurfaceView arSurfaceView;
private Canvas canvas;
private SurfaceHolder holder;
//GLSurfaceView used to draw 3D objects & camera input
private GLSurfaceView glSurfaceView;
@ -93,6 +99,11 @@ public class HelloSkARActivity extends AppCompatActivity implements GLSurfaceVie
// Temporary matrix allocated here to reduce number of allocations for each frame.
private final float[] anchorMatrix = new float[16];
private final float[] back = new float[16];
PointF previousEvent;
android.graphics.Matrix inverted;
// Anchors created from taps used for object placing.
private final ArrayList<Anchor> anchors = new ArrayList<>();
@ -108,6 +119,7 @@ public class HelloSkARActivity extends AppCompatActivity implements GLSurfaceVie
arSurfaceView = findViewById(R.id.arsurfaceview);
glSurfaceView = findViewById(R.id.glsurfaceview);
arSurfaceView.bringToFront();
arSurfaceView.setLayerType(View.LAYER_TYPE_HARDWARE, null);
displayRotationHelper = new DisplayRotationHelper(/*context=*/ this);
// Set up tap listener.
@ -259,6 +271,9 @@ public class HelloSkARActivity extends AppCompatActivity implements GLSurfaceVie
@Override
public void onDrawFrame(GL10 gl) {
canvas = null;
holder = null;
// Clear screen to notify driver it should not load any pixels from previous frame.
GLES20.glClear(GLES20.GL_COLOR_BUFFER_BIT | GLES20.GL_DEPTH_BUFFER_BIT);
@ -269,6 +284,7 @@ public class HelloSkARActivity extends AppCompatActivity implements GLSurfaceVie
// the video background can be properly adjusted.
displayRotationHelper.updateSessionIfNeeded(session);
try {
session.setCameraTextureName(backgroundRenderer.getTextureId());
Frame frame = session.update();
@ -297,6 +313,54 @@ public class HelloSkARActivity extends AppCompatActivity implements GLSurfaceVie
}
}
MotionEvent holdTap = tapHelper.holdPoll();
if (holdTap != null && camera.getTrackingState() == TrackingState.TRACKING) {
for (HitResult hit : frame.hitTest(holdTap)) {
// Check if any plane was hit, and if it was hit inside the plane polygon
Trackable trackable = hit.getTrackable();
// Creates an anchor if a plane or an oriented point was hit.
if ((trackable instanceof Plane
&& ((Plane) trackable).isPoseInPolygon(hit.getHitPose())
&& (DrawManager.calculateDistanceToPlane(hit.getHitPose(), camera.getPose())
> 0))
|| (trackable instanceof Point
&& ((Point) trackable).getOrientationMode()
== OrientationMode.ESTIMATED_SURFACE_NORMAL)) {
float[] pt = {hit.getHitPose().tx(), hit.getHitPose().tz()};
if (drawManager.fingerPainting.isEmpty()) {
float[] originalPt = {pt[0], pt[1]};
// Get model matrix of first point
float[] m = new float[16];
hit.getHitPose().toMatrix(m, 0);
drawManager.fingerPainting.setModelMatrix(m); //M0
// Construct the inverse matrix + the translation to the origin
float[] inv = new float[16];
hit.getHitPose().toMatrix(inv, 0);
Matrix.invertM(inv, 0, inv, 0);
drawManager.fingerPainting.setInverseModelMatrix(inv);
//inverted = SkARMatrix.createMatrixFrom4x4(inv); //M0 -1
// Map hit location using the raw inverse matrix
drawManager.fingerPainting.getInverseModelMatrix().mapPoints(originalPt);
// Translate the point back to the origin, and update the inverse matrix
Matrix.translateM(inv, 0, -originalPt[0], -originalPt[1], 0);
drawManager.fingerPainting.setInverseModelMatrix(inv);
//inverted = SkARMatrix.createMatrixFrom4x4(inv);
}
drawManager.fingerPainting.getInverseModelMatrix().mapPoints(pt);
PointF newPoint = new PointF(pt[0], pt[1]);
drawManager.fingerPainting.addPoint(newPoint);
break;
}
}
}
// Draw background with OpenGL.
// TODO: possibly find a way to extract texture and draw on Canvas
backgroundRenderer.draw(frame);
@ -350,12 +414,17 @@ public class HelloSkARActivity extends AppCompatActivity implements GLSurfaceVie
// Draw models
drawModels(canvas);
// Draw finger painting
drawFingerPainting(canvas);
// Unlock canvas
holder.unlockCanvasAndPost(canvas);
}
} catch (Throwable t) {
// Avoid crashing the application due to unhandled exceptions.
if (holder != null && canvas != null) {
holder.unlockCanvasAndPost(canvas);
}
Log.e(TAG, "Exception on the OpenGL thread", t);
}
}
@ -385,4 +454,8 @@ public class HelloSkARActivity extends AppCompatActivity implements GLSurfaceVie
drawManager.drawText(canvas, "HelloSkAR");
}
}
private void drawFingerPainting(Canvas canvas) {
drawManager.drawFingerPainting(canvas);
}
}

56
platform_tools/android/apps/skar_java/src/main/java/com/google/skar/SkARFingerPainting.java Normal file
View File

@ -0,0 +1,56 @@
package com.google.skar;
import android.graphics.Matrix;
import android.graphics.Path;
import android.graphics.PointF;
public class SkARFingerPainting { ;
public Path path = new Path();
private int numberOfPoints = 0;
// Holds the model matrix of the first point added to such that the path can be drawn at the
// model location (i.e on the Plane)
private float[] modelMatrix;
// Holds the inverse model matrix of the first point that was added such that the path is drawn
// first at (0, 0)
private float[] inverseModelMatrix;
public SkARFingerPainting() {}
// Adds another point to the path in Local space (i.e apply InverseModelMatrix to points located
// in Global space (e.g hit positions acquired through hit tests)
public void addPoint(PointF p) {
if (numberOfPoints == 0) {
path.moveTo(p.x, p.y);
} else {
path.lineTo(p.x, p.y);
}
numberOfPoints++;
}
public boolean isEmpty() {
return numberOfPoints == 0;
}
public float[] getModelMatrix() {
return modelMatrix;
}
public float[] getRawInverseModelMatrix() {
return inverseModelMatrix;
}
public Matrix getInverseModelMatrix() {
return SkARMatrix.createMatrixFrom4x4(inverseModelMatrix);
}
public void setModelMatrix(float[] m) {
modelMatrix = m;
}
public void setInverseModelMatrix(float[] m) {
inverseModelMatrix = m;
}
}