AuroraMiddleware/skia2
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

SkAR Java: better finger painting. Cleaner UI

Browse Source 
Bug: skia:
Change-Id: If3b595982deb42326213f2feffdddcaa46a5d8ff
Reviewed-on: https://skia-review.googlesource.com/142506
Reviewed-by: Mike Reed <reed@google.com>
This commit is contained in:
ziadb 2018-07-19 13:38:38 -04:00 committed by Ziad Ben Hadj-Alouane
parent 9c4dfadabd
commit 7ae4fcad7b
6 changed files with 229 additions and 156 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
platform_tools/android/apps/skar_java/src/main/AndroidManifest.xml
View File

@ -28,7 +28,7 @@
android:allowBackup="false"
android:icon="@drawable/ic_launcher"
android:label="@string/app_name"
android:theme="@style/AppTheme"
android:theme="@style/Theme.AppCompat.Light.NoActionBar"
android:usesCleartextTraffic="false"
tools:ignore="GoogleAppIndexingWarning">
@ -36,7 +36,7 @@
android:name="com.google.ar.core.examples.java.helloskar.HelloSkARActivity"
android:configChanges="orientation|screenSize"
android:exported="true"
android:theme="@style/Theme.AppCompat.NoActionBar"
android:theme="@style/Theme.AppCompat.DayNight.NoActionBar"
android:screenOrientation="locked">
<intent-filter>
<action android:name="android.intent.action.MAIN"/>

142
platform_tools/android/apps/skar_java/src/main/java/com/google/ar/core/examples/java/helloskar/DrawManager.java
View File

@ -18,6 +18,7 @@ import android.graphics.RectF;
import android.graphics.Shader;
import android.opengl.Matrix;
import android.os.Build;
import android.util.Log;
import com.google.ar.core.Plane;
import com.google.ar.core.PointCloud;
@ -78,8 +79,10 @@ public class DrawManager {
p.setARGB(180, 100, 0, 0);
canvas.save();
canvas.setMatrix(SkARMatrix.createPerspectiveMatrix(modelMatrices.get(0),
viewMatrix, projectionMatrix, viewportWidth, viewportHeight));
android.graphics.Matrix m = SkARMatrix.createPerspectiveMatrix(modelMatrices.get(0),
viewMatrix, projectionMatrix, viewportWidth, viewportHeight);
canvas.setMatrix(m);
canvas.drawCircle(0, 0, 0.1f, p);
canvas.restore();
}
@ -145,33 +148,46 @@ public class DrawManager {
}
// Get finger painting model matrix
float[] m = fingerPainting.getModelMatrix();
float[] model = new float[16];
Matrix.setIdentityM(model, 0);
Matrix.translateM(model, 0, m[12], m[13], m[14]);
float[] model = fingerPainting.getModelMatrix();
float[] in = new float[16];
Matrix.setIdentityM(in, 0);
Matrix.translateM(in, 0, model[12], model[13], model[14]);
float[] initRot = SkARMatrix.createXYtoXZRotationMatrix();
float[] scale = new float[16];
float s = 0.001f;
Matrix.setIdentityM(scale, 0);
Matrix.scaleM(scale, 0, s, s, s);
// Matrix = mvpv
float[][] matrices = {initRot, model, viewMatrix, projectionMatrix, SkARMatrix.createViewportMatrix(viewportWidth, viewportHeight)};
float[][] matrices = {scale, initRot, in, viewMatrix, projectionMatrix, SkARMatrix.createViewportMatrix(viewportWidth, viewportHeight)};
android.graphics.Matrix mvpv = SkARMatrix.createMatrixFrom4x4(SkARMatrix.multiplyMatrices4x4(matrices));
// Set up paint
Paint p = new Paint();
p.setColor(Color.GREEN);
p.setStyle(Paint.Style.STROKE);
p.setStrokeWidth(10f);
p.setStrokeWidth(30f);
p.setAlpha(120);
// Build destination path by transforming source path
Path pathDst = new Path();
fingerPainting.path.transform(mvpv, pathDst);
if (true) {
// Transform applied through canvas
canvas.save();
canvas.setMatrix(mvpv);
canvas.drawPath(fingerPainting.path, p);
canvas.restore();
} else {
// Transform path directly
Path pathDst = new Path();
fingerPainting.path.transform(mvpv, pathDst);
// Draw dest path
canvas.save();
canvas.setMatrix(new android.graphics.Matrix());
canvas.drawPath(pathDst, p);
canvas.restore();
// Draw dest path
canvas.save();
canvas.setMatrix(new android.graphics.Matrix());
canvas.drawPath(pathDst, p);
canvas.restore();
}
}
// Sample function for drawing the AR point cloud
@ -233,12 +249,7 @@ public class DrawManager {
float[][] matrices = {initRot, model, viewMatrix, projectionMatrix, SkARMatrix.createViewportMatrix(viewportWidth, viewportHeight)};
android.graphics.Matrix mvpv = SkARMatrix.createMatrixFrom4x4(SkARMatrix.multiplyMatrices4x4(matrices));
if (Build.VERSION.SDK_INT > Build.VERSION_CODES.O_MR1) {
// Android version P and higher
drawPlane(canvas, mvpv, plane);
} else {
drawPlaneAsPath(canvas, mvpv, plane);
}
drawPlaneAsPath(canvas, mvpv, plane);
}
}
@ -261,23 +272,38 @@ public class DrawManager {
p.setShader(planeShader);
p.setColorFilter(new PorterDuffColorFilter(Color.argb(0.4f, 1, 0, 0),
PorterDuff.Mode.SRC_ATOP));
p.setAlpha(120);
// Build destination path by transforming source path
Path pathDst = new Path();
pathSrc.transform(mvpv, pathDst);
p.setColor(Color.RED);
p.setAlpha(100);
// Shader local matrix
android.graphics.Matrix lm = new android.graphics.Matrix();
lm.setScale(0.00005f, 0.00005f);
lm.postConcat(mvpv);
planeShader.setLocalMatrix(lm);
if (true) {
// Shader local matrix
android.graphics.Matrix lm = new android.graphics.Matrix();
lm.setScale(0.00005f, 0.00005f);
planeShader.setLocalMatrix(lm);
// Draw dest path
canvas.save();
canvas.setMatrix(new android.graphics.Matrix());
canvas.drawPath(pathDst, p);
canvas.restore();
// Draw dest path
canvas.save();
canvas.setMatrix(mvpv);
canvas.drawPath(pathSrc, p);
canvas.restore();
} else {
// Build destination path by transforming source path
Path pathDst = new Path();
pathSrc.transform(mvpv, pathDst);
// Shader local matrix
android.graphics.Matrix lm = new android.graphics.Matrix();
lm.setScale(0.00005f, 0.00005f);
lm.postConcat(mvpv);
planeShader.setLocalMatrix(lm);
// Draw dest path
canvas.save();
canvas.setMatrix(new android.graphics.Matrix());
canvas.drawPath(pathDst, p);
canvas.restore();
}
}
public void initializePlaneShader(Context context, String gridDistanceTextureName) throws IOException {
@ -286,6 +312,7 @@ public class DrawManager {
BitmapFactory.decodeStream(context.getAssets().open(gridDistanceTextureName));
// Set up the shader
planeShader = new BitmapShader(planeTexture, Shader.TileMode.REPEAT, Shader.TileMode.REPEAT);
planeShader.setLocalMatrix(new android.graphics.Matrix());
}
private float[] getTextScaleMatrix(float size) {
@ -308,47 +335,4 @@ public class DrawManager {
+ (cameraY - planePose.ty()) * normal[1]
+ (cameraZ - planePose.tz()) * normal[2];
}
// Drawing plane with drawVertices
// TODO: Wait until latest Android release for this to work..
private void drawPlane(Canvas canvas, android.graphics.Matrix mvpv, Plane plane) {
int vertsSize = plane.getPolygon().limit() / 2;
FloatBuffer polygon = plane.getPolygon();
float[] polyVerts = new float[vertsSize * 2];
int[] polyColors = new int[vertsSize];
for (int i = 0; i < vertsSize; i++) {
polyVerts[i * 2] = polygon.get(i * 2);
polyVerts[i * 2 + 1] = polygon.get(i * 2 + 1);
polyColors[i] = Color.RED;
}
// Construct indices through a list
ArrayList<Short> indices = new ArrayList<>();
for (int i = 1; i < vertsSize - 1; ++i) {
indices.add((short) 0);
indices.add((short) i);
indices.add((short) (i + 1));
}
// Copy indices into an array
short[] indicesArray = new short[indices.size()];
for (int i = 0; i < indices.size(); i++) {
indicesArray[i] = indices.get(i);
}
Paint p = new Paint();
p.setShader(planeShader);
p.setColor(Color.RED);
p.setAlpha(100);
canvas.save();
canvas.setMatrix(mvpv);
canvas.drawVertices(Canvas.VertexMode.TRIANGLE_FAN, vertsSize, polyVerts, 0,
null, 0, polyColors, 0, indicesArray, 0,
indicesArray.length, p);
canvas.restore();
}
}

171
platform_tools/android/apps/skar_java/src/main/java/com/google/ar/core/examples/java/helloskar/HelloSkARActivity.java
View File

@ -27,10 +27,15 @@ import android.opengl.GLSurfaceView;
import android.opengl.Matrix;
import android.os.Bundle;
import android.support.v7.app.AppCompatActivity;
import android.support.v7.widget.Toolbar;
import android.util.Log;
import android.view.Menu;
import android.view.MenuInflater;
import android.view.MenuItem;
import android.view.MotionEvent;
import android.view.SurfaceHolder;
import android.view.View;
import android.view.WindowManager;
import android.widget.Toast;
import com.google.ar.core.Anchor;
@ -57,7 +62,6 @@ import com.google.ar.core.exceptions.UnavailableArcoreNotInstalledException;
import com.google.ar.core.exceptions.UnavailableDeviceNotCompatibleException;
import com.google.ar.core.exceptions.UnavailableSdkTooOldException;
import com.google.ar.core.exceptions.UnavailableUserDeclinedInstallationException;
import com.google.skar.SkARMatrix;
import java.io.IOException;
import java.util.ArrayList;
@ -72,6 +76,10 @@ import javax.microedition.khronos.opengles.GL10;
*/
public class HelloSkARActivity extends AppCompatActivity implements GLSurfaceView.Renderer {
public enum DrawingType {
circle, rect, text, animation
}
private static final String TAG = HelloSkARActivity.class.getSimpleName();
//Simple SurfaceView used to draw 2D objects on top of the GLSurfaceView
@ -95,14 +103,12 @@ public class HelloSkARActivity extends AppCompatActivity implements GLSurfaceVie
// 2D Renderer
private DrawManager drawManager = new DrawManager();
private DrawingType currentDrawabletype = DrawingType.circle;
// Temporary matrix allocated here to reduce number of allocations for each frame.
private final float[] anchorMatrix = new float[16];
private final float[] back = new float[16];
PointF previousEvent;
android.graphics.Matrix inverted;
PointF previousEvent;;
// Anchors created from taps used for object placing.
private final ArrayList<Anchor> anchors = new ArrayList<>();
@ -116,6 +122,15 @@ public class HelloSkARActivity extends AppCompatActivity implements GLSurfaceVie
protected void onCreate(Bundle savedInstanceState) {
super.onCreate(savedInstanceState);
setContentView(R.layout.activity_main);
Toolbar myToolbar = (Toolbar) findViewById(R.id.my_toolbar);
setSupportActionBar(myToolbar);
//hide notifications bar
getWindow().setFlags(WindowManager.LayoutParams.FLAG_FULLSCREEN,
WindowManager.LayoutParams.FLAG_FULLSCREEN);
arSurfaceView = findViewById(R.id.arsurfaceview);
glSurfaceView = findViewById(R.id.glsurfaceview);
arSurfaceView.bringToFront();
@ -313,54 +328,6 @@ public class HelloSkARActivity extends AppCompatActivity implements GLSurfaceVie
}
}
MotionEvent holdTap = tapHelper.holdPoll();
if (holdTap != null && camera.getTrackingState() == TrackingState.TRACKING) {
for (HitResult hit : frame.hitTest(holdTap)) {
// Check if any plane was hit, and if it was hit inside the plane polygon
Trackable trackable = hit.getTrackable();
// Creates an anchor if a plane or an oriented point was hit.
if ((trackable instanceof Plane
&& ((Plane) trackable).isPoseInPolygon(hit.getHitPose())
&& (DrawManager.calculateDistanceToPlane(hit.getHitPose(), camera.getPose())
> 0))
|| (trackable instanceof Point
&& ((Point) trackable).getOrientationMode()
== OrientationMode.ESTIMATED_SURFACE_NORMAL)) {
float[] pt = {hit.getHitPose().tx(), hit.getHitPose().tz()};
if (drawManager.fingerPainting.isEmpty()) {
float[] originalPt = {pt[0], pt[1]};
// Get model matrix of first point
float[] m = new float[16];
hit.getHitPose().toMatrix(m, 0);
drawManager.fingerPainting.setModelMatrix(m); //M0
// Construct the inverse matrix + the translation to the origin
float[] inv = new float[16];
hit.getHitPose().toMatrix(inv, 0);
Matrix.invertM(inv, 0, inv, 0);
drawManager.fingerPainting.setInverseModelMatrix(inv);
//inverted = SkARMatrix.createMatrixFrom4x4(inv); //M0 -1
// Map hit location using the raw inverse matrix
drawManager.fingerPainting.getInverseModelMatrix().mapPoints(originalPt);
// Translate the point back to the origin, and update the inverse matrix
Matrix.translateM(inv, 0, -originalPt[0], -originalPt[1], 0);
drawManager.fingerPainting.setInverseModelMatrix(inv);
//inverted = SkARMatrix.createMatrixFrom4x4(inv);
}
drawManager.fingerPainting.getInverseModelMatrix().mapPoints(pt);
PointF newPoint = new PointF(pt[0], pt[1]);
drawManager.fingerPainting.addPoint(newPoint);
break;
}
}
}
// Draw background with OpenGL.
// TODO: possibly find a way to extract texture and draw on Canvas
backgroundRenderer.draw(frame);
@ -384,6 +351,54 @@ public class HelloSkARActivity extends AppCompatActivity implements GLSurfaceVie
frame.getLightEstimate().getColorCorrection(colorCorrectionRgba, 0);
drawManager.updateLightColorFilter(colorCorrectionRgba);
// Building finger painting
MotionEvent holdTap = tapHelper.holdPoll();
if (holdTap != null && camera.getTrackingState() == TrackingState.TRACKING) {
for (HitResult hit : frame.hitTest(holdTap)) {
// Check if any plane was hit, and if it was hit inside the plane polygon
Trackable trackable = hit.getTrackable();
// Creates an anchor if a plane or an oriented point was hit.
if ((trackable instanceof Plane
&& ((Plane) trackable).isPoseInPolygon(hit.getHitPose())
&& (DrawManager.calculateDistanceToPlane(hit.getHitPose(), camera.getPose())
> 0))
|| (trackable instanceof Point
&& ((Point) trackable).getOrientationMode()
== OrientationMode.ESTIMATED_SURFACE_NORMAL)) {
// Get hit point transform, apply it to the origin
float[] gm = new float[16];
hit.getHitPose().toMatrix(gm, 0);
float[] point = {0, 0, 0, 1};
Matrix.multiplyMV(point, 0, gm, 0, point, 0);
if (drawManager.fingerPainting.isEmpty()) {
drawManager.fingerPainting.addPoint(new PointF(0, 0));
// Get model matrix of first point
float[] m = new float[16];
hit.getHitPose().toMatrix(m, 0);
drawManager.fingerPainting.setModelMatrix(m);
} else {
float localDistanceScale = 1000;
PointF distance = new PointF(point[0] - previousEvent.x,
point[2] - previousEvent.y);
// New point is distance + old point
PointF p = new PointF(distance.x * localDistanceScale
+ drawManager.fingerPainting.previousPoint.x,
distance.y * localDistanceScale
+ drawManager.fingerPainting.previousPoint.y);
drawManager.fingerPainting.addPoint(p);
}
previousEvent = new PointF(point[0], point[2]);
break;
}
}
}
// Drawing on Canvas (SurfaceView)
if (arSurfaceView.isRunning()) {
// Lock canvas
@ -448,14 +463,56 @@ public class HelloSkARActivity extends AppCompatActivity implements GLSurfaceVie
anchor.getPose().toMatrix(anchorMatrix, 0);
drawManager.modelMatrices.add(0, anchorMatrix);
drawManager.drawRect(canvas);
drawManager.drawCircle(canvas);
drawManager.drawAnimatedRoundRect(canvas, radius);
drawManager.drawText(canvas, "HelloSkAR");
switch (currentDrawabletype) {
case circle:
drawManager.drawCircle(canvas);
return;
case rect:
drawManager.drawRect(canvas);
return;
case animation:
drawManager.drawAnimatedRoundRect(canvas, radius);
return;
case text:
drawManager.drawText(canvas, "Android");
return;
default:
drawManager.drawCircle(canvas);
return;
}
}
}
private void drawFingerPainting(Canvas canvas) {
drawManager.drawFingerPainting(canvas);
}
// Menu functions
public boolean onCreateOptionsMenu(Menu menu) {
MenuInflater inflater = getMenuInflater();
inflater.inflate(R.menu.main_menu, menu);
return true;
}
public boolean onOptionsItemSelected(MenuItem item) {
switch (item.getItemId()) {
case R.id.reset_paint:
drawManager.fingerPainting.reset();
return true;
case R.id.draw_circle:
currentDrawabletype = DrawingType.circle;
return true;
case R.id.draw_rect:
currentDrawabletype = DrawingType.rect;
return true;
case R.id.draw_animation:
currentDrawabletype = DrawingType.animation;
return true;
case R.id.draw_text:
currentDrawabletype = DrawingType.text;
return true;
default:
return super.onOptionsItemSelected(item);
}
}
}

27
platform_tools/android/apps/skar_java/src/main/java/com/google/skar/SkARFingerPainting.java
View File

@ -1,32 +1,30 @@
package com.google.skar;
import android.graphics.Matrix;
import android.graphics.Path;
import android.graphics.PointF;
public class SkARFingerPainting { ;
public class SkARFingerPainting {
public Path path = new Path();
// Previous point added to the path. This points belongs to the path in local space.
public PointF previousPoint;
private int numberOfPoints = 0;
// Holds the model matrix of the first point added to such that the path can be drawn at the
// model location (i.e on the Plane)
private float[] modelMatrix;
// Holds the inverse model matrix of the first point that was added such that the path is drawn
// first at (0, 0)
private float[] inverseModelMatrix;
public SkARFingerPainting() {}
// Adds another point to the path in Local space (i.e apply InverseModelMatrix to points located
// in Global space (e.g hit positions acquired through hit tests)
// Adds another point to the path in Local space
public void addPoint(PointF p) {
if (numberOfPoints == 0) {
path.moveTo(p.x, p.y);
} else {
path.lineTo(p.x, p.y);
}
previousPoint = p;
numberOfPoints++;
}
@ -38,19 +36,12 @@ public class SkARFingerPainting { ;
return modelMatrix;
}
public float[] getRawInverseModelMatrix() {
return inverseModelMatrix;
}
public Matrix getInverseModelMatrix() {
return SkARMatrix.createMatrixFrom4x4(inverseModelMatrix);
}
public void setModelMatrix(float[] m) {
modelMatrix = m;
}
public void setInverseModelMatrix(float[] m) {
inverseModelMatrix = m;
public void reset() {
path = new Path();
numberOfPoints = 0;
}
}

9
platform_tools/android/apps/skar_java/src/main/res/layout/activity_main.xml
View File

@ -17,12 +17,21 @@
xmlns:tools="http://schemas.android.com/tools"
android:layout_width="match_parent"
android:layout_height="match_parent"
xmlns:app="http://schemas.android.com/apk/res-auto"
tools:context="com.google.ar.core.examples.java.helloskar.HelloSkARActivity">
<FrameLayout
android:layout_width="match_parent"
android:layout_height="match_parent">
<android.support.v7.widget.Toolbar
android:id="@+id/my_toolbar"
android:layout_width="match_parent"
android:layout_height="?attr/actionBarSize"
android:background="?attr/colorPrimary"
android:elevation="4dp"
android:theme="@style/ThemeOverlay.AppCompat.ActionBar"
app:popupTheme="@style/ThemeOverlay.AppCompat.Light"/>
<com.google.ar.core.examples.java.helloskar.ARSurfaceView
android:id="@+id/arsurfaceview"

32
platform_tools/android/apps/skar_java/src/main/res/menu/main_menu.xml Normal file
View File

@ -0,0 +1,32 @@
<?xml version="1.0" encoding="utf-8"?>
<menu xmlns:android="http://schemas.android.com/apk/res/android">
<group
android:id="@+id/menu_top">
<item
android:id="@+id/reset_paint"
android:title="Reset Finger Painting"/>
</group>
<group
android:id="@+id/menu_bottom"
android:checkableBehavior="single">
<item
android:id="@+id/draw_circle"
android:title="Circle" >
</item>
<item
android:id="@+id/draw_rect"
android:title="Rect" >
</item>
<item
android:id="@+id/draw_animation"
android:title="Animation" >
</item>
<item
android:id="@+id/draw_text"
android:title="Text" >
</item>
</group>
</menu>