remoting/android/java/src/org/chromium/chromoting/CardboardDesktopRenderer.java

   1 // Copyright 2015 The Chromium Authors. All rights reserved.
   2 // Use of this source code is governed by a BSD-style license that can be
   3 // found in the LICENSE file.
   4
   5 package org.chromium.chromoting;
   6
   7 import android.app.Activity;
   8 import android.graphics.Bitmap;
   9 import android.graphics.Point;
  10 import android.graphics.PointF;
  11 import android.opengl.GLES20;
  12 import android.opengl.Matrix;
  13
  14 import com.google.vrtoolkit.cardboard.CardboardView;
  15 import com.google.vrtoolkit.cardboard.Eye;
  16 import com.google.vrtoolkit.cardboard.HeadTransform;
  17 import com.google.vrtoolkit.cardboard.Viewport;
  18
  19 import org.chromium.chromoting.jni.JniInterface;
  20
  21 import java.nio.ByteBuffer;
  22 import java.nio.ByteOrder;
  23 import java.nio.FloatBuffer;
  24
  25 import javax.microedition.khronos.egl.EGLConfig;
  26
  27 /**
  28  * Renderer for Cardboard view.
  29  */
  30 public class CardboardDesktopRenderer implements CardboardView.StereoRenderer {
  31     private static final String TAG = "cr.CardboardRenderer";
  32
  33     private static final int BYTE_PER_FLOAT = 4;
  34     private static final int POSITION_DATA_SIZE = 3;
  35     private static final int TEXTURE_COORDINATE_DATA_SIZE = 2;
  36     private static final float Z_NEAR = 0.1f;
  37     private static final float Z_FAR = 100.0f;
  38     private static final float DESKTOP_POSITION_X = 0.0f;
  39     private static final float DESKTOP_POSITION_Y = 0.0f;
  40     private static final float DESKTOP_POSITION_Z = -2.0f;
  41     private static final float HALF_SKYBOX_SIZE = 100.0f;
  42     private static final float VIEW_POSITION_MIN = -1.0f;
  43     private static final float VIEW_POSITION_MAX = 3.0f;
  44
  45     // Allows user to click even when looking outside the desktop
  46     // but within edge margin.
  47     private static final float EDGE_MARGIN = 0.1f;
  48
  49     // Distance to move camera each time.
  50     private static final float CAMERA_MOTION_STEP = 0.5f;
  51
  52     private final Activity mActivity;
  53
  54     private float mCameraPosition;
  55
  56     // Lock to allow multithreaded access to mCameraPosition.
  57     private final Object mCameraPositionLock = new Object();
  58
  59     private float[] mCameraMatrix;
  60     private float[] mViewMatrix;
  61     private float[] mProjectionMatrix;
  62
  63     // Make matrix member variable to avoid unnecessary initialization.
  64     private float[] mDesktopModelMatrix;
  65     private float[] mDesktopCombinedMatrix;
  66     private float[] mEyePointModelMatrix;
  67     private float[] mEyePointCombinedMatrix;
  68     private float[] mSkyboxCombinedMatrix;
  69
  70     // Direction that user is looking towards.
  71     private float[] mForwardVector;
  72
  73     // Eye position in desktop.
  74     private float[] mEyePositionVector;
  75
  76     private CardboardActivityDesktop mDesktop;
  77     private CardboardActivityEyePoint mEyePoint;
  78     private CardboardActivitySkybox mSkybox;
  79
  80     // Flag to indicate whether reload the desktop texture or not.
  81     private boolean mReloadTexture;
  82
  83     /** Lock to allow multithreaded access to mReloadTexture. */
  84     private final Object mReloadTextureLock = new Object();
  85
  86     // Lock for eye position related operations.
  87     // This protects access to mEyePositionVector.
  88     private final Object mEyePositionLock = new Object();
  89
  90     public CardboardDesktopRenderer(Activity activity) {
  91         mActivity = activity;
  92         mReloadTexture = false;
  93         mCameraPosition = 0.0f;
  94
  95         mCameraMatrix = new float[16];
  96         mViewMatrix = new float[16];
  97         mProjectionMatrix = new float[16];
  98         mDesktopModelMatrix = new float[16];
  99         mDesktopCombinedMatrix = new float[16];
 100         mEyePointModelMatrix = new float[16];
 101         mEyePointCombinedMatrix = new float[16];
 102         mSkyboxCombinedMatrix = new float[16];
 103
 104         mForwardVector = new float[3];
 105         mEyePositionVector = new float[3];
 106
 107         attachRedrawCallback();
 108     }
 109
 110     // This can be called on any thread.
 111     public void attachRedrawCallback() {
 112         JniInterface.provideRedrawCallback(new Runnable() {
 113             @Override
 114             public void run() {
 115                 synchronized (mReloadTextureLock) {
 116                     mReloadTexture = true;
 117                 }
 118             }
 119         });
 120     }
 121
 122     @Override
 123     public void onSurfaceCreated(EGLConfig config) {
 124         // Set the background clear color to black.
 125         GLES20.glClearColor(0.0f, 0.0f, 0.0f, 0.0f);
 126
 127         // Use culling to remove back faces.
 128         GLES20.glEnable(GLES20.GL_CULL_FACE);
 129
 130         // Enable depth testing.
 131         GLES20.glEnable(GLES20.GL_DEPTH_TEST);
 132
 133         mDesktop = new CardboardActivityDesktop();
 134         mEyePoint = new CardboardActivityEyePoint();
 135         mSkybox = new CardboardActivitySkybox(mActivity);
 136     }
 137
 138     @Override
 139     public void onSurfaceChanged(int width, int height) {
 140     }
 141
 142     @Override
 143     public void onNewFrame(HeadTransform headTransform) {
 144         // Position the eye at the origin.
 145         float eyeX = 0.0f;
 146         float eyeY = 0.0f;
 147         float eyeZ;
 148         synchronized (mCameraPositionLock) {
 149             eyeZ = mCameraPosition;
 150         }
 151
 152         // We are looking toward the negative Z direction.
 153         float lookX = DESKTOP_POSITION_X;
 154         float lookY = DESKTOP_POSITION_Y;
 155         float lookZ = DESKTOP_POSITION_Z;
 156
 157         // Set our up vector. This is where our head would be pointing were we holding the camera.
 158         float upX = 0.0f;
 159         float upY = 1.0f;
 160         float upZ = 0.0f;
 161
 162         Matrix.setLookAtM(mCameraMatrix, 0, eyeX, eyeY, eyeZ, lookX, lookY, lookZ, upX, upY, upZ);
 163
 164         headTransform.getForwardVector(mForwardVector, 0);
 165         getLookingPosition();
 166         maybeLoadDesktopTexture();
 167         mSkybox.maybeLoadTextureAndCleanImages();
 168     }
 169
 170     @Override
 171     public void onDrawEye(Eye eye) {
 172         GLES20.glClear(GLES20.GL_COLOR_BUFFER_BIT | GLES20.GL_DEPTH_BUFFER_BIT);
 173
 174         // Apply the eye transformation to the camera.
 175         Matrix.multiplyMM(mViewMatrix, 0, eye.getEyeView(), 0, mCameraMatrix, 0);
 176
 177         mProjectionMatrix = eye.getPerspective(Z_NEAR, Z_FAR);
 178
 179         drawSkybox();
 180         drawDesktop();
 181         drawEyePoint();
 182     }
 183
 184     @Override
 185     public void onRendererShutdown() {
 186         mDesktop.cleanup();
 187         mEyePoint.cleanup();
 188         mSkybox.cleanup();
 189     }
 190
 191     @Override
 192     public void onFinishFrame(Viewport viewport) {
 193     }
 194
 195     private void drawDesktop() {
 196         if (!mDesktop.hasVideoFrame()) {
 197             // This can happen if the client is connected, but a complete
 198             // video frame has not yet been decoded.
 199             return;
 200         }
 201
 202         Matrix.setIdentityM(mDesktopModelMatrix, 0);
 203         Matrix.translateM(mDesktopModelMatrix, 0, DESKTOP_POSITION_X,
 204                 DESKTOP_POSITION_Y, DESKTOP_POSITION_Z);
 205
 206         // Pass in Model View Matrix and Model View Project Matrix.
 207         Matrix.multiplyMM(mDesktopCombinedMatrix, 0, mViewMatrix, 0, mDesktopModelMatrix, 0);
 208         Matrix.multiplyMM(mDesktopCombinedMatrix, 0, mProjectionMatrix,
 209                 0, mDesktopCombinedMatrix, 0);
 210         mDesktop.setCombinedMatrix(mDesktopCombinedMatrix);
 211
 212         mDesktop.draw();
 213     }
 214
 215     private void drawEyePoint() {
 216         if (!isLookingAtDesktop()) {
 217             return;
 218         }
 219
 220         float eyePointX = clamp(mEyePositionVector[0], -mDesktop.getHalfWidth(),
 221                 mDesktop.getHalfWidth());
 222         float eyePointY = clamp(mEyePositionVector[1], -mDesktop.getHalfHeight(),
 223                 mDesktop.getHalfHeight());
 224         Matrix.setIdentityM(mEyePointModelMatrix, 0);
 225         Matrix.translateM(mEyePointModelMatrix, 0, -eyePointX, -eyePointY,
 226                 DESKTOP_POSITION_Z);
 227         Matrix.multiplyMM(mEyePointCombinedMatrix, 0, mViewMatrix, 0, mEyePointModelMatrix, 0);
 228         Matrix.multiplyMM(mEyePointCombinedMatrix, 0, mProjectionMatrix,
 229                 0, mEyePointCombinedMatrix, 0);
 230
 231         mEyePoint.setCombinedMatrix(mEyePointCombinedMatrix);
 232         mEyePoint.draw();
 233     }
 234
 235     private void drawSkybox() {
 236         // Since we will always put the skybox center in the origin, so skybox
 237         // model matrix will always be identity matrix which we could ignore.
 238         Matrix.multiplyMM(mSkyboxCombinedMatrix, 0, mProjectionMatrix,
 239                 0, mViewMatrix, 0);
 240         mSkybox.draw(mSkyboxCombinedMatrix);
 241     }
 242
 243     /**
 244      * Returns coordinates in units of pixels in the desktop bitmap.
 245      * This can be called on any thread.
 246      */
 247     public PointF getMouseCoordinates() {
 248         PointF result = new PointF();
 249         Point shapePixels = mDesktop.getFrameSizePixels();
 250         int heightPixels = shapePixels.x;
 251         int widthPixels = shapePixels.y;
 252
 253         synchronized (mEyePositionLock) {
 254             // Due to the coordinate direction, we only have to inverse x.
 255             result.x = (-mEyePositionVector[0] + mDesktop.getHalfWidth())
 256                     / (2 * mDesktop.getHalfWidth()) * widthPixels;
 257             result.y = (mEyePositionVector[1] + mDesktop.getHalfHeight())
 258                     / (2 * mDesktop.getHalfHeight()) * heightPixels;
 259             result.x = clamp(result.x, 0, widthPixels);
 260             result.y = clamp(result.y, 0, heightPixels);
 261         }
 262         return result;
 263     }
 264
 265     /**
 266      * Returns the passed in value if it resides within the specified range (inclusive).  If not,
 267      * it will return the closest boundary from the range.  The ordering of the boundary values
 268      * does not matter.
 269      *
 270      * @param value The value to be compared against the range.
 271      * @param a First boundary range value.
 272      * @param b Second boundary range value.
 273      * @return The passed in value if it is within the range, otherwise the closest boundary value.
 274      */
 275     private static float clamp(float value, float a, float b) {
 276         float min = (a > b) ? b : a;
 277         float max = (a > b) ? a : b;
 278         if (value < min) {
 279             value = min;
 280         } else if (value > max) {
 281             value = max;
 282         }
 283         return value;
 284     }
 285
 286     /**
 287      * Move the camera towards desktop.
 288      * This method can be called on any thread.
 289      */
 290     public void moveTowardsDesktop() {
 291         synchronized (mCameraPositionLock) {
 292             float newPosition = mCameraPosition - CAMERA_MOTION_STEP;
 293             if (newPosition >= VIEW_POSITION_MIN) {
 294                 mCameraPosition = newPosition;
 295             }
 296         }
 297     }
 298
 299     /**
 300      * Move the camera away from desktop.
 301      * This method can be called on any thread.
 302      */
 303     public void moveAwayFromDesktop() {
 304         synchronized (mCameraPositionLock) {
 305             float newPosition = mCameraPosition + CAMERA_MOTION_STEP;
 306             if (newPosition <= VIEW_POSITION_MAX) {
 307                 mCameraPosition = newPosition;
 308             }
 309         }
 310     }
 311
 312     /**
 313      * Return true if user is looking at the desktop.
 314      * This method can be called on any thread.
 315      */
 316     public boolean isLookingAtDesktop() {
 317         synchronized (mEyePositionLock) {
 318             return Math.abs(mEyePositionVector[0]) <= (mDesktop.getHalfWidth() + EDGE_MARGIN)
 319                 && Math.abs(mEyePositionVector[1]) <= (mDesktop.getHalfHeight() + EDGE_MARGIN);
 320         }
 321     }
 322
 323     /**
 324      * Return true if user is looking at the space to the left of the desktop.
 325      * This method can be called on any thread.
 326      */
 327     public boolean isLookingLeftOfDesktop() {
 328         synchronized (mEyePositionLock) {
 329             return mEyePositionVector[0] >= (mDesktop.getHalfWidth() + EDGE_MARGIN);
 330         }
 331     }
 332
 333     /**
 334      * Return true if user is looking at the space to the right of the desktop.
 335      * This method can be called on any thread.
 336      */
 337     public boolean isLookingRightOfDesktop() {
 338         synchronized (mEyePositionLock) {
 339             return mEyePositionVector[0] <= -(mDesktop.getHalfWidth() + EDGE_MARGIN);
 340         }
 341     }
 342
 343     /**
 344      * Return true if user is looking at the space above the desktop.
 345      * This method can be called on any thread.
 346      */
 347     public boolean isLookingAboveDesktop() {
 348         synchronized (mEyePositionLock) {
 349             return mEyePositionVector[1] <= -(mDesktop.getHalfHeight() + EDGE_MARGIN);
 350         }
 351     }
 352
 353     /**
 354      * Return true if user is looking at the space below the desktop.
 355      * This method can be called on any thread.
 356      */
 357     public boolean isLookingBelowDesktop() {
 358         synchronized (mEyePositionLock) {
 359             return mEyePositionVector[1] >= (mDesktop.getHalfHeight() + EDGE_MARGIN);
 360         }
 361     }
 362
 363     /**
 364      * Get position on desktop where user is looking at.
 365      */
 366     private void getLookingPosition() {
 367         synchronized (mEyePositionLock) {
 368             if (Math.abs(mForwardVector[2]) < 0.00001f) {
 369                 mEyePositionVector[0] = Math.signum(mForwardVector[0]) * Float.MAX_VALUE;
 370                 mEyePositionVector[1] = Math.signum(mForwardVector[1]) * Float.MAX_VALUE;
 371             } else {
 372                 mEyePositionVector[0] = mForwardVector[0] * DESKTOP_POSITION_Z / mForwardVector[2];
 373                 mEyePositionVector[1] = mForwardVector[1] * DESKTOP_POSITION_Z / mForwardVector[2];
 374             }
 375             mEyePositionVector[2] = DESKTOP_POSITION_Z;
 376         }
 377     }
 378
 379     /**
 380      * Link desktop texture with {@link CardboardActivityDesktop} if {@link mReloadTexture} is true.
 381      * @param textureDataHandle the handle we want attach texture to
 382      */
 383     private void maybeLoadDesktopTexture() {
 384         synchronized (mReloadTextureLock) {
 385             if (!mReloadTexture) {
 386                 return;
 387             }
 388         }
 389
 390         // TODO(shichengfeng): Record the time desktop drawing takes.
 391         Bitmap bitmap = JniInterface.getVideoFrame();
 392
 393         if (bitmap == null) {
 394             // This can happen if the client is connected, but a complete video frame has not yet
 395             // been decoded.
 396             return;
 397         }
 398
 399         mDesktop.updateVideoFrame(bitmap);
 400
 401         synchronized (mReloadTextureLock) {
 402             mReloadTexture = false;
 403         }
 404     }
 405
 406     /**
 407      * Convert float array to a FloatBuffer for use in OpenGL calls.
 408      */
 409     public static FloatBuffer makeFloatBuffer(float[] data) {
 410         FloatBuffer result = ByteBuffer
 411                 .allocateDirect(data.length * BYTE_PER_FLOAT)
 412                 .order(ByteOrder.nativeOrder()).asFloatBuffer();
 413         result.put(data).position(0);
 414         return result;
 415     }
 416 }