DepthImageRenderer.cpp

   1 /***********************************************************************
   2 DepthImageRenderer - Class to centralize storage of raw or filtered
   3 depth images on the GPU, and perform simple repetitive rendering tasks
   4 such as rendering elevation values into a frame buffer.
   5 Copyright (c) 2014-2018 Oliver Kreylos
   6
   7 This file is part of the Augmented Reality Sandbox (SARndbox).
   8
   9 The Augmented Reality Sandbox is free software; you can redistribute it
  10 and/or modify it under the terms of the GNU General Public License as
  11 published by the Free Software Foundation; either version 2 of the
  12 License, or (at your option) any later version.
  13
  14 The Augmented Reality Sandbox is distributed in the hope that it will be
  15 useful, but WITHOUT ANY WARRANTY; without even the implied warranty of
  16 MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
  17 General Public License for more details.
  18
  19 You should have received a copy of the GNU General Public License along
  20 with the Augmented Reality Sandbox; if not, write to the Free Software
  21 Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307 USA
  22 ***********************************************************************/
  23
  24 #include "DepthImageRenderer.h"
  25
  26 #include <GL/gl.h>
  27 #include <GL/GLVertexArrayParts.h>
  28 #include <GL/GLContextData.h>
  29 #include <GL/Extensions/GLARBFragmentShader.h>
  30 #include <GL/Extensions/GLARBMultitexture.h>
  31 #include <GL/Extensions/GLARBShaderObjects.h>
  32 #include <GL/Extensions/GLARBTextureFloat.h>
  33 #include <GL/Extensions/GLARBTextureRectangle.h>
  34 #include <GL/Extensions/GLARBTextureRg.h>
  35 #include <GL/Extensions/GLARBVertexBufferObject.h>
  36 #include <GL/Extensions/GLARBVertexShader.h>
  37 #include <GL/GLTransformationWrappers.h>
  38
  39 #include "ShaderHelper.h"
  40
  41 /*********************************************
  42 Methods of class DepthImageRenderer::DataItem:
  43 *********************************************/
  44
  45 DepthImageRenderer::DataItem::DataItem(void)
  46     : vertexBuffer(0), indexBuffer(0),
  47       depthTexture(0), depthTextureVersion(0),
  48       depthShader(0), elevationShader(0) {
  49     /* Initialize all required extensions: */
  50     GLARBFragmentShader::initExtension();
  51     GLARBMultitexture::initExtension();
  52     GLARBShaderObjects::initExtension();
  53     GLARBTextureFloat::initExtension();
  54     GLARBTextureRectangle::initExtension();
  55     GLARBTextureRg::initExtension();
  56     GLARBVertexBufferObject::initExtension();
  57     GLARBVertexShader::initExtension();
  58
  59     /* Allocate the buffers and textures: */
  60     glGenBuffersARB(1, &vertexBuffer);
  61     glGenBuffersARB(1, &indexBuffer);
  62     glGenTextures(1, &depthTexture);
  63 }
  64
  65 DepthImageRenderer::DataItem::~DataItem(void) {
  66     /* Release all allocated buffers, textures, and shaders: */
  67     glDeleteBuffersARB(1, &vertexBuffer);
  68     glDeleteBuffersARB(1, &indexBuffer);
  69     glDeleteTextures(1, &depthTexture);
  70     glDeleteObjectARB(depthShader);
  71     glDeleteObjectARB(elevationShader);
  72 }
  73
  74 /***********************************
  75 Methods of class DepthImageRenderer:
  76 ***********************************/
  77
  78 DepthImageRenderer::DepthImageRenderer(const unsigned int sDepthImageSize[2])
  79     : depthImageVersion(0) {
  80     /* Copy the depth image size: */
  81     for(int i = 0; i < 2; ++i)
  82         depthImageSize[i] = sDepthImageSize[i];
  83
  84     /* Initialize the depth image: */
  85     depthImage = Kinect::FrameBuffer(depthImageSize[0], depthImageSize[1],
  86                                      depthImageSize[1] * depthImageSize[0] * sizeof(float));
  87     float* diPtr = depthImage.getData<float>();
  88     for(unsigned int y = 0; y < depthImageSize[1]; ++y)
  89         for(unsigned int x = 0; x < depthImageSize[0]; ++x, ++diPtr)
  90             *diPtr = 0.0f;
  91     ++depthImageVersion;
  92 }
  93
  94 void DepthImageRenderer::initContext(GLContextData& contextData) const {
  95     /* Create a data item and add it to the context: */
  96     DataItem* dataItem = new DataItem;
  97     contextData.addDataItem(this, dataItem);
  98
  99     /* Upload the grid of template vertices into the vertex buffer: */
 100     glBindBufferARB(GL_ARRAY_BUFFER_ARB, dataItem->vertexBuffer);
 101     glBufferDataARB(GL_ARRAY_BUFFER_ARB, depthImageSize[1]*depthImageSize[0]*sizeof(Vertex), 0,
 102                     GL_STATIC_DRAW_ARB);
 103     Vertex* vPtr = static_cast<Vertex*>(glMapBufferARB(GL_ARRAY_BUFFER_ARB, GL_WRITE_ONLY_ARB));
 104     if(lensDistortion.isIdentity()) {
 105         /* Create uncorrected pixel positions: */
 106         for(unsigned int y = 0; y < depthImageSize[1]; ++y)
 107             for(unsigned int x = 0; x < depthImageSize[0]; ++x, ++vPtr) {
 108                 vPtr->position[0] = Scalar(x) + Scalar(0.5);
 109                 vPtr->position[1] = Scalar(y) + Scalar(0.5);
 110             }
 111     } else {
 112         /* Create lens distortion-corrected pixel positions: */
 113         for(unsigned int y = 0; y < depthImageSize[1]; ++y)
 114             for(unsigned int x = 0; x < depthImageSize[0]; ++x, ++vPtr) {
 115                 /* Undistort the image point: */
 116                 Kinect::LensDistortion::Point dp(Kinect::LensDistortion::Scalar(x) +
 117                                                  Kinect::LensDistortion::Scalar(0.5),
 118                                                  Kinect::LensDistortion::Scalar(y) + Kinect::LensDistortion::Scalar(0.5));
 119                 Kinect::LensDistortion::Point up = lensDistortion.undistortPixel(dp);
 120
 121                 /* Store the undistorted point: */
 122                 vPtr->position[0] = Scalar(up[0]);
 123                 vPtr->position[1] = Scalar(up[1]);
 124             }
 125     }
 126     glUnmapBufferARB(GL_ARRAY_BUFFER_ARB);
 127     glBindBufferARB(GL_ARRAY_BUFFER_ARB, 0);
 128
 129     /* Upload the surface's triangle indices into the index buffer: */
 130     glBindBufferARB(GL_ELEMENT_ARRAY_BUFFER_ARB, dataItem->indexBuffer);
 131     glBufferDataARB(GL_ELEMENT_ARRAY_BUFFER_ARB,
 132                     (depthImageSize[1] - 1)*depthImageSize[0] * 2 * sizeof(GLuint), 0, GL_STATIC_DRAW_ARB);
 133     GLuint* iPtr = static_cast<GLuint*>(glMapBufferARB(GL_ELEMENT_ARRAY_BUFFER_ARB,
 134                                         GL_WRITE_ONLY_ARB));
 135     for(unsigned int y = 1; y < depthImageSize[1]; ++y)
 136         for(unsigned int x = 0; x < depthImageSize[0]; ++x, iPtr += 2) {
 137             iPtr[0] = GLuint(y * depthImageSize[0] + x);
 138             iPtr[1] = GLuint((y - 1) * depthImageSize[0] + x);
 139         }
 140     glUnmapBufferARB(GL_ELEMENT_ARRAY_BUFFER_ARB);
 141     glBindBufferARB(GL_ELEMENT_ARRAY_BUFFER_ARB, 0);
 142
 143     /* Initialize the depth image texture: */
 144     glBindTexture(GL_TEXTURE_RECTANGLE_ARB, dataItem->depthTexture);
 145     glTexParameteri(GL_TEXTURE_RECTANGLE_ARB, GL_TEXTURE_MIN_FILTER, GL_NEAREST);
 146     glTexParameteri(GL_TEXTURE_RECTANGLE_ARB, GL_TEXTURE_MAG_FILTER, GL_NEAREST);
 147     glTexParameteri(GL_TEXTURE_RECTANGLE_ARB, GL_TEXTURE_WRAP_S, GL_CLAMP);
 148     glTexParameteri(GL_TEXTURE_RECTANGLE_ARB, GL_TEXTURE_WRAP_T, GL_CLAMP);
 149     glTexImage2D(GL_TEXTURE_RECTANGLE_ARB, 0, GL_LUMINANCE32F_ARB, depthImageSize[0],
 150                  depthImageSize[1], 0, GL_LUMINANCE, GL_FLOAT, 0);
 151     glBindTexture(GL_TEXTURE_RECTANGLE_ARB, 0);
 152
 153     /* Create the depth rendering shader: */
 154     dataItem->depthShader = linkVertexAndFragmentShader("SurfaceDepthShader");
 155     dataItem->depthShaderUniforms[0] = glGetUniformLocationARB(dataItem->depthShader, "depthSampler");
 156     dataItem->depthShaderUniforms[1] = glGetUniformLocationARB(dataItem->depthShader,
 157                                        "projectionModelviewDepthProjection");
 158
 159     /* Create the elevation rendering shader: */
 160     dataItem->elevationShader = linkVertexAndFragmentShader("SurfaceElevationShader");
 161     dataItem->elevationShaderUniforms[0] = glGetUniformLocationARB(dataItem->elevationShader,
 162                                            "depthSampler");
 163     dataItem->elevationShaderUniforms[1] = glGetUniformLocationARB(dataItem->elevationShader,
 164                                            "basePlaneDic");
 165     dataItem->elevationShaderUniforms[2] = glGetUniformLocationARB(dataItem->elevationShader,
 166                                            "weightDic");
 167     dataItem->elevationShaderUniforms[3] = glGetUniformLocationARB(dataItem->elevationShader,
 168                                            "projectionModelviewDepthProjection");
 169 }
 170
 171 void DepthImageRenderer::setDepthProjection(const PTransform& newDepthProjection) {
 172     /* Set the depth unprojection matrix: */
 173     depthProjection = newDepthProjection;
 174
 175     /* Convert the depth projection matrix to column-major OpenGL format: */
 176     GLfloat* dpmPtr = depthProjectionMatrix;
 177     for(int j = 0; j < 4; ++j)
 178         for(int i = 0; i < 4; ++i, ++dpmPtr)
 179             *dpmPtr = GLfloat(depthProjection.getMatrix()(i, j));
 180
 181     /* Create the weight calculation equation: */
 182     for(int i = 0; i < 4; ++i)
 183         weightDicEq[i] = GLfloat(depthProjection.getMatrix()(3, i));
 184
 185     /* Recalculate the base plane equation in depth image space: */
 186     setBasePlane(basePlane);
 187 }
 188
 189 void DepthImageRenderer::setIntrinsics(const Kinect::FrameSource::IntrinsicParameters& ips) {
 190     /* Set the lens distortion parameters: */
 191     lensDistortion = ips.depthLensDistortion;
 192
 193     /* Set the depth unprojection matrix: */
 194     depthProjection = ips.depthProjection;
 195
 196     /* Convert the depth projection matrix to column-major OpenGL format: */
 197     GLfloat* dpmPtr = depthProjectionMatrix;
 198     for(int j = 0; j < 4; ++j)
 199         for(int i = 0; i < 4; ++i, ++dpmPtr)
 200             *dpmPtr = GLfloat(depthProjection.getMatrix()(i, j));
 201
 202     /* Create the weight calculation equation: */
 203     for(int i = 0; i < 4; ++i)
 204         weightDicEq[i] = GLfloat(depthProjection.getMatrix()(3, i));
 205
 206     /* Recalculate the base plane equation in depth image space: */
 207     setBasePlane(basePlane);
 208 }
 209
 210 void DepthImageRenderer::setBasePlane(const Plane& newBasePlane) {
 211     /* Set the base plane: */
 212     basePlane = newBasePlane;
 213
 214     /* Transform the base plane to depth image space and into a GLSL-compatible format: */
 215     const PTransform::Matrix& dpm = depthProjection.getMatrix();
 216     const Plane::Vector& bpn = basePlane.getNormal();
 217     Scalar bpo = basePlane.getOffset();
 218     for(int i = 0; i < 4; ++i)
 219         basePlaneDicEq[i] = GLfloat(dpm(0, i) * bpn[0] + dpm(1, i) * bpn[1] + dpm(2, i) * bpn[2] - dpm(3,
 220                                     i) * bpo);
 221 }
 222
 223 void DepthImageRenderer::setDepthImage(const Kinect::FrameBuffer& newDepthImage) {
 224     /* Update the depth image: */
 225     depthImage = newDepthImage;
 226     ++depthImageVersion;
 227 }
 228
 229 Scalar DepthImageRenderer::intersectLine(const Point& p0, const Point& p1, Scalar elevationMin,
 230         Scalar elevationMax) const {
 231     /* Initialize the line segment: */
 232     Scalar lambda0 = Scalar(0);
 233     Scalar lambda1 = Scalar(1);
 234
 235     /* Intersect the line segment with the upper elevation plane: */
 236     Scalar d0 = basePlane.calcDistance(p0);
 237     Scalar d1 = basePlane.calcDistance(p1);
 238     if(d0 * d1 < Scalar(0)) {
 239         /* Calculate the intersection parameter: */
 240
 241         // IMPLEMENT ME!
 242
 243         return Scalar(2);
 244     } else if(d1 > Scalar(0)) {
 245         /* Trivially reject with maximum intercept: */
 246         return Scalar(2);
 247     }
 248
 249     return Scalar(2);
 250 }
 251
 252 void DepthImageRenderer::uploadDepthProjection(GLint location) const {
 253     /* Upload the matrix to OpenGL: */
 254     glUniformMatrix4fvARB(location, 1, GL_FALSE, depthProjectionMatrix);
 255 }
 256
 257 void DepthImageRenderer::bindDepthTexture(GLContextData& contextData) const {
 258     /* Get the data item: */
 259     DataItem* dataItem = contextData.retrieveDataItem<DataItem>(this);
 260
 261     /* Bind the depth image texture: */
 262     glBindTexture(GL_TEXTURE_RECTANGLE_ARB, dataItem->depthTexture);
 263
 264     /* Check if the texture is outdated: */
 265     if(dataItem->depthTextureVersion != depthImageVersion) {
 266         /* Upload the new depth texture: */
 267         glTexSubImage2D(GL_TEXTURE_RECTANGLE_ARB, 0, 0, 0, depthImageSize[0], depthImageSize[1],
 268                         GL_LUMINANCE, GL_FLOAT, depthImage.getData<GLfloat>());
 269
 270         /* Mark the depth texture as current: */
 271         dataItem->depthTextureVersion = depthImageVersion;
 272     }
 273 }
 274
 275 void DepthImageRenderer::renderSurfaceTemplate(GLContextData& contextData) const {
 276     /* Get the data item: */
 277     DataItem* dataItem = contextData.retrieveDataItem<DataItem>(this);
 278
 279     /* Bind the vertex and index buffers: */
 280     glBindBufferARB(GL_ARRAY_BUFFER_ARB, dataItem->vertexBuffer);
 281     glBindBufferARB(GL_ELEMENT_ARRAY_BUFFER_ARB, dataItem->indexBuffer);
 282
 283     /* Draw the surface template: */
 284     GLVertexArrayParts::enable(Vertex::getPartsMask());
 285     glVertexPointer(static_cast<const Vertex*>(0));
 286     GLuint* indexPtr = 0;
 287     for(unsigned int y = 1; y < depthImageSize[1]; ++y, indexPtr += depthImageSize[0] * 2)
 288         glDrawElements(GL_QUAD_STRIP, depthImageSize[0] * 2, GL_UNSIGNED_INT, indexPtr);
 289     GLVertexArrayParts::disable(Vertex::getPartsMask());
 290
 291     /* Unbind the vertex and index buffers: */
 292     glBindBufferARB(GL_ARRAY_BUFFER_ARB, 0);
 293     glBindBufferARB(GL_ELEMENT_ARRAY_BUFFER_ARB, 0);
 294 }
 295
 296 void DepthImageRenderer::renderDepth(const PTransform& projectionModelview,
 297                                      GLContextData& contextData) const {
 298     /* Get the data item: */
 299     DataItem* dataItem = contextData.retrieveDataItem<DataItem>(this);
 300
 301     /* Bind the depth rendering shader: */
 302     glUseProgramObjectARB(dataItem->depthShader);
 303
 304     /* Bind the vertex and index buffers: */
 305     glBindBufferARB(GL_ARRAY_BUFFER_ARB, dataItem->vertexBuffer);
 306     glBindBufferARB(GL_ELEMENT_ARRAY_BUFFER_ARB, dataItem->indexBuffer);
 307
 308     /* Bind the depth image texture: */
 309     glActiveTextureARB(GL_TEXTURE0_ARB);
 310     glBindTexture(GL_TEXTURE_RECTANGLE_ARB, dataItem->depthTexture);
 311
 312     /* Check if the texture is outdated: */
 313     if(dataItem->depthTextureVersion != depthImageVersion) {
 314         /* Upload the new depth texture: */
 315         glTexSubImage2D(GL_TEXTURE_RECTANGLE_ARB, 0, 0, 0, depthImageSize[0], depthImageSize[1],
 316                         GL_LUMINANCE, GL_FLOAT, depthImage.getData<GLfloat>());
 317
 318         /* Mark the depth texture as current: */
 319         dataItem->depthTextureVersion = depthImageVersion;
 320     }
 321     glUniform1iARB(dataItem->depthShaderUniforms[0],
 322                    0); // Tell the shader that the depth texture is in texture unit 0
 323
 324     /* Upload the combined projection, modelview, and depth projection matrix: */
 325     PTransform pmvdp = projectionModelview;
 326     pmvdp *= depthProjection;
 327     glUniformARB(dataItem->depthShaderUniforms[1], pmvdp);
 328
 329     /* Draw the surface: */
 330     GLVertexArrayParts::enable(Vertex::getPartsMask());
 331     glVertexPointer(static_cast<const Vertex*>(0));
 332     GLuint* indexPtr = 0;
 333     for(unsigned int y = 1; y < depthImageSize[1]; ++y, indexPtr += depthImageSize[0] * 2)
 334         glDrawElements(GL_QUAD_STRIP, depthImageSize[0] * 2, GL_UNSIGNED_INT, indexPtr);
 335     GLVertexArrayParts::disable(Vertex::getPartsMask());
 336
 337     /* Unbind all textures and buffers: */
 338     glBindTexture(GL_TEXTURE_RECTANGLE_ARB, 0);
 339     glBindBufferARB(GL_ARRAY_BUFFER_ARB, 0);
 340     glBindBufferARB(GL_ELEMENT_ARRAY_BUFFER_ARB, 0);
 341
 342     /* Unbind the depth rendering shader: */
 343     glUseProgramObjectARB(0);
 344 }
 345
 346 void DepthImageRenderer::renderElevation(const PTransform& projectionModelview,
 347         GLContextData& contextData) const {
 348     /* Get the data item: */
 349     DataItem* dataItem = contextData.retrieveDataItem<DataItem>(this);
 350
 351     /* Bind the elevation rendering shader: */
 352     glUseProgramObjectARB(dataItem->elevationShader);
 353
 354     /* Set up the depth image texture: */
 355     glActiveTextureARB(GL_TEXTURE0_ARB);
 356     glBindTexture(GL_TEXTURE_RECTANGLE_ARB, dataItem->depthTexture);
 357
 358     /* Check if the texture is outdated: */
 359     if(dataItem->depthTextureVersion != depthImageVersion) {
 360         /* Upload the new depth texture: */
 361         glTexSubImage2D(GL_TEXTURE_RECTANGLE_ARB, 0, 0, 0, depthImageSize[0], depthImageSize[1],
 362                         GL_LUMINANCE, GL_FLOAT, depthImage.getData<GLfloat>());
 363
 364         /* Mark the depth texture as current: */
 365         dataItem->depthTextureVersion = depthImageVersion;
 366     }
 367     glUniform1iARB(dataItem->elevationShaderUniforms[0],
 368                    0); // Tell the shader that the depth texture is in texture unit 0
 369
 370     /* Upload the base plane equation in depth image space: */
 371     glUniformARB<4>(dataItem->elevationShaderUniforms[1], 1, basePlaneDicEq);
 372
 373     /* Upload the base weight equation in depth image space: */
 374     glUniformARB<4>(dataItem->elevationShaderUniforms[2], 1, weightDicEq);
 375
 376     /* Upload the combined projection, modelview, and depth projection matrix: */
 377     PTransform pmvdp = projectionModelview;
 378     pmvdp *= depthProjection;
 379     glUniformARB(dataItem->elevationShaderUniforms[3], pmvdp);
 380
 381     /* Bind the vertex and index buffers: */
 382     glBindBufferARB(GL_ARRAY_BUFFER_ARB, dataItem->vertexBuffer);
 383     glBindBufferARB(GL_ELEMENT_ARRAY_BUFFER_ARB, dataItem->indexBuffer);
 384
 385     /* Draw the surface: */
 386     GLVertexArrayParts::enable(Vertex::getPartsMask());
 387     glVertexPointer(static_cast<const Vertex*>(0));
 388     GLuint* indexPtr = 0;
 389     for(unsigned int y = 1; y < depthImageSize[1]; ++y, indexPtr += depthImageSize[0] * 2)
 390         glDrawElements(GL_QUAD_STRIP, depthImageSize[0] * 2, GL_UNSIGNED_INT, indexPtr);
 391     GLVertexArrayParts::disable(Vertex::getPartsMask());
 392
 393     /* Unbind all textures and buffers: */
 394     glBindBufferARB(GL_ARRAY_BUFFER_ARB, 0);
 395     glBindBufferARB(GL_ELEMENT_ARRAY_BUFFER_ARB, 0);
 396     glBindTexture(GL_TEXTURE_RECTANGLE_ARB, 0);
 397
 398     /* Unbind the elevation rendering shader: */
 399     glUseProgramObjectARB(0);
 400 }