title: OpenCV Unity - Get 3D position from 2D image position
body: <p>I'm programming an Hololens application and I'm trying to do a real time tracking system. I will use OpenCV for the computer vision algorithm. I saw there that this is possible to get a contour of a color object -> <a href="https://www.youtube.com/watch?v=hQ-bpfdWQh8&amp;t=26s" rel="nofollow noreferrer">https://www.youtube.com/watch?v=hQ-bpfdWQh8&amp;t=26s</a></p>
  
  <p>So with this solution I can get the contour of a color zone and then have the middle pixel position of this zone, right but now how to convert that 2D position in a 3D position in the coordinate system of the Hololens?</p>
  
  <p>I saw there some solution for pose estimation -> <a href="http://opencv-python-tutroals.readthedocs.io/en/latest/py_tutorials/py_calib3d/py_pose/py_pose.html" rel="nofollow noreferrer">http://opencv-python-tutroals.readthedocs.io/en/latest/py_tutorials/py_calib3d/py_pose/py_pose.html</a></p>
  
  <p>This is using this method: <a href="https://docs.opencv.org/3.0-beta/modules/cuda/doc/calib3d.html" rel="nofollow noreferrer">https://docs.opencv.org/3.0-beta/modules/cuda/doc/calib3d.html</a></p>
  
  <p>I have all the parameters that the function need except "object" that is object 3D position if I read correctly but I don't have that object 3D pos I'm trying to calculate it.</p>
  
  <p>So here is what I have so far</p>
  
  <pre><code>#include &lt;sstream&gt;
  #include &lt;string&gt;
  #include &lt;iostream&gt;
  #include &lt;vector&gt;
  
  #include "Object.h"
  
  // default capture width and height
  const int FRAME_WIDTH = 640;
  const int FRAME_HEIGHT = 480;
  
  // max number of objects to be detected in frame
  const int MAX_NUM_OBJECTS = 50;
  
  // minimum and maximum object area
  const int MIN_OBJECT_AREA = 20 * 20;
  const int MAX_OBJECT_AREA = FRAME_HEIGHT * FRAME_WIDTH / 1.5;
  
  // names that will appear at the top of each window
  const string windowName = "Original Image";
  const string threesholdName = "Threeshold Image";
  
  // List of objects to track
  std::vector&lt;Object&gt; objects;
  
  
  string intToString(int number) {
  
      std::stringstream ss;
      ss &lt;&lt; number;
      return ss.str();
  }
  
  Object* getObjectByCoord(int x, int y) {
      for (Object &amp;o : objects) {
          if (o.getXPos() == x &amp;&amp; o.getYPos() == y)
              return &amp;o;
      }
  
      return NULL;
  }
  
  void drawObject(Object o, int index, Mat &amp;frame, vector&lt; vector&lt;Point&gt; &gt; contours, vector&lt;Vec4i&gt; hierarchy) {
      cv::drawContours(frame, contours, index, o.getColor(), 3, 8, hierarchy);
      cv::circle(frame, cv::Point(o.getXPos(), o.getYPos()), 5, o.getColor());
      cv::putText(frame, intToString(o.getXPos()) + " , " + intToString(o.getYPos()), cv::Point(o.getXPos(), o.getYPos() + 20), 1, 1, o.getColor());
      cv::putText(frame, o.getType(), cv::Point(o.getXPos(), o.getYPos() - 20), 1, 2, o.getColor());
  }
  
  void morphOps(Mat &amp;thresh) {
  
      //create structuring element that will be used to "dilate" and "erode" image.
      //the element chosen here is a 3px by 3px rectangle
      Mat erodeElement = getStructuringElement(MORPH_RECT, Size(3, 3));
      //dilate with larger element so make sure object is nicely visible
      Mat dilateElement = getStructuringElement(MORPH_RECT, Size(8, 8));
  
      erode(thresh, thresh, erodeElement);
      erode(thresh, thresh, erodeElement);
  
      dilate(thresh, thresh, dilateElement);
      dilate(thresh, thresh, dilateElement);
  }
  
  void trackFilteredObject(Object theObject, Mat threshold, Mat &amp;cameraFeed) {
      Mat temp;
      threshold.copyTo(temp);
  
      //these two vectors needed for output of findContours
      vector&lt; vector&lt;Point&gt; &gt; contours;
      vector&lt;Vec4i&gt; hierarchy;
  
      //find contours of filtered image using openCV findContours function
      findContours(temp, contours, hierarchy, CV_RETR_CCOMP, CV_CHAIN_APPROX_SIMPLE);
  
      //use moments method to find our filtered object
      double refArea = 0;
  
      if (hierarchy.size() &gt; 0) {
          int numObjects = hierarchy.size();
  
          for (int index = 0; index &gt;= 0; index = hierarchy[index][0]) {
  
              Moments moment = moments((cv::Mat)contours[index]);
              double area = moment.m00;
  
              if (area&gt;MIN_OBJECT_AREA) {
                  int x = moment.m10 / area;
                  int y = moment.m01 / area;
  
                  if (getObjectByCoord(x, y) != NULL)
                      continue;
  
                  theObject.setXPos(x);
                  theObject.setYPos(y);
  
                  drawObject(theObject, index, cameraFeed, contours, hierarchy);
  
                  break;
              }
          }
      }
  }
  
  int main(int argc, char* argv[])
  {
      //Matrix to store each frame of the webcam feed
      Mat cameraFeed;
      Mat threshold;
      Mat HSV;
  
      namedWindow(windowName, 0);
  
      // Preparing all the objects to track
      // Color in BGR and HSV -&gt; SV are to 255
      objects.push_back(Object("blue", Scalar(160, 70, 50), Scalar(190, 255, 255), Scalar(255, 252, 123)));
  
      //video capture object to acquire webcam feed
      VideoCapture capture;
  
      //open capture object at location zero (default location for webcam)
      capture.open(0);
  
      //set height and width of capture frame
      capture.set(CV_CAP_PROP_FRAME_WIDTH, FRAME_WIDTH);
      capture.set(CV_CAP_PROP_FRAME_HEIGHT, FRAME_HEIGHT);
  
      //start an infinite loop where webcam feed is copied to cameraFeed matrix
      //all of our operations will be performed within this loop
      waitKey(1000);
  
      while (1) {
          capture.read(cameraFeed);
  
          if (!cameraFeed.data)
          {
              return -1;
          }
  
          // Track objects
          for (Object &amp;o : objects) {
              cvtColor(cameraFeed, HSV, COLOR_BGR2HSV);
              inRange(HSV, o.getHSVmin(), o.getHSVmax(), threshold);
              morphOps(threshold);
              imshow(threesholdName, threshold);
              trackFilteredObject(o, threshold, cameraFeed);
          }
  
          imshow(windowName, cameraFeed);
  
          //delay 30ms so that screen can refresh.
          waitKey(30);
      }
      return 0;
  }
  </code></pre>
  
  <p>I can get the contour of an object within a color range but I need to translate the 2D point in a 3D point fitting the coordinate system of the Hololens. (I can have projectionMatrix, etc)</p>
  
  <p>Any of you here can help me?</p>