title: Detect largest rectangle from an image using OpenCV
body: <p>I asked a previous question <a href="https://stackoverflow.com/questions/15744520/cropping-out-an-image-from-an-existing-image">here</a> and following the advice from the answer I built the below program which I thought would detect large rectangle but it doesn't detect the rectangle at all. It does work on this <a href="http://cl.ly/1e363U230V342g0l123t" rel="nofollow noreferrer">image</a> though. </p>
  
  <p><a href="https://i.stack.imgur.com/MKcXt.png" rel="nofollow noreferrer">Original Image</a></p>
  
  <p><a href="https://i.stack.imgur.com/hMTnv.png" rel="nofollow noreferrer">Desired Image</a></p>
  
  <p>I want the solution to work on not only this image but different images of this kind. Major part of the code below is from different answers on SO</p>
  
  <p>My full program:</p>
  
  <pre><code>#include &lt;cv.h&gt;
  #include &lt;highgui.h&gt;
  using namespace cv;
  using namespace std;
  
  double angle(  Point pt1,  Point pt2,  Point pt0 ) {
      double dx1 = pt1.x - pt0.x;
      double dy1 = pt1.y - pt0.y;
      double dx2 = pt2.x - pt0.x;
      double dy2 = pt2.y - pt0.y;
      return (dx1*dx2 + dy1*dy2)/sqrt((dx1*dx1 + dy1*dy1)*(dx2*dx2 + dy2*dy2) + 1e-10);
  }
  
  void find_squares( Mat&amp; image,  vector&lt; vector&lt; Point&gt; &gt;&amp; squares)
  {
      // blur will enhance edge detection
      Mat blurred(image);
      medianBlur(image, blurred, 9);
  
       Mat gray0(blurred.size(), CV_8U), gray;
       vector&lt; vector&lt; Point&gt; &gt; contours;
  
      // find squares in every color plane of the image
      for (int c = 0; c &lt; 3; c++)
      {
          int ch[] = {c, 0};
           mixChannels(&amp;blurred, 1, &amp;gray0, 1, ch, 1);
  
          // try several threshold levels
          const int threshold_level = 2;
          for (int l = 0; l &lt; threshold_level; l++)
          {
              // Use Canny instead of zero threshold level!
              // Canny helps to catch squares with gradient shading
              if (l == 0)
              {
                   Canny(gray0, gray, 10, 20, 3); // 
  
                  // Dilate helps to remove potential holes between edge segments
                   dilate(gray, gray,  Mat(),  Point(-1,-1));
              }
              else
              {
                      gray = gray0 &gt;= (l+1) * 255 / threshold_level;
              }
  
              // Find contours and store them in a list
               findContours(gray, contours, CV_RETR_LIST, CV_CHAIN_APPROX_SIMPLE);
  
              // Test contours
               vector&lt; Point&gt; approx;
              for (size_t i = 0; i &lt; contours.size(); i++)
              {
                      // approximate contour with accuracy proportional
                      // to the contour perimeter
                       approxPolyDP( Mat(contours[i]), approx,  arcLength( Mat(contours[i]), true)*0.02, true);
  
                      // Note: absolute value of an area is used because
                      // area may be positive or negative - in accordance with the
                      // contour orientation
                      if (approx.size() == 4 &amp;&amp;
                              fabs(contourArea( Mat(approx))) &gt; 1000 &amp;&amp;
                              isContourConvex( Mat(approx)))
                      {
                              double maxCosine = 0;
  
                              for (int j = 2; j &lt; 5; j++)
                              {
                                      double cosine = fabs(angle(approx[j%4], approx[j-2], approx[j-1]));
                                      maxCosine = MAX(maxCosine, cosine);
                              }
  
                              if (maxCosine &lt; 0.3)
                                      squares.push_back(approx);
                      }
              }
          }
      }
  }
  
  void find_largest_square(const vector&lt;vector &lt;Point&gt; &gt;&amp; squares, vector&lt;Point&gt;&amp; biggest_square) {
      if (!squares.size()) {
          return;
      }
  
      int max_width = 0;
      int max_height = 0;
      int max_square_idx = 0;
      const int n_points = 4;
  
      for (size_t i = 0; i &lt; squares.size(); i++) {
          Rect rectangle = boundingRect(Mat(squares[i]));
          if ((rectangle.width &gt;= max_width) &amp;&amp; (rectangle.height &gt;= max_height)) {
              max_width = rectangle.width;
              max_height = rectangle.height;
              max_square_idx = i;
          }
      }
      biggest_square = squares[max_square_idx];
  
  }
  
  int main(int argc, char* argv[])
  {
      Mat img =  imread(argv[1]);
      if (img.empty())
      {
           cout &lt;&lt; "!!! imread() failed to open target image" &lt;&lt;  endl;
          return -1;        
      }
      vector&lt; vector&lt; Point&gt; &gt; squares;
      find_squares(img, squares);
      vector&lt;Point&gt; largest_square;
      find_largest_square(squares, largest_square);
      for (int i = 0; i &lt; 4; ++i) {
          line(img, largest_square[i], largest_square[(i+1)%4], Scalar(0, 255, 0), 1, CV_AA);
      }
      imwrite("squares.png", img);
      imshow("squares", img);
      waitKey(0);
      return 0;
  }
  </code></pre>