Skip to content
This repository was archived by the owner on Aug 19, 2024. It is now read-only.

Enhanced Rectangle/Document detection in android #33

Open
wants to merge 2 commits into
base: master
Choose a base branch
from
Open

Enhanced Rectangle/Document detection in android #33

Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
0b3f31d
Look for largest rectangle not just first rectangle
humphreyja Mar 15, 2020
784ba39
messing with image before checking for rectangles
humphreyja Mar 23, 2020
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
133 changes: 80 additions & 53 deletions android/src/main/java/com/rectanglescanner/helpers/ImageProcessor.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -18,10 +18,14 @@
import org.opencv.core.Core;
import org.opencv.core.CvType;
import org.opencv.core.Mat;
import org.opencv.core.MatOfDouble;
import org.opencv.core.MatOfPoint;
import org.opencv.core.MatOfPoint2f;
import org.opencv.core.Point;
import org.opencv.core.Range;
import org.opencv.core.Scalar;
import org.opencv.core.Size;
import org.opencv.core.Rect;
import org.opencv.imgcodecs.Imgcodecs;
import org.opencv.imgproc.Imgproc;

Expand Down Expand Up @@ -108,8 +112,7 @@ private void detectRectangleInFrame(Mat inputRgba) {
Size srcSize = inputRgba.size();
this.lastDetectedRectangle = getQuadrilateral(contours, srcSize);
Bundle data = new Bundle();
boolean focused = mMainActivity.isFocused();
if (focused && this.lastDetectedRectangle != null) {
if (this.lastDetectedRectangle != null) {
Bundle quadMap = this.lastDetectedRectangle.toBundle();
data.putBundle("detectedRectangle", quadMap);
} else {
Expand Down Expand Up @@ -151,26 +154,27 @@ private Quadrilateral getQuadrilateral(ArrayList<MatOfPoint> contours, Size srcS
int width = Double.valueOf(srcSize.width).intValue();
Size size = new Size(width, height);

double areaOfPreview = height * width;
double minArea = size.area() * 0.01;
MatOfPoint bestFittingContour = null;

Log.i(TAG, "Size----->" + size);
for (MatOfPoint c : contours) {
double contourArea = Imgproc.contourArea(c);

if (contourArea < minArea) {
continue;
}

MatOfPoint2f c2f = new MatOfPoint2f(c.toArray());
double peri = Imgproc.arcLength(c2f, true);
MatOfPoint2f approx = new MatOfPoint2f();
Imgproc.approxPolyDP(c2f, approx, 0.02 * peri, true);

Point[] points = approx.toArray();

// select biggest 4 angles polygon
// if (points.length == 4) {
Point[] foundPoints = sortPoints(points);

if (insideArea(foundPoints, size)) {

return new Quadrilateral(c, foundPoints, new Size(srcSize.width, srcSize.height));
Point[] foundPoints = sortPoints(approx.toArray());
if (isValidRectangle(foundPoints)) {
return new Quadrilateral(c, foundPoints, new Size(srcSize.width, srcSize.height));
}
// }
}

return null;
}

Expand Down Expand Up @@ -210,28 +214,36 @@ public int compare(Point lhs, Point rhs) {
return result;
}

private boolean insideArea(Point[] rp, Size size) {

int width = Double.valueOf(size.width).intValue();
int height = Double.valueOf(size.height).intValue();
private boolean isValidRectangle(Point[] rp) {
boolean isANormalShape = rp[0].x != rp[1].x && rp[1].y != rp[0].y && rp[2].y != rp[3].y && rp[3].x != rp[2].x;
if (!isANormalShape) {
return false;
}

int minimumSize = width / 10;
double leftOffset = Math.abs(rp[0].x - rp[3].x);
double rightOffset = Math.abs(rp[1].x - rp[2].x);
double bottomOffset = Math.abs(rp[0].y - rp[1].y);
double topOffset = Math.abs(rp[2].y - rp[3].y);

boolean isANormalShape = rp[0].x != rp[1].x && rp[1].y != rp[0].y && rp[2].y != rp[3].y && rp[3].x != rp[2].x;
boolean isBigEnough = ((rp[1].x - rp[0].x >= minimumSize) && (rp[2].x - rp[3].x >= minimumSize)
&& (rp[3].y - rp[0].y >= minimumSize) && (rp[2].y - rp[1].y >= minimumSize));
double largestVertical = Math.max(leftOffset, rightOffset);
double verticalOffset = Math.abs(leftOffset - rightOffset);
double largestHorizontal = Math.max(topOffset, bottomOffset);
double horizontalOffset = Math.abs(topOffset - bottomOffset);
double largestSide = Math.max(largestHorizontal, largestVertical);
double sideOffset = Math.abs(largestHorizontal - largestVertical);

double leftOffset = rp[0].x - rp[3].x;
double rightOffset = rp[1].x - rp[2].x;
double bottomOffset = rp[0].y - rp[1].y;
double topOffset = rp[2].y - rp[3].y;
if (verticalOffset > (largestVertical * 0.9)) {
return false;
}

boolean isAnActualRectangle = ((leftOffset <= minimumSize && leftOffset >= -minimumSize)
&& (rightOffset <= minimumSize && rightOffset >= -minimumSize)
&& (bottomOffset <= minimumSize && bottomOffset >= -minimumSize)
&& (topOffset <= minimumSize && topOffset >= -minimumSize));
if (horizontalOffset > (largestHorizontal * 0.9)) {
return false;
}

return isANormalShape && isAnActualRectangle && isBigEnough;
// if (sideOffset > (largestSide * 0.995)) {
// return false;
// }
return true;
}

private Mat fourPointTransform(Mat src, Point[] pts) {
Expand Down Expand Up @@ -269,28 +281,12 @@ private Mat fourPointTransform(Mat src, Point[] pts) {
}

private ArrayList<MatOfPoint> findContours(Mat src) {

Mat grayImage;
Mat cannedImage;
Mat resizedImage;

int height = Double.valueOf(src.size().height).intValue();
int width = Double.valueOf(src.size().width).intValue();
Size size = new Size(width, height);

resizedImage = new Mat(size, CvType.CV_8UC4);
grayImage = new Mat(size, CvType.CV_8UC4);
cannedImage = new Mat(size, CvType.CV_8UC1);

Imgproc.resize(src, resizedImage, size);
Imgproc.cvtColor(resizedImage, grayImage, Imgproc.COLOR_RGBA2GRAY, 4);
Imgproc.GaussianBlur(grayImage, grayImage, new Size(5, 5), 0);
Imgproc.Canny(grayImage, cannedImage, 80, 100, 3, false);
Mat cannedImage = applyCannedFilterToImage(src);

ArrayList<MatOfPoint> contours = new ArrayList<>();
Mat hierarchy = new Mat();

Imgproc.findContours(cannedImage, contours, hierarchy, Imgproc.RETR_TREE, Imgproc.CHAIN_APPROX_SIMPLE);
Imgproc.findContours(cannedImage, contours, hierarchy, Imgproc.RETR_EXTERNAL, Imgproc.CHAIN_APPROX_SIMPLE);

hierarchy.release();

Expand All @@ -302,8 +298,6 @@ public int compare(MatOfPoint lhs, MatOfPoint rhs) {
}
});

resizedImage.release();
grayImage.release();
cannedImage.release();

return contours;
Expand Down Expand Up @@ -336,6 +330,20 @@ public void applyFilters(Mat image) {
}
}

/**
* Returns the mean brightness level of the provided image
* @param image
* @return brightness level
*/
public double brightnessOfImage(Mat image)
{
MatOfDouble meansrc= new MatOfDouble();
MatOfDouble stdsrc= new MatOfDouble();

Core.meanStdDev(image, meansrc, stdsrc);
return meansrc.get(0,0)[0];
}

/*!
Slightly enhances the black and white image
*/
Expand All @@ -350,8 +358,9 @@ public Mat applyGreyscaleFilterToImage(Mat image)
*/
public Mat applyBlackAndWhiteFilterToImage(Mat image)
{
double imageBrightnessLevel = brightnessOfImage(image);
image.convertTo(image, -1, 2.8 - (imageBrightnessLevel * 0.0045), imageBrightnessLevel * -1.15);
Imgproc.cvtColor(image, image, Imgproc.COLOR_RGBA2GRAY);
image.convertTo(image, -1, 1, 10);
return image;
}

Expand All @@ -360,10 +369,28 @@ public Mat applyBlackAndWhiteFilterToImage(Mat image)
*/
public Mat applyColorFilterToImage(Mat image)
{
image.convertTo(image, -1, 1.2, 0);
double imageBrightnessLevel = brightnessOfImage(image);
image.convertTo(image, -1, 2.8 - (imageBrightnessLevel * 0.0045), imageBrightnessLevel * -1.15);
return image;
}

public Mat applyCannedFilterToImage(Mat image)
{
int blurSize = 5;
int thresh = 100;
double imageBrightnessLevel = brightnessOfImage(image);
Mat colorAdjustedImage = new Mat();
Mat cannedImage = new Mat();

image.convertTo(colorAdjustedImage, -1, 2.8 - (imageBrightnessLevel * 0.0045), imageBrightnessLevel * -1.15);
Imgproc.cvtColor(colorAdjustedImage, colorAdjustedImage, Imgproc.COLOR_RGBA2GRAY, 4);
// Imgproc.equalizeHist(colorAdjustedImage, colorAdjustedImage);
Imgproc.bilateralFilter(colorAdjustedImage, cannedImage, blurSize, blurSize * 2, blurSize / 2);
Imgproc.Canny(cannedImage, cannedImage, thresh, thresh * 3, 3, true);
colorAdjustedImage.release();
return cannedImage;
}


public void rotateImageForScreen(Mat image) {
switch (this.mMainActivity.lastDetectedRotation) {
Expand Down