Update AScannerDarkly

AScannerDarkly/CMakeLists.txt

@@ -4,3 +4,4 @@ find_package( OpenCV REQUIRED )
 include_directories( ${OpenCV_INCLUDE_DIRS} )
 add_executable( AScannerDarkly main.cpp )
 target_link_libraries( AScannerDarkly ${OpenCV_LIBS} )
+target_link_libraries( AScannerDarkly -llept -ltesseract )

AScannerDarkly/main.cpp

@@ -1,7 +1,10 @@
 #include <opencv2/opencv.hpp>
+#include <tesseract/baseapi.h>
+#include <leptonica/allheaders.h>
 #include <string>
 
-const std::string WINDOW_NAME = "A Scanner Darkly";
+const std::string SCANNER_WINDOW_NAME = "A Scanner Darkly";
+const std::string CARD_WINDOW_NAME = "Detected Card";
 
 const std::string CANNY_LOWER_THRESHOLD_TRACKBAR_NAME = "Canny: Lower Threshold";
 
@@ -10,9 +13,112 @@ const std::string ASPECT_RATIO_TOLERANCE_TRACKBAR_NAME = "Aspect Ratio Tolerance
 int g_aspect_ratio_tolerance = 1;
 const int MAX_ASPECT_RATIO_TOLERANCE = 100;
 
-int g_Canny_lower_threshold = 110;
+int g_Canny_lower_threshold = 195;
 const int CANNY_UPPER_THRESHOLD = 255;
 
+cv::Mat cropImageToRoi(cv::Mat img, cv::Rect roi) {
+	return img(roi);
+}
+
+cv::Mat detectCardInFrame(cv::Mat frame) {
+	cv::Mat grayscaleFrame, blurFrame, cannyFrame, cardFrame;
+
+	// Perform edge detection
+	cv::cvtColor(frame, grayscaleFrame, cv::COLOR_BGR2GRAY);
+	cv::GaussianBlur(grayscaleFrame, blurFrame, cv::Size(5, 5), 2, 2);
+	cv::Canny(blurFrame, cannyFrame, g_Canny_lower_threshold, CANNY_UPPER_THRESHOLD);
+
+	// Perform contour detection
+	std::vector<std::vector<cv::Point>> contours;
+	std::vector<cv::Vec4i> hierarchy;
+	cv::findContours(cannyFrame, contours, hierarchy, cv::RETR_TREE, cv::CHAIN_APPROX_SIMPLE);
+
+	// Detect convex hulls
+	std::vector<std::vector<cv::Point>> hull(contours.size());
+	for(size_t i = 0; i < contours.size(); i++) {
+		cv::convexHull(contours[i], hull[i]);
+	}
+
+	// Detect RotatedRects
+	std::vector<cv::RotatedRect> minRect(contours.size());
+	for(size_t i = 0; i < contours.size(); i++) {
+		minRect[i] = cv::minAreaRect(contours[i]);
+	}
+
+	// Draw lines around RotatedRects
+	/*
+	cv::Scalar rectangleColor(255, 0, 0, 0);
+	cv::Point2f rect_points[4];
+	minRect[i].points(rect_points);
+	for (int j = 0; j < 4; j++) {
+		cv::line(frame, rect_points[j], rect_points[(j + 1) % 4], rectangleColor);
+	}
+	*/
+
+	// Find possible cards
+	std::vector<cv::RotatedRect> possibleCards(contours.size());
+	int possibleCount = 0;
+	for (size_t i = 0; i < minRect.size(); i++) {
+		float aspectRatio = minRect[i].size.height / minRect[i].size.width;
+		float aspectRatioTolerance = g_aspect_ratio_tolerance / 100.0f;
+		if (aspectRatio < (CARD_ASPECT_RATIO - aspectRatioTolerance) || aspectRatio > (CARD_ASPECT_RATIO + aspectRatioTolerance)) {
+			continue;
+		}
+
+		cv::putText(frame, std::to_string(minRect[i].angle), minRect[i].center, cv::FONT_HERSHEY_SIMPLEX, 1, cv::Scalar(255, 0, 0, 0));
+
+		possibleCards[possibleCount] = minRect[i];
+		possibleCount++;
+	}
+
+	if (possibleCount == 1) {
+		return cropImageToRoi(frame, possibleCards[0].boundingRect());
+	} else if (possibleCount > 1) {
+		// If we have more than one possible match, take the largest one
+		int largestCardIndex = 0;
+		int largestCardSize = 0;
+
+		for (size_t i = 0; i < possibleCards.size(); i++) {
+			float size = possibleCards[i].size.width * possibleCards[i].size.height;
+			if (size >= largestCardSize) {
+				largestCardIndex = i;
+				largestCardSize = size;
+			}
+		}
+
+		// Some basic error checking to ensure the RotatedRect is roughly card-sized
+		if (largestCardSize >= 14400 && largestCardSize <= 200000) {
+			return cropImageToRoi(frame, possibleCards[largestCardIndex].boundingRect());
+		}
+	}
+
+	return cv::Mat();
+}
+
+const char* readTextFromImage(cv::InputArray img) {
+	const char* tempFilename = "temp.bmp";
+
+	// Initialize Tesseract api
+	tesseract::TessBaseAPI* tess = new tesseract::TessBaseAPI();
+	tess->Init(NULL, "eng");
+	tess->SetPageSegMode(tesseract::PSM_SPARSE_TEXT);
+
+	// Load image into Tesseract
+	cv::imwrite(tempFilename, img);
+	Pix* pixd = pixRead(tempFilename);
+	tess->SetImage(pixd);
+	tess->Recognize(0);
+
+	// Perform OCR
+	const char* out = tess->GetUTF8Text();
+
+	// Cleanup
+	pixDestroy(&pixd);
+	std::remove(tempFilename);
+
+	return out;
+}
+
 int main(int argc, char** argv ) {
 	cv::VideoCapture cap;
 	cap.open(0);
@@ -22,71 +128,34 @@ int main(int argc, char** argv ) {
 		return -1;
 	}
 
-	std::cout << "Card aspect ratio: " << CARD_ASPECT_RATIO << std::endl;
+	cv::namedWindow(SCANNER_WINDOW_NAME);
 
-	cv::namedWindow(WINDOW_NAME);
+	cv::createTrackbar(CANNY_LOWER_THRESHOLD_TRACKBAR_NAME, SCANNER_WINDOW_NAME, &g_Canny_lower_threshold, CANNY_UPPER_THRESHOLD, NULL);
+	cv::createTrackbar(ASPECT_RATIO_TOLERANCE_TRACKBAR_NAME, SCANNER_WINDOW_NAME, &g_aspect_ratio_tolerance, MAX_ASPECT_RATIO_TOLERANCE, NULL);
 
-	cv::createTrackbar(CANNY_LOWER_THRESHOLD_TRACKBAR_NAME, WINDOW_NAME, &g_Canny_lower_threshold, CANNY_UPPER_THRESHOLD, NULL);
+	cv::Mat frame, cardFrame;
-	cv::createTrackbar(ASPECT_RATIO_TOLERANCE_TRACKBAR_NAME, WINDOW_NAME, &g_aspect_ratio_tolerance, MAX_ASPECT_RATIO_TOLERANCE, NULL);
-
-	cv::Mat frame, grayscaleFrame, blurFrame, cannyFrame;
 	while (true) {
 		cap >> frame;
+		//cardFrame = detectCardInFrame(frame);
+		cv::imshow(SCANNER_WINDOW_NAME, frame);
 
-		cv::cvtColor(frame, grayscaleFrame, cv::COLOR_BGR2GRAY);
+		/*
-		cv::GaussianBlur(grayscaleFrame, blurFrame, cv::Size(5, 5), 2, 2);
+		if (cardFrame.size().width > 0) {
-		cv::Canny(blurFrame, cannyFrame, g_Canny_lower_threshold, CANNY_UPPER_THRESHOLD);
+			printf("Card detected\n");
-
+			while (true) {
-		std::vector<std::vector<cv::Point> > contours;
+				cv::imshow(CARD_WINDOW_NAME, cardFrame);
-		std::vector<cv::Vec4i> hierarchy;
+				char c = (char)cv::waitKey(33);
-		cv::findContours(cannyFrame, contours, hierarchy, cv::RETR_TREE, cv::CHAIN_APPROX_SIMPLE);
+				if (c == 27) {
-
+					cv::destroyWindow(CARD_WINDOW_NAME);
-		std::vector<std::vector<cv::Point>> hull(contours.size());
+					break;
-		for(size_t i = 0; i < contours.size(); i++) {
-			cv::convexHull(contours[i], hull[i]);
 				}
-
-	  std::vector<cv::RotatedRect> minRect(contours.size());
-		for(size_t i = 0; i < contours.size(); i++) {
-		  minRect[i] = cv::minAreaRect(contours[i]);
 			}
-		
-		//cv::Scalar contourColor = cv::Scalar(255, 0, 0);
-		cv::Scalar hullColor = cv::Scalar(0, 255, 0);
-		cv::Scalar rectangleColor = cv::Scalar(0, 0, 255);
-
-		for (size_t i = 0; i< contours.size(); i++) {
-			//cv::drawContours(frame, contours, (int)i, contourColor, 2, cv::LINE_8, hierarchy, 0);
-			//cv::drawContours(frame, hull, (int)i, hullColor, 2, cv::LINE_8);
-			
-			// TODO: A purely aspect-ratio-based detection method returns too many false positives
-			// inside of a card, even after screwing around with various algorithm parameters.
-			// We have two potential options to fix this:
-			// - Only take the largest detected RotatedRect with the correct aspect ratio
-			// - Perform the camera calibration necessary to take accurate real-world measurements
-			float aspectRatio = minRect[i].size.height / minRect[i].size.width;
-			float aspectRatioTolerance = g_aspect_ratio_tolerance / 100.0f;
-			if (aspectRatio < (CARD_ASPECT_RATIO - aspectRatioTolerance) || aspectRatio > (CARD_ASPECT_RATIO + aspectRatioTolerance)) {
-				continue;
 		}
-			
+		*/
-			cv::Point2f rect_points[4];
-			minRect[i].points(rect_points);
-			for (int j = 0; j < 4; j++) {
-				cv::line(frame, rect_points[j], rect_points[(j + 1) % 4], rectangleColor);
-			}
-			cv::putText(frame, std::to_string(aspectRatio), minRect[i].center, cv::FONT_HERSHEY_COMPLEX, 1, hullColor);
-		}
-
-		cv::imshow(WINDOW_NAME, frame);
 
 		char c = (char)cv::waitKey(33);
 		if (c == 27) {
 			break;
-		} else if (c == 's') {
-			cv::imwrite("output.png", frame);
 		}
 	}
-
-	cv::destroyWindow(WINDOW_NAME);
 }