diff --git a/AScannerDarkly/CMakeLists.txt b/AScannerDarkly/CMakeLists.txt index 4cc5716..34ecf8d 100644 --- a/AScannerDarkly/CMakeLists.txt +++ b/AScannerDarkly/CMakeLists.txt @@ -4,3 +4,4 @@ find_package( OpenCV REQUIRED ) include_directories( ${OpenCV_INCLUDE_DIRS} ) add_executable( AScannerDarkly main.cpp ) target_link_libraries( AScannerDarkly ${OpenCV_LIBS} ) +target_link_libraries( AScannerDarkly -llept -ltesseract ) diff --git a/AScannerDarkly/TestImages/--psm.txt b/AScannerDarkly/TestImages/--psm.txt new file mode 100644 index 0000000..e69de29 diff --git a/AScannerDarkly/TestImages/collector_number_0006.jpg b/AScannerDarkly/TestImages/collector_number_0006.jpg new file mode 100644 index 0000000..2fdc9d3 Binary files /dev/null and b/AScannerDarkly/TestImages/collector_number_0006.jpg differ diff --git a/AScannerDarkly/TestImages/emrakul_the_world_anew.jpg b/AScannerDarkly/TestImages/emrakul_the_world_anew.jpg new file mode 100644 index 0000000..c1f9c12 Binary files /dev/null and b/AScannerDarkly/TestImages/emrakul_the_world_anew.jpg differ diff --git a/AScannerDarkly/TestImages/emrakul_the_world_anew_97.jpg b/AScannerDarkly/TestImages/emrakul_the_world_anew_97.jpg new file mode 100644 index 0000000..2141e73 Binary files /dev/null and b/AScannerDarkly/TestImages/emrakul_the_world_anew_97.jpg differ diff --git a/AScannerDarkly/TestImages/emrakul_the_world_anew_foil.jpg b/AScannerDarkly/TestImages/emrakul_the_world_anew_foil.jpg new file mode 100644 index 0000000..0cf8403 Binary files /dev/null and b/AScannerDarkly/TestImages/emrakul_the_world_anew_foil.jpg differ diff --git a/AScannerDarkly/TestImages/emrakul_the_world_anew_title_97.jpg b/AScannerDarkly/TestImages/emrakul_the_world_anew_title_97.jpg new file mode 100644 index 0000000..181f1ca Binary files /dev/null and b/AScannerDarkly/TestImages/emrakul_the_world_anew_title_97.jpg differ diff --git a/AScannerDarkly/TestImages/emrakul_the_world_anew_title_black.jpg b/AScannerDarkly/TestImages/emrakul_the_world_anew_title_black.jpg new file mode 100644 index 0000000..c93678a Binary files /dev/null and b/AScannerDarkly/TestImages/emrakul_the_world_anew_title_black.jpg differ diff --git a/AScannerDarkly/TestImages/emrakul_the_world_anew_title_white.jpg b/AScannerDarkly/TestImages/emrakul_the_world_anew_title_white.jpg new file mode 100644 index 0000000..aa7ddf1 Binary files /dev/null and b/AScannerDarkly/TestImages/emrakul_the_world_anew_title_white.jpg differ diff --git a/AScannerDarkly/TestImages/indicator_foil.jpg b/AScannerDarkly/TestImages/indicator_foil.jpg new file mode 100644 index 0000000..1ce0ffc Binary files /dev/null and b/AScannerDarkly/TestImages/indicator_foil.jpg differ diff --git a/AScannerDarkly/TestImages/indicator_nonfoil.jpg b/AScannerDarkly/TestImages/indicator_nonfoil.jpg new file mode 100644 index 0000000..9394b11 Binary files /dev/null and b/AScannerDarkly/TestImages/indicator_nonfoil.jpg differ diff --git a/AScannerDarkly/TestImages/language_code_en.jpg b/AScannerDarkly/TestImages/language_code_en.jpg new file mode 100644 index 0000000..2e72093 Binary files /dev/null and b/AScannerDarkly/TestImages/language_code_en.jpg differ diff --git a/AScannerDarkly/TestImages/set_code_mh3.jpg b/AScannerDarkly/TestImages/set_code_mh3.jpg new file mode 100644 index 0000000..c5ff37f Binary files /dev/null and b/AScannerDarkly/TestImages/set_code_mh3.jpg differ diff --git a/AScannerDarkly/main.cpp b/AScannerDarkly/main.cpp index ee8481a..c4af211 100644 --- a/AScannerDarkly/main.cpp +++ b/AScannerDarkly/main.cpp @@ -1,7 +1,10 @@ #include +#include +#include #include -const std::string WINDOW_NAME = "A Scanner Darkly"; +const std::string SCANNER_WINDOW_NAME = "A Scanner Darkly"; +const std::string CARD_WINDOW_NAME = "Detected Card"; const std::string CANNY_LOWER_THRESHOLD_TRACKBAR_NAME = "Canny: Lower Threshold"; @@ -10,9 +13,112 @@ const std::string ASPECT_RATIO_TOLERANCE_TRACKBAR_NAME = "Aspect Ratio Tolerance int g_aspect_ratio_tolerance = 1; const int MAX_ASPECT_RATIO_TOLERANCE = 100; -int g_Canny_lower_threshold = 110; +int g_Canny_lower_threshold = 195; const int CANNY_UPPER_THRESHOLD = 255; +cv::Mat cropImageToRoi(cv::Mat img, cv::Rect roi) { + return img(roi); +} + +cv::Mat detectCardInFrame(cv::Mat frame) { + cv::Mat grayscaleFrame, blurFrame, cannyFrame, cardFrame; + + // Perform edge detection + cv::cvtColor(frame, grayscaleFrame, cv::COLOR_BGR2GRAY); + cv::GaussianBlur(grayscaleFrame, blurFrame, cv::Size(5, 5), 2, 2); + cv::Canny(blurFrame, cannyFrame, g_Canny_lower_threshold, CANNY_UPPER_THRESHOLD); + + // Perform contour detection + std::vector> contours; + std::vector hierarchy; + cv::findContours(cannyFrame, contours, hierarchy, cv::RETR_TREE, cv::CHAIN_APPROX_SIMPLE); + + // Detect convex hulls + std::vector> hull(contours.size()); + for(size_t i = 0; i < contours.size(); i++) { + cv::convexHull(contours[i], hull[i]); + } + + // Detect RotatedRects + std::vector minRect(contours.size()); + for(size_t i = 0; i < contours.size(); i++) { + minRect[i] = cv::minAreaRect(contours[i]); + } + + // Draw lines around RotatedRects + /* + cv::Scalar rectangleColor(255, 0, 0, 0); + cv::Point2f rect_points[4]; + minRect[i].points(rect_points); + for (int j = 0; j < 4; j++) { + cv::line(frame, rect_points[j], rect_points[(j + 1) % 4], rectangleColor); + } + */ + + // Find possible cards + std::vector possibleCards(contours.size()); + int possibleCount = 0; + for (size_t i = 0; i < minRect.size(); i++) { + float aspectRatio = minRect[i].size.height / minRect[i].size.width; + float aspectRatioTolerance = g_aspect_ratio_tolerance / 100.0f; + if (aspectRatio < (CARD_ASPECT_RATIO - aspectRatioTolerance) || aspectRatio > (CARD_ASPECT_RATIO + aspectRatioTolerance)) { + continue; + } + + cv::putText(frame, std::to_string(minRect[i].angle), minRect[i].center, cv::FONT_HERSHEY_SIMPLEX, 1, cv::Scalar(255, 0, 0, 0)); + + possibleCards[possibleCount] = minRect[i]; + possibleCount++; + } + + if (possibleCount == 1) { + return cropImageToRoi(frame, possibleCards[0].boundingRect()); + } else if (possibleCount > 1) { + // If we have more than one possible match, take the largest one + int largestCardIndex = 0; + int largestCardSize = 0; + + for (size_t i = 0; i < possibleCards.size(); i++) { + float size = possibleCards[i].size.width * possibleCards[i].size.height; + if (size >= largestCardSize) { + largestCardIndex = i; + largestCardSize = size; + } + } + + // Some basic error checking to ensure the RotatedRect is roughly card-sized + if (largestCardSize >= 14400 && largestCardSize <= 200000) { + return cropImageToRoi(frame, possibleCards[largestCardIndex].boundingRect()); + } + } + + return cv::Mat(); +} + +const char* readTextFromImage(cv::InputArray img) { + const char* tempFilename = "temp.bmp"; + + // Initialize Tesseract api + tesseract::TessBaseAPI* tess = new tesseract::TessBaseAPI(); + tess->Init(NULL, "eng"); + tess->SetPageSegMode(tesseract::PSM_SPARSE_TEXT); + + // Load image into Tesseract + cv::imwrite(tempFilename, img); + Pix* pixd = pixRead(tempFilename); + tess->SetImage(pixd); + tess->Recognize(0); + + // Perform OCR + const char* out = tess->GetUTF8Text(); + + // Cleanup + pixDestroy(&pixd); + std::remove(tempFilename); + + return out; +} + int main(int argc, char** argv ) { cv::VideoCapture cap; cap.open(0); @@ -22,71 +128,34 @@ int main(int argc, char** argv ) { return -1; } - std::cout << "Card aspect ratio: " << CARD_ASPECT_RATIO << std::endl; + cv::namedWindow(SCANNER_WINDOW_NAME); - cv::namedWindow(WINDOW_NAME); + cv::createTrackbar(CANNY_LOWER_THRESHOLD_TRACKBAR_NAME, SCANNER_WINDOW_NAME, &g_Canny_lower_threshold, CANNY_UPPER_THRESHOLD, NULL); + cv::createTrackbar(ASPECT_RATIO_TOLERANCE_TRACKBAR_NAME, SCANNER_WINDOW_NAME, &g_aspect_ratio_tolerance, MAX_ASPECT_RATIO_TOLERANCE, NULL); - cv::createTrackbar(CANNY_LOWER_THRESHOLD_TRACKBAR_NAME, WINDOW_NAME, &g_Canny_lower_threshold, CANNY_UPPER_THRESHOLD, NULL); - cv::createTrackbar(ASPECT_RATIO_TOLERANCE_TRACKBAR_NAME, WINDOW_NAME, &g_aspect_ratio_tolerance, MAX_ASPECT_RATIO_TOLERANCE, NULL); - - cv::Mat frame, grayscaleFrame, blurFrame, cannyFrame; + cv::Mat frame, cardFrame; while (true) { cap >> frame; + //cardFrame = detectCardInFrame(frame); + cv::imshow(SCANNER_WINDOW_NAME, frame); - cv::cvtColor(frame, grayscaleFrame, cv::COLOR_BGR2GRAY); - cv::GaussianBlur(grayscaleFrame, blurFrame, cv::Size(5, 5), 2, 2); - cv::Canny(blurFrame, cannyFrame, g_Canny_lower_threshold, CANNY_UPPER_THRESHOLD); - - std::vector > contours; - std::vector hierarchy; - cv::findContours(cannyFrame, contours, hierarchy, cv::RETR_TREE, cv::CHAIN_APPROX_SIMPLE); - - std::vector> hull(contours.size()); - for(size_t i = 0; i < contours.size(); i++) { - cv::convexHull(contours[i], hull[i]); - } - - std::vector minRect(contours.size()); - for(size_t i = 0; i < contours.size(); i++) { - minRect[i] = cv::minAreaRect(contours[i]); - } - - //cv::Scalar contourColor = cv::Scalar(255, 0, 0); - cv::Scalar hullColor = cv::Scalar(0, 255, 0); - cv::Scalar rectangleColor = cv::Scalar(0, 0, 255); - - for (size_t i = 0; i< contours.size(); i++) { - //cv::drawContours(frame, contours, (int)i, contourColor, 2, cv::LINE_8, hierarchy, 0); - //cv::drawContours(frame, hull, (int)i, hullColor, 2, cv::LINE_8); - - // TODO: A purely aspect-ratio-based detection method returns too many false positives - // inside of a card, even after screwing around with various algorithm parameters. - // We have two potential options to fix this: - // - Only take the largest detected RotatedRect with the correct aspect ratio - // - Perform the camera calibration necessary to take accurate real-world measurements - float aspectRatio = minRect[i].size.height / minRect[i].size.width; - float aspectRatioTolerance = g_aspect_ratio_tolerance / 100.0f; - if (aspectRatio < (CARD_ASPECT_RATIO - aspectRatioTolerance) || aspectRatio > (CARD_ASPECT_RATIO + aspectRatioTolerance)) { - continue; + /* + if (cardFrame.size().width > 0) { + printf("Card detected\n"); + while (true) { + cv::imshow(CARD_WINDOW_NAME, cardFrame); + char c = (char)cv::waitKey(33); + if (c == 27) { + cv::destroyWindow(CARD_WINDOW_NAME); + break; + } } - - cv::Point2f rect_points[4]; - minRect[i].points(rect_points); - for (int j = 0; j < 4; j++) { - cv::line(frame, rect_points[j], rect_points[(j + 1) % 4], rectangleColor); - } - cv::putText(frame, std::to_string(aspectRatio), minRect[i].center, cv::FONT_HERSHEY_COMPLEX, 1, hullColor); } - - cv::imshow(WINDOW_NAME, frame); + */ char c = (char)cv::waitKey(33); if (c == 27) { break; - } else if (c == 's') { - cv::imwrite("output.png", frame); } } - - cv::destroyWindow(WINDOW_NAME); } diff --git a/AScannerDarkly/temp.bmp b/AScannerDarkly/temp.bmp new file mode 100644 index 0000000..b354853 Binary files /dev/null and b/AScannerDarkly/temp.bmp differ