来自窗口屏幕截图的模板匹配 [英] Template matching from a screenshot of a window

问题描述

我有一个小模板图像，用于在较大的屏幕截图图像中查找匹配子图像的坐标.屏幕截图本身在 BitBlt 的帮助下被捕获到内存 DC，然后通过 GetDIBits 转换为 cv::Mat，如下所示:

I have a small template image which is meant to be used to find coordinates of matching subimages within a larger screenshot image. The screenshot itself is captured into a memory DC with the help of BitBlt, then converted into a cv::Mat via GetDIBits, like so:

HDC windowDc = GetWindowDC(hwndTarget);
HDC memDc = CreateCompatibleDC(windowDc);

// ...

HBITMAP hbmp = CreateCompatibleBitmap(windowDc, width, height);
SelectObject(memDc, hbmp);
BITMAPINFOHEADER bi =
{
    sizeof(BITMAPINFOHEADER), // biSize
    width,                    // biWidth
    -height,                  // biHeight
    1,                        // biPlanes
    32,                       // biBitCount
    BI_RGB,                   // biCompression
    0,                        // biSizeImage
    0,                        // biXPelsPerMeter
    0,                        // biYPelsPerMeter
    0,                        // biClrUser
    0                         // biClrImportant
};

// ...

BitBlt(memDc, 0, 0, width, height, windowDc, offsetX, offsetY, SRCCOPY);
matScreen.create(height, width, CV_8UC4);
GetDIBits(memDc, hbmp, 0, (UINT)height, matScreen.data, (BITMAPINFO*)&bi, DIB_RGB_COLORS);

这似乎工作正常，并且快速 imshow("Source", matScreen) 可以正确显示图像.

This appears to work fine, and a quick imshow("Source", matScreen) displays the image correctly.

由于此屏幕截图图像已创建为 32 位 RGB 内存位图，并使用 CV_8UC4 标志放置在 cv::Mat 中，因此它失败了几个断言在 OpenCV 中(以及在使用几种 OpenCV 方法时输出奇怪的结果).最值得注意的是，matchTemplate 总是在以下行失败:

Since this screenshot image has been created as a 32-bit RGB memory bitmap, and placed inside a cv::Mat using the CV_8UC4 flag, it fails several assertions in OpenCV (as well as outputs whacky results when utilizing several OpenCV methods). Most notably, matchTemplate always fails on the following line:

CV_Assert( (depth == CV_8U || depth == CV_32F) && type == _templ.type() && _img.dims() <= 2 );

我已经尝试匹配屏幕截图的深度标志和模板/结果 Mats，最好的情况是我能够显示所有 3 张图像，但模板匹配不起作用，因为 我假设源(屏幕截图)图像以彩色显示，而其他图像以灰度显示.其他转换要么失败，要么显示奇怪的结果并且模板匹配失败(或者只是引发错误)......并且将屏幕截图图像的 Mat 更改为使用任何其他深度标志最终会错误地显示图像.

I've tried matching up the depth flags of the screenshot and the template/result Mats, and the best case scenario is that I am able to display all 3 images, but the template matching doesn't work because I'm assuming the source (screenshot) image is displayed in color whilst the others are in grayscale. Other conversions either fail, or display strange results and fail the template matching (or simply raise an error)... And changing the screenshot image's Mat to use any other depth flag ends up displaying the image incorrectly.

如何利用 OpenCV 的模板匹配 C++ 中的屏幕截图?我应该对屏幕截图图像进行某种手动转换吗?

What can I do to utilize OpenCV's template matching with a screenshot taken in C++? Is there some sort of manual conversion I should be doing to the screenshot image, or something?

截屏代码取自:github.com/acdx/opencv-screen-capture
我的主要代码:codeshare.io/vLio1

推荐答案

我已经包含了用于从桌面图像中查找模板图像的代码.希望这能解决您的问题！

I have included my code for finding a Template image from your Desktop image. Hope this solves your problem!

#include <fstream>
#include <memory>
#include <string>
#include <iostream>
#include <strstream>
#include <functional>
#include <Windows.h>
#include <iostream>
#include <string>

using namespace std;
using namespace cv;

Mat hwnd2mat(HWND hwnd){

    HDC hwindowDC,hwindowCompatibleDC;

    int height,width,srcheight,srcwidth;
    HBITMAP hbwindow;
    Mat src;
    BITMAPINFOHEADER  bi;

    hwindowDC=GetDC(hwnd);
    hwindowCompatibleDC=CreateCompatibleDC(hwindowDC);
    SetStretchBltMode(hwindowCompatibleDC,COLORONCOLOR);  

    RECT windowsize;    // get the height and width of the screen
    GetClientRect(hwnd, &windowsize);

    srcheight = windowsize.bottom;
    srcwidth = windowsize.right;
    height = windowsize.bottom;  //change this to whatever size you want to resize to
    width = windowsize.right;

    src.create(height,width,CV_8UC4);

    // create a bitmap
    hbwindow = CreateCompatibleBitmap( hwindowDC, width, height);
    bi.biSize = sizeof(BITMAPINFOHEADER);    //http://msdn.microsoft.com/en-us/library/windows/window/dd183402%28v=vs.85%29.aspx
    bi.biWidth = width;    
    bi.biHeight = -height;  //this is the line that makes it draw upside down or not
    bi.biPlanes = 1;    
    bi.biBitCount = 32;    
    bi.biCompression = BI_RGB;    
    bi.biSizeImage = 0;  
    bi.biXPelsPerMeter = 0;    
    bi.biYPelsPerMeter = 0;    
    bi.biClrUsed = 0;    
    bi.biClrImportant = 0;

    // use the previously created device context with the bitmap
    SelectObject(hwindowCompatibleDC, hbwindow);
    // copy from the window device context to the bitmap device context
    StretchBlt( hwindowCompatibleDC, 0,0, width, height, hwindowDC, 0, 0,srcwidth,srcheight, SRCCOPY); //change SRCCOPY to NOTSRCCOPY for wacky colors !
    GetDIBits(hwindowCompatibleDC,hbwindow,0,height,src.data,(BITMAPINFO *)&bi,DIB_RGB_COLORS);  //copy from hwindowCompatibleDC to hbwindow

    // avoid memory leak
    DeleteObject (hbwindow); DeleteDC(hwindowCompatibleDC); ReleaseDC(hwnd, hwindowDC);

    return src;
}

bool NMultipleTemplateMatching(Mat mInput,Mat mTemplate,float Threshold,float Closeness,vector<Point2f> &List_Matches)
{
    Mat mResult;
    Size szTemplate= mTemplate.size();
    Size szTemplateCloseRadius((szTemplate.width/2)* Closeness,(szTemplate.height/2)* Closeness);

    matchTemplate(mInput, mTemplate, mResult, TM_CCOEFF_NORMED);
    threshold(mResult, mResult, Threshold, 1.0, THRESH_TOZERO);
    while (true) 
    {
        double minval, maxval ;
        Point minloc, maxloc;
        minMaxLoc(mResult, &minval, &maxval, &minloc, &maxloc);

        if (maxval >= Threshold)
        {
            List_Matches.push_back(maxloc);
            rectangle(mResult,Point2f(maxloc.x-szTemplateCloseRadius.width,maxloc.y-szTemplateCloseRadius.height),Point2f(maxloc.x+szTemplateCloseRadius.width,maxloc.y+szTemplateCloseRadius.height),Scalar(0),-1);
        }
        else
            break;
    }
    //imshow("reference", mDebug_Bgr);
    return true;
}


int main(int argc, char** argv)
{
    Mat mTemplate_Bgr,mTemplate_Gray;
    mTemplate_Bgr= imread("Template.png",1);
    imshow("mTemplate_Bgr",mTemplate_Bgr);

    HWND hDesktopWnd;
    hDesktopWnd=GetDesktopWindow();
    Mat mScreenShot= hwnd2mat(hDesktopWnd);
    Mat mSource_Gray,mResult_Bgr= mScreenShot.clone();


    float Threshold= 0.9;
    float Closeness= 0.9;
    vector<Point2f> List_Matches;

    cvtColor(mScreenShot,mSource_Gray,COLOR_BGR2GRAY);
    cvtColor(mTemplate_Bgr,mTemplate_Gray,COLOR_BGR2GRAY);

    namedWindow("Screen Shot",WINDOW_AUTOSIZE);
    imshow("Screen Shot",mSource_Gray);

    NMultipleTemplateMatching(mSource_Gray,mTemplate_Gray,Threshold,Closeness,List_Matches);

    for (int i = 0; i < List_Matches.size(); i++)
    {
        rectangle(mResult_Bgr,List_Matches[i],Point(List_Matches[i].x + mTemplate_Bgr.cols, List_Matches[i].y + mTemplate_Bgr.rows),Scalar(0,255,0), 2);
    }

    imshow("Final Results",mResult_Bgr);

    waitKey(0);

    return 0;
}   

这篇关于来自窗口屏幕截图的模板匹配的文章就介绍到这了，希望我们推荐的答案对大家有所帮助，也希望大家多多支持IT屋！