使用OpenCV检测图像中的文本

Question

我需要在图片中检测文本..

这里有一些代码，大多数情况下都可以正常运行.. 但并非所有情况都如此.. 可以查看附加的输入/输出图片

代码

#include "string"
#include "fstream"
#include "/var/bin/opencv/include/opencv2/opencv.hpp"

using namespace std;
using namespace cv;

void detect_text(string input){
    Mat large = imread(input);
    
    Mat rgb;
    // downsample and use it for processing
    pyrDown(large, rgb);
    Mat small;
    cvtColor(rgb, small, CV_BGR2GRAY);
    // morphological gradient
    Mat grad;
    Mat morphKernel = getStructuringElement(MORPH_ELLIPSE, Size(3, 3));
    morphologyEx(small, grad, MORPH_GRADIENT, morphKernel);
    // binarize
    Mat bw;
    threshold(grad, bw, 0.0, 255.0, THRESH_BINARY | THRESH_OTSU);
    // connect horizontally oriented regions
    Mat connected;
    morphKernel = getStructuringElement(MORPH_RECT, Size(9, 1));
    morphologyEx(bw, connected, MORPH_CLOSE, morphKernel);
    // find contours
    Mat mask = Mat::zeros(bw.size(), CV_8UC1);
    vector<vector<Point> > contours;
    vector<Vec4i> hierarchy;
    findContours(connected, contours, hierarchy, CV_RETR_CCOMP, CV_CHAIN_APPROX_SIMPLE, Point(0, 0));
    // filter contours
    for(int idx = 0; idx >= 0; idx = hierarchy[idx][0]){
        Rect rect = boundingRect(contours[idx]);
        Mat maskROI(mask, rect);
        maskROI = Scalar(0, 0, 0);
        // fill the contour
        drawContours(mask, contours, idx, Scalar(255, 255, 255), CV_FILLED);
        // ratio of non-zero pixels in the filled region
        double r = (double)countNonZero(maskROI) / (rect.width * rect.height);
        
        // assume at least 45% of the area is filled if it contains text
        if (r > 0.45 && 
        (rect.height > 8 && rect.width > 8) // constraints on region size
        // these two conditions alone are not very robust. better to use something 
        //like the number of significant peaks in a horizontal projection as a third condition
        ){
            rectangle(rgb, rect, Scalar(0, 255, 0), 2);
        }
    }
    
    imwrite(string("test_text_contours.jpg"), rgb);
}

int main(int argc, char* argv[]){
    detect_text(string("input.jpg"));
}

输入

输出

更新

/*
 *  Compile
 *  # g++ txtbin.cpp -o txtbin `pkg-config opencv --cflags --libs`
 *
 *  Get opencv version
 *  # pkg-config --modversion opencv
 *
 *  Run
 *  # ./txtbin input.jpg output.png
 */

#include "string"
#include "fstream"
#include "/var/bin/opencv/include/opencv2/opencv.hpp"
//#include "/usr/include/opencv2/opencv.hpp"
#include "/usr/include/boost/tuple/tuple.hpp"

using namespace std;
using namespace cv;
using namespace boost;

void CalcBlockMeanVariance(Mat& Img, Mat& Res, float blockSide=21, float contrast=0.01){
    /*
     *  blockSide: set greater for larger fonts in image and vice versa
     *  contrast: set smaller for lower contrast image
     */
    
    Mat I;
    Img.convertTo(I, CV_32FC1);
    Res = Mat::zeros(Img.rows / blockSide, Img.cols / blockSide, CV_32FC1);
    Mat inpaintmask;
    Mat patch;
    Mat smallImg;
    Scalar m, s;
    
    for(int i = 0; i < Img.rows - blockSide; i += blockSide){
        for(int j = 0; j < Img.cols - blockSide; j += blockSide){
            patch = I(Range(i, i + blockSide + 1), Range(j, j + blockSide + 1));
            meanStdDev(patch, m, s);
            
            if(s[0] > contrast){
                Res.at<float>(i / blockSide, j / blockSide) = m[0];
            }
            else{
                Res.at<float>(i / blockSide, j / blockSide) = 0;
            }
        }
    }
    
    resize(I, smallImg, Res.size());
    
    threshold(Res, inpaintmask, 0.02, 1.0, THRESH_BINARY);
    
    Mat inpainted;
    smallImg.convertTo(smallImg, CV_8UC1, 255);
    
    inpaintmask.convertTo(inpaintmask, CV_8UC1);
    inpaint(smallImg, inpaintmask, inpainted, 5, INPAINT_TELEA);
    
    resize(inpainted, Res, Img.size());
    Res.convertTo(Res, CV_32FC1, 1.0 / 255.0);
}

tuple<int, int, int, int> detect_text_box(string input, Mat& res, bool draw_contours=false){
    Mat large = imread(input);
    
    bool test_output = false;
    
    int
        top = large.rows,
        bottom = 0,
        left = large.cols,
        right = 0;
    
    int
        rect_bottom,
        rect_right;
    
    Mat rgb;
    // downsample and use it for processing
    pyrDown(large, rgb);
    pyrDown(rgb, rgb);
    Mat small;
    cvtColor(rgb, small, CV_BGR2GRAY);
    // morphological gradient
    Mat grad;
    Mat morphKernel = getStructuringElement(MORPH_ELLIPSE, Size(3, 3));
    morphologyEx(small, grad, MORPH_GRADIENT, morphKernel);
    // binarize
    Mat bw;
    threshold(grad, bw, 0.0, 255.0, THRESH_BINARY | THRESH_OTSU);
    // connect horizontally oriented regions
    Mat connected;
    morphKernel = getStructuringElement(MORPH_RECT, Size(9, 1));
    morphologyEx(bw, connected, MORPH_CLOSE, morphKernel);
    // find contours
    Mat mask = Mat::zeros(bw.size(), CV_8UC1);
    vector<vector<Point> > contours;
    vector<Vec4i> hierarchy;
    findContours(connected, contours, hierarchy, CV_RETR_CCOMP, CV_CHAIN_APPROX_SIMPLE, Point(0, 0));
    
    Scalar color = Scalar(0, 255, 0);
    Scalar color2 = Scalar(0, 0, 255);
    int thickness = 2;
    
    // filter contours
    for(int idx = 0; idx >= 0; idx = hierarchy[idx][0]){
        Rect rect = boundingRect(contours[idx]);
        Mat maskROI(mask, rect);
        maskROI = Scalar(0, 0, 0);
        // fill the contour
        drawContours(mask, contours, idx, Scalar(255, 255, 255), CV_FILLED);
        // ratio of non-zero pixels in the filled region
        double r = (double)countNonZero(maskROI) / (rect.width * rect.height);
        
        // assume at least 25% of the area is filled if it contains text
        if (r > 0.25 && 
        (rect.height > 8 && rect.width > 8) // constraints on region size
        // these two conditions alone are not very robust. better to use something 
        //like the number of significant peaks in a horizontal projection as a third condition
        ){
            if(draw_contours){
                rectangle(res, Rect(rect.x * 4, rect.y * 4, rect.width * 4, rect.height * 4), color, thickness);
            }
            
            if(test_output){
                rectangle(rgb, rect, color, thickness);
            }
            
            if(rect.y < top){
                top = rect.y;
            }
            rect_bottom = rect.y + rect.height;
            if(rect_bottom > bottom){
                bottom = rect_bottom;
            }
            if(rect.x < left){
                left = rect.x;
            }
            rect_right = rect.x + rect.width;
            if(rect_right > right){
                right = rect_right;
            }
        }
    }
    
    if(draw_contours){
        rectangle(res, Point(left * 4, top * 4), Point(right * 4, bottom * 4), color2, thickness);
    }
    
    if(test_output){
        rectangle(rgb, Point(left, top), Point(right, bottom), color2, thickness);
        imwrite(string("test_text_contours.jpg"), rgb);
    }
    
    return make_tuple(left * 4, top * 4, (right - left) * 4, (bottom - top) * 4);
}

int main(int argc, char* argv[]){
    string input;
    string output = "output.png";
    
    int
        width = 0,
        height = 0,
        blockside = 9;
    
    bool
        crop = false,
        draw = false;
    
    float margin = 0;
    
    cout << "OpenCV version: " << CV_VERSION << endl;
    
    //  Return error if arguments are missing
    if(argc < 3){
        cerr << "\nUsage: txtbin input [options] output\n\n"
            "Options:\n"
            "\t-w <number>          -- set max width (keeps aspect ratio)\n"
            "\t-h <number>          -- set max height (keeps aspect ratio)\n"
            "\t-c                   -- crop text content contour\n"
            "\t-m <number>          -- add margins (number in %)\n"
            "\t-b <number>          -- set blockside\n"
            "\t-d                   -- draw text content contours (debugging)\n" << endl;
        return 1;
    }
    
    //  Parse arguments
    for(int i = 1; i < argc; i++){
        if(i == 1){
            input = string(argv[i]);
            
            //  Return error if input file is invalid
            ifstream stream(input.c_str());
            if(!stream.good()){
                cerr << "Error: Input file is invalid!" << endl;
                return 1;
            }
        }
        else if(string(argv[i]) == "-w"){
            width = atoi(argv[++i]);
        }
        else if(string(argv[i]) == "-h"){
            height = atoi(argv[++i]);
        }
        else if(string(argv[i]) == "-c"){
            crop = true;
        }
        else if(string(argv[i]) == "-m"){
            margin = atoi(argv[++i]);
        }
        else if(string(argv[i]) == "-b"){
            blockside = atoi(argv[++i]);
        }
        else if(string(argv[i]) == "-d"){
            draw = true;
        }
        else if(i == argc - 1){
            output = string(argv[i]);
        }
    }
    
    Mat Img = imread(input, CV_LOAD_IMAGE_GRAYSCALE);
    Mat res;
    Img.convertTo(Img, CV_32FC1, 1.0 / 255.0);
    CalcBlockMeanVariance(Img, res, blockside);
    res = 1.0 - res;
    res = Img + res;
    threshold(res, res, 0.85, 1, THRESH_BINARY);
    
    int
        txt_x,
        txt_y,
        txt_width,
        txt_height;
    
    if(crop || draw){
        tie(txt_x, txt_y, txt_width, txt_height) = detect_text_box(input, res, draw);
    }
    
    if(crop){
        //res = res(Rect(txt_x, txt_y, txt_width, txt_height)).clone();
        res = res(Rect(txt_x, txt_y, txt_width, txt_height));
    }
    
    if(margin){
        int border = res.cols * margin / 100;
        copyMakeBorder(res, res, border, border, border, border, BORDER_CONSTANT, Scalar(255, 255, 255));
    }
    
    float
        width_input = res.cols,
        height_input = res.rows;
    
    bool resized = false;
    
    //  Downscale image
    if(width > 0 && width_input > width){
        float scale = width_input / width;
        width_input /= scale;
        height_input /= scale;
        resized = true;
    }
    if(height > 0 && height_input > height){
        float scale = height_input / height;
        width_input /= scale;
        height_input /= scale;
        resized = true;
    }
    if(resized){
        resize(res, res, Size(round(width_input), round(height_input)));
    }
    
    imwrite(output, res * 255);
    
    return 0;
}

- clarkk

---

我看到二值化现在对你有用了。 - Andrey Smorodov

这是另一个进程 :) 程序 txtbin 有两种不同的二值化方式。 - clarkk

好的，那祝你好运 :) 希望你看过这个主题：https://dev59.com/hWAg5IYBdhLWcg3whrMV - Andrey Smorodov

@AndreySmorodov，那正是我得到灵感的帖子 :) - clarkk

Answer 1

你的detect_text代码与我关于文本检测的帖子这里非常相似。如果你使用了那个代码，你会发现原始帖子中的输入图片大小为1400 x 800。但是你在这个帖子和你之前的帖子中的输入图片通常要大四倍。因此，首先你可以尝试将输入图片缩小两倍。另外你的文本看起来有点倾斜，所以你可以尝试旋转矩形而不是正立矩形。然后你可以调整参数以适应你的情况。如我所述，轮廓过滤标准不是很稳健。在对代码进行这些更改后，我得到了一个合理的输出，如下所示。请注意，我已经用绿色突出显示了检测到的文本区域的旋转矩形。

代码：

void detect_text(string input){
    Mat large = imread(input);

    Mat rgb;
    // downsample and use it for processing
    pyrDown(large, rgb);
    pyrDown(rgb, rgb);
    Mat small;
    cvtColor(rgb, small, CV_BGR2GRAY);
    // morphological gradient
    Mat grad;
    Mat morphKernel = getStructuringElement(MORPH_ELLIPSE, Size(3, 3));
    morphologyEx(small, grad, MORPH_GRADIENT, morphKernel);
    // binarize
    Mat bw;
    threshold(grad, bw, 0.0, 255.0, THRESH_BINARY | THRESH_OTSU);
    // connect horizontally oriented regions
    Mat connected;
    morphKernel = getStructuringElement(MORPH_RECT, Size(9, 1));
    morphologyEx(bw, connected, MORPH_CLOSE, morphKernel);
    // find contours
    Mat mask = Mat::zeros(bw.size(), CV_8UC1);
    vector<vector<Point> > contours;
    vector<Vec4i> hierarchy;
    findContours(connected, contours, hierarchy, CV_RETR_CCOMP, CV_CHAIN_APPROX_SIMPLE, Point(0, 0));
    // filter contours
    for(int idx = 0; idx >= 0; idx = hierarchy[idx][0]){
        Rect rect = boundingRect(contours[idx]);
        Mat maskROI(mask, rect);
        maskROI = Scalar(0, 0, 0);
        // fill the contour
        drawContours(mask, contours, idx, Scalar(255, 255, 255), CV_FILLED);

        RotatedRect rrect = minAreaRect(contours[idx]);
        double r = (double)countNonZero(maskROI) / (rrect.size.width * rrect.size.height);

        Scalar color;
        int thickness = 1;
        // assume at least 25% of the area is filled if it contains text
        if (r > 0.25 && 
        (rrect.size.height > 8 && rrect.size.width > 8) // constraints on region size
        // these two conditions alone are not very robust. better to use something 
        //like the number of significant peaks in a horizontal projection as a third condition
        ){
            thickness = 2;
            color = Scalar(0, 255, 0);
        }
        else
        {
            thickness = 1;
            color = Scalar(0, 0, 255);
        }

        Point2f pts[4];
        rrect.points(pts);
        for (int i = 0; i < 4; i++)
        {
            line(rgb, Point((int)pts[i].x, (int)pts[i].y), Point((int)pts[(i+1)%4].x, (int)pts[(i+1)%4].y), color, thickness);
        }
    }

    imwrite("cont.jpg", rgb);
}

- dhanushka

---

我也用了你的帖子来创建自己的代码。正如你所看到的，评论是你的 :) 看起来很棒.. 我会尝试你的代码。 - clarkk

我不需要旋转矩形。这只会使后处理更加复杂。因为我需要在所有文本轮廓周围绘制一个新的矩形。函数必须返回顶部、底部、右侧和左侧，然后裁剪图像而不丢失文本。 - clarkk

已经更新了我的问题并合并了您的解决方案。我已经编译和测试过了，它可以正常工作。但是是否有必要添加这个遮罩层 Mat maskROI(mask, rect); 呢？ :) - clarkk

@clarkk 你可以像我在这里做的那样，使用轮廓面积来解决问题。思路是找出区域边界框被区域轮廓填充的比例。正如你可以在这里看到的，轮廓面积和非零像素点数可能不相同，所以如果你使用轮廓面积而不是基于掩码的非零像素点数，你可能需要相应地调整决策参数，但如果存在自交叉轮廓，仍然会得到错误的结果。 - dhanushka

好的，那么您认为保留 Mat maskROI(mask, rect); 的解决方案会更好吗？ :) - clarkk

@clarkk 我建议使用基于掩码的非零像素计数来获取轮廓面积，并将其除以旋转矩形面积来计算面积比。这样，您将检测到可能的文本区域，而不考虑它们的方向。我已经更新了我的代码，加入了这个修改。 - dhanushka

Answer 2

你可能会发现使用阈值法来确定文本和背景并不是最好的方法。我曾经为一个研究小组工作，他们使用Otsu的方法来确定荧光细胞的阈值。但是当背景非均匀时，例如图像中的纸张上的阴影，它的效果就不太好了。

相反，你可以通过比较相邻像素并寻找颜色的急剧变化来区分文本和纸张。这对我所做的工作效果要好得多。然后，你可以通过突出显示颜色值变化大的相邻像素来检测字符。

- Flash_Steel

---

你有代码示例（可运行的示例）吗？我对c++和opencv都很新 :) - clarkk

很抱歉，不行。那是专有代码，而且是用MATLAB编写的。 - Flash_Steel

伪代码可能是这样的：逐像素 - 获取像素颜色值（R0，G0，B0） 获取相邻像素值（R1，G1，B1） 如果（（R1-R0）^ 2 +（G1-G0）^ 2 +（B1-B0）^ 2>阈值颜色变化）{ 像素是文本像素的轮廓 }这应该足以处理白纸上的文本。 - Flash_Steel