给LUA脚本插上图像识别翅膀

• 前言
• 准备知识
• 思路
• LUA端类实现
• LUA端调用
• C++端实现

• 测试用图
• 测试用LUA代码
• 运行结果

前言

这个春节一直猫在家，不给政府添乱。今天是元宵节，正想出去一趟，微信群看到一条信息，立马就不动了：今天千万不要外出，不然会被病毒笑话的：原来你们躲得过初一，躲不过十五。 那就先把这个十五躲过去，继续宅。
废话少说，其实我用LUA也就是点皮毛。就用这点皮毛扎个小辫，给LUA提供图像识别功能

准备知识

• C++与LUA交互，这个在网上一抓一大把
• 基于C++与LUA的分工，需要首先在C++端提供图像识别的各项功能。我用OpenCV实现图像识别功能。

思路

LUA端类实现

以类的方式实现LUA端的调用代码，最终实现一个drMAT类，并创建一个实例cv。如下代码【后续持续完善，以实现更多功能】

drMAT = class()
cv = drMAT.new()

function drMAT:ctor()
end

function drMAT:MatFromFile(fileName, destMat)
return drMAT_MatFromFile(fileName, cbw(destMat))
end

function drMAT:SaveMat(mat, fileName)
return drMAT_SaveMat(mat, fileName)
end

function drMAT:SubMat(mat, rect, destMat, desc)
return drMAT_SubMat(mat, rect, cbw(destMat), cbw(desc))
end

function drMAT:Convert(mat, method, destMat, desc)
return drMAT_Convert(mat, method, cbw(destMat), cbw(desc))
end

function drMAT:ToMat_MASK(mat, method, destMat, desc)
return drMAT_ToMat_MASK(mat, method, cbw(destMat), cbw(desc))
end

function drMAT:Bitwise(mat, bitType, mat2, destMat, desc)
return drMAT_Bitwise(mat, bitType, cbw(mat2), cbw(destMat), cbw(desc))
end

function drMAT:Filter(mat, filterContent, destMat, desc)
return drMAT_Filter(mat, cbw(filterContent), cbw(destMat), cbw(desc))
end

function drMAT:GetPosition(mat, subMat, index)
return drMAT_GetPosition(mat, subMat, cbw(index, "0"))
end

function drMAT:GetFeature(mat, featureName)
return drMAT_GetFeature(mat, featureName)
end

function drMAT:countNonZero(mat, destMat)
return drMAT_countNonZero(mat, cbw(destMat))
end

function drMAT:resize(mat, size, destMat, desc)
return drMAT_resize(mat, size, cbw(destMat), cbw(desc))
end

function drMAT:GetIcon(mat, mask, iconSize, index, destMat, desc)
return drMAT_GetIcon(mat, mask, iconSize, cbw(index, "0"), cbw(destMat), cbw(desc))
end

function drMAT:FillMat(mat, value, rect, desc)
return drMAT_FillMat(mat, value, cbw(rect, "(0, 0, 0, 0)"), cbw(desc))
end

LUA端调用

直接调用实例对象的方法函数即可。
如要取得某Mat子图，直接调用mat = cv:SubMat(mat, “86, 712, 1120, 4”)

C++端实现

主要是响应相应的LUA调用

bool __fastcall TDrGraphScriptManager::OnLuaRequest_MAT(TDrLUA * lua, UnicodeString funName, TStrings * params) {
int paramIndex = 0;
UnicodeString firstParam = THelper::GetLuaParamAt(params, paramIndex++);
if(IsSame(L"drMAT_MatFromFile", funName)) 	{
UnicodeString fileName = lua->ParseFileName(firstParam);
cv::Mat mat = CvHelper::EmptyMat();
if(FileExists(fileName))
mat = CvHelper::MatFromFile(fileName);
lua->AddReturnValue(mat, NextTwoParam(params, paramIndex), "文件读入图像");
} else if(THelper::String::IsStartWith(funName, L"drMAT_")) { // 以下首参为图像对象
cv::Mat srcMat = lua->ParseMat(firstParam);
cv::Mat resultMat = CvHelper::EmptyMat();
if(srcMat.empty()) {
funName.Delete(1, 6);
lua->LogWarning(THelper::FormatString(L"第 %d 行警告：ApiMat调用%s函数时，待处理图像对象(第一个参数)为空，请检查！", lua->CurrentLine, funName));
return false;
}
if(IsSame(L"drMAT_SaveMat", funName)) {
UnicodeString fileName = lua->ParseFileName(THelper::GetLuaParamAt(params, paramIndex++), !FILE_MUST_EXISTED);
CvHelper::MatToFile(srcMat, fileName);
} else if(IsSame(L"drMAT_SubMat", funName)) {
cv::Rect rect = lua->ParseRect(THelper::GetLuaParamAt(params, paramIndex++));
resultMat = CvHelper::CopySubMat(srcMat, rect);
lua->AddReturnValue(resultMat, NextTwoParam(params, paramIndex), "子图");
} else if(IsSame(L"drMAT_FillMat", funName)) {
BYTE value = TTypeConvert::Str2Int(THelper::GetLuaParamAt(params, paramIndex++));
cv::Rect rect = lua->ParseRect(THelper::GetLuaParamAt(params, paramIndex++));
cv::Mat dstMat = srcMat;
if(rect.width > 0) {
CvHelper::ConstraintRect(rect, srcMat);
dstMat = srcMat(rect);
}
CvHelper::FillMat(dstMat, value);
resultMat = srcMat;
lua->AddReturnValue(resultMat, firstParam + THelper::GetLuaParamAt(params, paramIndex++, L""), "填充图像");
} else if(IsSame(L"drMAT_Convert", funName)) {
UnicodeString method = THelper::GetLuaParamAt(params, paramIndex++);
UnicodeString desc = L"转换图像";
if(IsSame(L"Gray", method)) {
resultMat = CvHelper::ToMat_GRAY(srcMat);
desc = L"灰度图";
} else if(IsSame(L"GrayMAX", method)) {
resultMat = CvHelper::ToMat_MaxGRAY(srcMat);
desc = L"最大灰度图";
} else if(IsSame(L"GrayMin", method)) {
resultMat = CvHelper::ToMat_MinGRAY(srcMat);
desc = L"最小灰度图";
} else if(IsSame(L"WhiteBackground", method)) {
cv::Mat foreMaskMat = GetForeMat(srcMat);
resultMat = cv::Mat(srcMat.rows, srcMat.cols, srcMat.type());
CvHelper::FillMat(resultMat, 0xFF);
srcMat.copyTo(resultMat, foreMaskMat);
desc = L"白色背景图";
} else if(method.Pos(L"=")) {
UnicodeString name = THelper::String::GetStringAt(method, L"=", 0).Trim();
UnicodeString value = THelper::String::GetStringAt(method, L"=", 1).Trim();
if(IsSame(L"threshold", name)) {
BYTE thresValue = TTypeConvert::Str2Int(value);
threshold(srcMat, resultMat, thresValue, 0xFF, cv::THRESH_BINARY);
desc = L"二值图";
} else if(IsSame(L"ratio", name)) {
double ratio = value.ToDouble();
if(ratio > 0)
resize(srcMat, resultMat, cv::Size(srcMat.cols * ratio, srcMat.rows * ratio));
desc = L"缩放图";
} else if(IsSame(L"Background", name)) {
cv::Mat foreMaskMat = GetForeMat(srcMat);
cv::Mat foreSrcMat = srcMat.clone();
BYTE bkValue = TTypeConvert::Str2Int(THelper::String::GetStringAt(value, L",", 0).Trim());
resultMat = cv::Mat(srcMat.rows, srcMat.cols, srcMat.type());
CvHelper::FillMat(resultMat, bkValue);
UnicodeString foreValue = THelper::String::GetStringAt(value, L",", 1).Trim();
if(foreValue.Length()) {
bkValue = TTypeConvert::Str2Int(foreValue);
CvHelper::FillMat(foreSrcMat, bkValue);
}
foreSrcMat.copyTo(resultMat, foreMaskMat);
}
} else if(method.LowerCase().Pos(L"threshold") == 1) {
BYTE thresValue = TTypeConvert::Str2Int(THelper::String::GetStringAt(method, L"=", 1));
threshold(srcMat, resultMat, thresValue, 0xFF, cv::THRESH_BINARY);
desc = L"二值图";
} else if(method.LowerCase().Pos(L"ratio") == 1) {
double ratio = THelper::String::GetStringAt(method, L"=", 1).Trim().ToDouble();
if(ratio > 0)
resize(srcMat, resultMat, cv::Size(srcMat.cols * ratio, srcMat.rows * ratio));
desc = L"缩放图";
}
lua->AddReturnValue(resultMat, NextTwoParam(params, paramIndex), desc);
} else if(IsSame(L"drMAT_Bitwise", funName)) 	{
UnicodeString type = THelper::GetLuaParamAt(params, paramIndex++);
if(IsSame(L"not", type)) {
bitwise_not(srcMat, resultMat);
} else {
cv::Mat mat2 = lua->ParseMat(THelper::GetLuaParamAt(params, paramIndex++));
if(IsSame(L"and", type))
bitwise_and(srcMat, mat2, resultMat);
else if(IsSame(L"or", type))
bitwise_or(srcMat, mat2, resultMat);
else if(IsSame(L"xor", type))
bitwise_xor(srcMat, mat2, resultMat);
}
lua->AddReturnValue(resultMat, NextTwoParam(params, paramIndex), THelper::FormatString(L"%s位操作结果图", type));
} else if(IsSame(L"drMAT_Filter", funName)) 	{
UnicodeString filterContent = THelper::GetLuaParamAt(params, paramIndex++);
UnicodeString filterName = THelper::String::GetStringAt(filterContent, L";").Trim();
THelper::String::EnsureWithStart(filterName, L"TFilter_");

TFilterBase * filter = NewFilterByType(filterName);
if(filter) {
int count = THelper::String::SplitNumber(filterContent, L";");
for(int i = 1; i < count; ++i) {
UnicodeString param = THelper::String::GetStringAt(filterContent, L";", i).Trim();
int index = THelper::String::GetStringAt(param, L"=", 0).ToInt();
param = THelper::String::GetStringAt(param, L"=", 1).Trim();
double value = THelper::String::GetStringAt(param, L",", 0, true, L"0").ToDouble();
double ratio = THelper::String::GetStringAt(param, L",", 1, true, L"1").ToDouble();
filter->SetValue(index, value, ratio);
}
resultMat = srcMat.clone();
filter->FilterProcess(resultMat);
delete filter;
}
lua->AddReturnValue(resultMat, NextTwoParam(params, paramIndex), THelper::FormatString(L"%s滤镜结果图", filterName));
} else if(IsSame(L"drMAT_GetPosition", funName)) {
UnicodeString subMatContent = THelper::GetLuaParamAt(params, paramIndex++);
TPoint resultPos(-1, -1);
if(subMatContent.Pos(L"<<")) {
TPositionOption option;
option.ReadFromString(subMatContent);
CvRects allRects = CvHelper::UIParse_GetEdit(srcMat, option);
int size = allRects.size();
UnicodeString indexText = THelper::GetLuaParamAt(params, paramIndex++, L"0").Trim().LowerCase();
int index = indexText.ToInt();
if(index < 0)
index += size;
if(IS_IN_RANGE(index, 0, size - 1)) {
CvRect r = allRects[index];
resultPos = TPoint(r.x + r.width / 2, r.y + r.height / 2);
}
} else if(THelper::String::IsStartWith(subMatContent.LowerCase(), L"icon")) {
UnicodeString iconName = THelper::String::GetStringAt(subMatContent, L"=", 1).Trim();
cv::Mat iconMat = lua->ParseMat(iconName);
cv::Mat whiteBkMat = GetWhiteBackgroundMat(srcMat);
cv::Mat maskMat = CvHelper::BuildTransMaskMat(whiteBkMat, clWhite, 5);
bitwise_not(maskMat, maskMat);
CvRects rects = GetIconRects(maskMat, iconMat.cols, iconMat.rows);
CBW_ITERATOR(CvRects, rects) {
cv::Mat dstMat = CvHelper::CopySubMat(whiteBkMat, *it);
if(CvHelper::IsSameIcons(dstMat, iconMat, false, true, 0.90, false, 3, false, false)) {
resultPos = TPoint(it->x + it->width / 2, it->y + it->height / 2);
break;
}
}
} else {
UnicodeString fileName = lua->ParseFileName(subMatContent, !FILE_MUST_EXISTED);
cv::Mat subMat;
if(FileExists(fileName))
subMat = CvHelper::MatFromFile(fileName);
else
subMat = lua->ParseMat(subMatContent);
if(!subMat.empty())
resultPos = CvHelper::GetSubMatPos(srcMat, subMat);
}
lua->AddReturnValue(resultPos);
} else if(IsSame(L"drMAT_ToMat_MASK", funName)) {
UnicodeString method = THelper::GetLuaParamAt(params, paramIndex++);
UnicodeString type = THelper::String::GetStringAt(method, L",", 0).Trim();
UnicodeString value1 = THelper::String::GetStringAt(method, L",", 1).Trim();
UnicodeString value2 = THelper::String::GetStringAt(method, L",", 2).Trim();
int v1 = TTypeConvert::Str2Int(value1);
if(IsSame(L"Color", type)) {
if(value2.Length()) {
int delta = TTypeConvert::Str2Int(value2);
resultMat = CvHelper::BuildTransMaskMat(srcMat, TColor(v1), delta);
} else
resultMat = CvHelper::BuildTransMaskMat_ColorPass(srcMat, v1);
} else if(IsSame(L"Gray", type)) {
if(value2.Length()) {
int delta = TTypeConvert::Str2Int(value2);
resultMat = CvHelper::BuildTransMaskMat(srcMat, BYTE(v1), delta);
} else
resultMat = CvHelper::BuildTransMaskMat(srcMat, BYTE(v1));
}
lua->AddReturnValue(resultMat, NextTwoParam(params, paramIndex), L"屏蔽图");
} else if(IsSame(L"drMAT_GetIcon", funName)) {
cv::Mat maskMat = lua->ParseMat(THelper::GetLuaParamAt(params, paramIndex++));
UnicodeString iconSizeString = THelper::GetLuaParamAt(params, paramIndex++);
int iconWidth = THelper::String::GetRegMatchAt_Int(iconSizeString, L"-?\\d+", 0);
int iconHeight = THelper::String::GetRegMatchAt_Int(iconSizeString, L"-?\\d+", 1);
int index = THelper::GetLuaParamAt(params, paramIndex++, L"0").Trim().ToInt();

CvRects rects = GetIconRects(maskMat, iconWidth, iconHeight);
if(index < 0)	index += rects.size();
if(IS_IN_RANGE(index, 0, rects.size() - 1)) {
cv::Rect r = rects[index];
resultMat = CvHelper::CopySubMat(srcMat, r);
}
lua->AddReturnValue(resultMat, NextTwoParam(params, paramIndex), L"目标图标");
} else if(IsSame(L"drMAT_GetFeature", funName)) {
UnicodeString featureName = THelper::GetLuaParamAt(params, paramIndex++);
if(IsSame(L"countNonZero", featureName)) {
cv::Mat tempMat = CvHelper::ToMat_GRAY(srcMat);
threshold(tempMat, tempMat, 0x7F, 0xFF, cv::THRESH_BINARY);
lua->AddReturnValue(int(countNonZero(tempMat)));
} else if(IsSame(L"whiteWeight", featureName)) {	// 白色比重
cv::Mat tempMat = CvHelper::ToMat_GRAY(srcMat);
threshold(tempMat, tempMat, 0x7F, 0xFF, cv::THRESH_BINARY);
double result = int(countNonZero(tempMat)) * 100.0 / tempMat.total();
lua->AddReturnValue(result);
}
} else if(IsSame(L"drMAT_countNonZero", funName)) {
cv::Mat tempMat = CvHelper::ToMat_GRAY(srcMat);
threshold(tempMat, tempMat, 0x7F, 0xFF, cv::THRESH_BINARY);
int count = countNonZero(tempMat);
lua->AddReturnValue(count);
} else if(IsSame(L"drMAT_resize", funName)) {
TPoint size = lua->ParsePoint(THelper::GetLuaParamAt(params, paramIndex++));
resize(srcMat, resultMat, cv::Size(size.x, size.y));
lua->AddReturnValue(resultMat, NextTwoParam(params, paramIndex), L"缩放图");
} else return false;
} else return false;
return true;
}

实例调用

简单测试一个背景处理功能

测试用图

测试用LUA代码

function test()
Initial()
realTimeMat = win:GetRealtimeMat(realTimeMat)
tempMat = cv:Convert(realTimeMat, "Background=0xFF, 0x0") 	-- 白色背景，黑色前景
tempMat = cv:Convert(realTimeMat, "Background=0x0, 0xFF")	-- 黑色背景，白色前景
tempMat = cv:Convert(realTimeMat, "Background=0xFF")		-- 白色背景，前景不变
tempMat = cv:Convert(realTimeMat, "Background=0x0")			-- 黑色背景，前景不变
return LUA_STATUS_OK
end

运行结果

总结

方法可行，基于此可实现各种脚本自动化。先把已有功能写成文档，便于后续开发使用。