基于opencv的手势识别和鼠标控制
opencv3.4.5和vs2015
新建工程->空项目->新建项
摄像头使用
VideoCapture capture;
capture.open(0);
if (!capture.isOpened()){//打开失败返回-1
cout << "No camera!\n" << endl;
return -1;
while (1) {
capture >> frame;//刷新
//Mat frame = imread("1.jpg"); //工程测试
if (frame.empty())//帧空跳出循环
break;
imshow("frame_", frame);
}
}
分类器的使用
这里使用了自己做的分类器用作手势识别
定义、读取
CascadeClassifier open_palm_cascade;
CascadeClassifier closed_palm_cascade;
open_palm_cascade.load("palm.xml");
closed_palm_cascade.load("closed_palm.xml");
if (open_palm_cascade.empty()) {
cout << "Could not load open_palm configuration file! "
"Check directory! " << endl << "Press Q to Quit!" << endl;
while (char(waitKey(0)) != 'q') {}
return -2;
}
if (closed_palm_cascade.empty()) {
cout << "Could not load closed_palm configuration file! "
"Check directory! " << endl << "Press Q to Quit!" << endl;
while (char(waitKey(0)) != 'q') {}
return -2;
}
使用
open_palm_cascade.detectMultiScale(frame, open_palms, 1.3, 4, 0, Size(50, 50));
closed_palm_cascade.detectMultiScale(frame, closed_palms, 1.3, 4, 0, Size(50, 50));
if (closed_palms.size() != 0) {
for (int i = 0; i < closed_palms.size(); i++) {
//cout << "=============Detected a closed_palm!=============" << endl;
// Top left and bottom right points of rectangle.
Point closed_palm_rect_p1(closed_palms[i].x, closed_palms[i].y);
Point closed_palm_rect_p2(closed_palms[i].x + closed_palms[i].width, closed_palms[i].y + closed_palms[i].height);
// Draw the rectangle in the image.
rectangle(frame, closed_palm_rect_p1, closed_palm_rect_p2, Scalar(0, 255, 0));
putText(frame, "Closed Palm", closed_palm_rect_p1, FONT_HERSHEY_SIMPLEX,
1, Scalar(0, 255, 0), 1, 5, false);
hand_mode[0] = 1;
}
}
for (int i = 0; i < open_palms.size(); i++) {
//cout << "=============Detected an open_palm!=============" << endl;
// Top left and bottom right points of rectangle.
Point open_palm_rect_p1(open_palms[i].x, open_palms[i].y);
Point open_palm_rect_p2(open_palms[i].x + open_palms[i].width, open_palms[i].y + open_palms[i].height);
// Draw the rectangle in the image.
rectangle(frame, open_palm_rect_p1, open_palm_rect_p2, Scalar(255, 0, 0));
putText(frame, "Open Palm", open_palm_rect_p1, FONT_HERSHEY_SIMPLEX,
1, Scalar(255, 0, 0), 1, 5, false);
hand_mode[0] = 2;
}
图像处理、滤波
medianBlur(frame, frame, 5);
cvtColor(frame, frame_hsv, CV_BGR2HSV);//转HSV图像
split(frame_hsv, frameSplit);//三通道图像分离
//显示HSV 3通道
//imshow("WIN_H", frameSplit[0]);
//imshow("WIN_S", frameSplit[1]);
//imshow("WIN_V", frameSplit[2]);
///////////////////////////////////////////////
Mat dstTemp1(frame.rows, frame.cols, CV_8UC1);
Mat dstTemp2(frame.rows, frame.cols, CV_8UC1);
// 对HSV空间进行量化，得到2值图像，亮的部分为手的形状
inRange(frame_hsv, Scalar(0, 50, 30), Scalar(20, 100, 256), dstTemp1);
inRange(frame_hsv, Scalar(156, 30, 30), Scalar(180, 170, 256), dstTemp2);
bitwise_or(dstTemp1, dstTemp2, mask);//按位或
//imshow("mask", mask);
// 形态学操作，去除噪声，并使手的边界更加清晰,提取边缘
Mat element = getStructuringElement(MORPH_RECT, Size(3, 3));//返回指定形状和尺寸的结构元素，3*3的矩阵
erode(mask, mask, element);//腐蚀
morphologyEx(mask, mask, MORPH_OPEN, element);
dilate(mask, mask, element);//膨胀
morphologyEx(mask, mask, MORPH_CLOSE, element);
//frame.copyTo(dst, mask);
frame.copyTo(show_img, mask);
轮廓提取、重心计算
//寻找最外层轮廓 //只保存拐点信息
findContours(mask, contours, hierarchy, RETR_EXTERNAL, CHAIN_APPROX_SIMPLE);
// 去除伪轮廓
for (size_t i = 0; i < contours.size(); i++) {
if (fabs(contourArea(Mat(contours[i]))) > 30000) {//判断手进入区域的阈值
filterContours.push_back(contours[i]);
}
}
// 画轮廓
drawContours(show_img, filterContours, -1, Scalar(0, 0, 255), 3/*, 8, hierarchy*/);
// 得到轮廓的凸包络
for (size_t j = 0; j<filterContours.size(); j++) {
convexHull(Mat(filterContours[j]), hull, true);
int hullcount = (int)hull.size();
for (int i = 0; i<hullcount - 1; i++) {
line(show_img, hull[i + 1], hull[i], Scalar(255, 0, 0), 2, 0);
}
line(show_img, hull[hullcount - 1], hull[0], Scalar(255, 0, 0), 2, 0);
}
//////////////重心//////////////
Moments moment = moments(mask, true);
Point center(moment.m10 / moment.m00, moment.m01 / moment.m00);
circle(show_img, center, 8, Scalar(0, 0, 255), CV_FILLED);//画红色实心圆
鼠标控制
TEMP.x = center.x - LAST_CENTER.x;
TEMP.y = center.y - LAST_CENTER.y;
LAST_CENTER.x = center.x;
LAST_CENTER.y = center.y;
///////////////////////////
if (hand_mode[0] == 1 && hand_mode[1] == 1){
mouse_event(MOUSEEVENTF_LEFTDOWN, TEMP.x, TEMP.y, 0, 0);
mouse_event(MOUSEEVENTF_MOVE, TEMP.x, TEMP.y, 0, 0);
cout << "leftdown" << endl;
}
else
{
mouse_event(MOUSEEVENTF_LEFTUP, TEMP.x, TEMP.y, 0, 0);
mouse_event(MOUSEEVENTF_MOVE, TEMP.x, TEMP.y, 0, 0);
cout << "up" << endl;
}
hand_mode[1] = hand_mode[0];
鼠标取阈值
if (mode == "test"){//阈值测试
while (1) {
capture >> frame;//刷新
if (frame.empty())//帧空跳出循环
break;
//////////////////////////////////////
// 中值滤波，去除椒盐噪声
medianBlur(frame, frame, 5);
cvtColor(frame, frame_hsv, CV_BGR2HSV);//转HSV图像
setMouseCallback("frame_hsv", on_mouse, &frame_hsv);//鼠标取阈值
imshow("frame_hsv", frame);
if (cvWaitKey(20) == 'q')
break;
}
}
void on_mouse(int EVENT, int x, int y, int flags, void* userdata)
{
Mat hh;
hh = *(Mat*)userdata;
Point p(x, y);
switch (EVENT)
{
case EVENT_LBUTTONDOWN:
{
//cout << "h=" << hh.at<Vec3b>(p)[0] << endl;
//cout << "s=" << hh.at<Vec3b>(p)[1] << endl;
//cout << "v=" << hh.at<Vec3b>(p)[2] << endl;
printf("h=%d\t", hh.at<Vec3b>(p)[0]);
printf("s=%d\t", hh.at<Vec3b>(p)[1]);
printf("v=%d\n", hh.at<Vec3b>(p)[2]);
circle(hh, p, 2, Scalar(255), 3);
}
break;
}
}
整体程序
/*
*日期：12月15日
*功能：手势识别
*原理简述：预处理->HSV三通道阈值分割图像->提取轮廓->计算重心点->重心每帧偏差用来控制鼠标移动
*/
#include "opencv2/opencv.hpp"
#include <windows.h>
using namespace cv;
using namespace std;
POINT TEMP;
POINT LAST_CENTER;
vector<Rect> open_palms;
vector<Rect> closed_palms;
void skinExtract(const Mat &frame, Mat &skinArea);
void on_mouse(int EVENT, int x, int y, int flags, void* userdata);
int main(int argc, char* argv[])
{
Mat frame, skinArea;
Mat frame_hsv;//HSV图像
Mat mask(frame.rows, frame.cols, CV_8UC1); // 2值掩膜
Mat frameSplit[4];//分离通道
Mat show_img;//结果图像
vector<vector<Point> > contours;// 轮廓
vector<Vec4i> hierarchy;//四维int向量// 轮廓的结构信息
vector< vector<Point> > filterContours; // 筛选后的轮廓
vector< Point > hull; // 凸包络的点集
//摄像头初始化、开启
VideoCapture capture;
////////////////////
string mode;
int hand_mode[3];
capture.open(0);
if (!capture.isOpened()){//打开失败返回-1
cout << "No camera!\n" << endl;
return -1;
}
/////////////////////////////////////////////////////////////////////////
CascadeClassifier open_palm_cascade;
CascadeClassifier closed_palm_cascade;
open_palm_cascade.load("palm.xml");
closed_palm_cascade.load("closed_palm.xml");
if (open_palm_cascade.empty()) {
cout << "Could not load open_palm configuration file! "
"Check directory! " << endl << "Press Q to Quit!" << endl;
while (char(waitKey(0)) != 'q') {}
return -2;
}
if (closed_palm_cascade.empty()) {
cout << "Could not load closed_palm configuration file! "
"Check directory! " << endl << "Press Q to Quit!" << endl;
while (char(waitKey(0)) != 'q') {}
return -2;
}
// Start the open_palm and eye detection phase
////////////////////////////////////////////////////////////////////////
cin >>mode;//输入模式
if (mode == "hsv") {
while (1) {
capture >> frame;//刷新
//Mat frame = imread("1.jpg"); //工程测试
if (frame.empty())//帧空跳出循环
break;
/////////////////////////////////////////////////////////////////////////////////////////////
//skinArea.create(frame.rows, frame.cols, CV_8UC1);//和原图像等行数/等列数的/8位无符号通/道数1
//skinExtract(frame, skinArea);//找到皮肤
// 中值滤波，去除椒盐噪声
medianBlur(frame, frame, 5);
cvtColor(frame, frame_hsv, CV_BGR2HSV);//转HSV图像
split(frame_hsv, frameSplit);//三通道图像分离
//显示HSV 3通道
//imshow("WIN_H", frameSplit[0]);
//imshow("WIN_S", frameSplit[1]);
//imshow("WIN_V", frameSplit[2]);
///////////////////////////////////////////////
Mat dstTemp1(frame.rows, frame.cols, CV_8UC1);
Mat dstTemp2(frame.rows, frame.cols, CV_8UC1);
// 对HSV空间进行量化，得到2值图像，亮的部分为手的形状
inRange(frame_hsv, Scalar(0, 30, 30), Scalar(40, 170, 256), dstTemp1);
inRange(frame_hsv, Scalar(156, 30, 30), Scalar(180, 170, 256), dstTemp2);
bitwise_or(dstTemp1, dstTemp2, mask);//按位或
//imshow("mask", mask);
// 形态学操作，去除噪声，并使手的边界更加清晰,提取边缘
Mat element = getStructuringElement(MORPH_RECT, Size(3, 3));//返回指定形状和尺寸的结构元素，3*3的矩阵
erode(mask, mask, element);//腐蚀
morphologyEx(mask, mask, MORPH_OPEN, element);
dilate(mask, mask, element);//膨胀
morphologyEx(mask, mask, MORPH_CLOSE, element);
//frame.copyTo(dst, mask);
frame.copyTo(show_img, mask);
//imshow("mask", mask);
/////////////////////////////////////////////////////////////
//清空
contours.clear();
hierarchy.clear();
filterContours.clear();
/////////////////////////////////////////////////////////////
//寻找最外层轮廓 //只保存拐点信息
findContours(mask, contours, hierarchy, RETR_EXTERNAL, CHAIN_APPROX_SIMPLE);
// 去除伪轮廓
for (size_t i = 0; i < contours.size(); i++){
if (fabs(contourArea(Mat(contours[i]))) > 30000){//判断手进入区域的阈值
filterContours.push_back(contours[i]);
}
}
// 画轮廓
drawContours(show_img, filterContours, -1, Scalar(0, 0, 255), 3/*, 8, hierarchy*/);
// 得到轮廓的凸包络
for (size_t j = 0; j<filterContours.size(); j++){
convexHull(Mat(filterContours[j]), hull, true);
int hullcount = (int)hull.size();
for (int i = 0; i<hullcount - 1; i++){
line(show_img, hull[i + 1], hull[i], Scalar(255, 0, 0), 2, 0);
}
line(show_img, hull[hullcount - 1], hull[0], Scalar(255, 0, 0), 2, 0);
}
//////////////重心//////////////
Moments moment = moments(mask, true);
Point center(moment.m10 / moment.m00, moment.m01 / moment.m00);
circle(show_img, center, 8, Scalar(0, 0, 255), CV_FILLED);//画红色实心圆
////////////////////////////
TEMP.x = center.x - LAST_CENTER.x;
TEMP.y = center.y - LAST_CENTER.y;
LAST_CENTER.x = center.x;
LAST_CENTER.y = center.y;
///////////////////////////
///////////////////////////
//cout << "TEMP.x=" << TEMP.x << "\tTEMP.y=" << TEMP.y << endl;
//mouse_event(MOUSEEVENTF_MOVE, TEMP.x, TEMP.y, 0, 0);
setMouseCallback("frame_hsv", on_mouse, &frame_hsv);//鼠标取阈值
imshow("frame_hsv", frame);
imshow("result", show_img);
show_img.release();
if (cvWaitKey(20) == 'q')
break;
}
}
if (mode == "test"){//阈值测试
while (1) {
capture >> frame;//刷新
if (frame.empty())//帧空跳出循环
break;
//////////////////////////////////////
// 中值滤波，去除椒盐噪声
medianBlur(frame, frame, 5);
cvtColor(frame, frame_hsv, CV_BGR2HSV);//转HSV图像
setMouseCallback("frame_hsv", on_mouse, &frame_hsv);//鼠标取阈值
imshow("frame_hsv", frame);
if (cvWaitKey(20) == 'q')
break;
}
}
if (mode == "1") {
while (1){
capture >> frame;//刷新
if (frame.empty())//帧空跳出循环
break;
open_palm_cascade.detectMultiScale(frame, open_palms, 1.3, 4, 0, Size(50, 50));
closed_palm_cascade.detectMultiScale(frame, closed_palms, 1.3, 4, 0, Size(50, 50));
if (closed_palms.size() != 0) {
for (int i = 0; i < closed_palms.size(); i++) {
//cout << "=============Detected a closed_palm!=============" << endl;
// Top left and bottom right points of rectangle.
Point closed_palm_rect_p1(closed_palms[i].x, closed_palms[i].y);
Point closed_palm_rect_p2(closed_palms[i].x + closed_palms[i].width, closed_palms[i].y + closed_palms[i].height);
// Draw the rectangle in the image.
rectangle(frame, closed_palm_rect_p1, closed_palm_rect_p2, Scalar(0, 255, 0));
putText(frame, "Closed Palm", closed_palm_rect_p1, FONT_HERSHEY_SIMPLEX,
1, Scalar(0, 255, 0), 1, 5, false);
hand_mode[0] = 1;
}
}
for (int i = 0; i < open_palms.size(); i++) {
//cout << "=============Detected an open_palm!=============" << endl;
// Top left and bottom right points of rectangle.
Point open_palm_rect_p1(open_palms[i].x, open_palms[i].y);
Point open_palm_rect_p2(open_palms[i].x + open_palms[i].width, open_palms[i].y + open_palms[i].height);
// Draw the rectangle in the image.
rectangle(frame, open_palm_rect_p1, open_palm_rect_p2, Scalar(255, 0, 0));
putText(frame, "Open Palm", open_palm_rect_p1, FONT_HERSHEY_SIMPLEX,
1, Scalar(255, 0, 0), 1, 5, false);
hand_mode[0] = 2;
}
imshow("fram", frame);
/////////////////////////////////////////////////////////////////////////////////////////////
//skinArea.create(frame.rows, frame.cols, CV_8UC1);//和原图像等行数/等列数的/8位无符号通/道数1
//skinExtract(frame, skinArea);//找到皮肤
// 中值滤波，去除椒盐噪声
medianBlur(frame, frame, 5);
cvtColor(frame, frame_hsv, CV_BGR2HSV);//转HSV图像
split(frame_hsv, frameSplit);//三通道图像分离
//显示HSV 3通道
//imshow("WIN_H", frameSplit[0]);
//imshow("WIN_S", frameSplit[1]);
//imshow("WIN_V", frameSplit[2]);
///////////////////////////////////////////////
Mat dstTemp1(frame.rows, frame.cols, CV_8UC1);
Mat dstTemp2(frame.rows, frame.cols, CV_8UC1);
// 对HSV空间进行量化，得到2值图像，亮的部分为手的形状
inRange(frame_hsv, Scalar(0, 50, 30), Scalar(20, 100, 256), dstTemp1);
inRange(frame_hsv, Scalar(156, 30, 30), Scalar(180, 170, 256), dstTemp2);
bitwise_or(dstTemp1, dstTemp2, mask);//按位或
//imshow("mask", mask);
// 形态学操作，去除噪声，并使手的边界更加清晰,提取边缘
Mat element = getStructuringElement(MORPH_RECT, Size(3, 3));//返回指定形状和尺寸的结构元素，3*3的矩阵
erode(mask, mask, element);//腐蚀
morphologyEx(mask, mask, MORPH_OPEN, element);
dilate(mask, mask, element);//膨胀
morphologyEx(mask, mask, MORPH_CLOSE, element);
//frame.copyTo(dst, mask);
frame.copyTo(show_img, mask);
//imshow("mask", mask);
/////////////////////////////////////////////////////////////
//清空
contours.clear();
hierarchy.clear();
filterContours.clear();
/////////////////////////////////////////////////////////////
//寻找最外层轮廓 //只保存拐点信息
findContours(mask, contours, hierarchy, RETR_EXTERNAL, CHAIN_APPROX_SIMPLE);
// 去除伪轮廓
for (size_t i = 0; i < contours.size(); i++) {
if (fabs(contourArea(Mat(contours[i]))) > 30000) {//判断手进入区域的阈值
filterContours.push_back(contours[i]);
}
}
// 画轮廓
drawContours(show_img, filterContours, -1, Scalar(0, 0, 255), 3/*, 8, hierarchy*/);
// 得到轮廓的凸包络
for (size_t j = 0; j<filterContours.size(); j++) {
convexHull(Mat(filterContours[j]), hull, true);
int hullcount = (int)hull.size();
for (int i = 0; i<hullcount - 1; i++) {
line(show_img, hull[i + 1], hull[i], Scalar(255, 0, 0), 2, 0);
}
line(show_img, hull[hullcount - 1], hull[0], Scalar(255, 0, 0), 2, 0);
}
//////////////重心//////////////
Moments moment = moments(mask, true);
Point center(moment.m10 / moment.m00, moment.m01 / moment.m00);
circle(show_img, center, 8, Scalar(0, 0, 255), CV_FILLED);//画红色实心圆
////////////////////////////
TEMP.x = center.x - LAST_CENTER.x;
TEMP.y = center.y - LAST_CENTER.y;
LAST_CENTER.x = center.x;
LAST_CENTER.y = center.y;
///////////////////////////
if (hand_mode[0] == 1 && hand_mode[1] == 1){
mouse_event(MOUSEEVENTF_LEFTDOWN, TEMP.x, TEMP.y, 0, 0);
mouse_event(MOUSEEVENTF_MOVE, TEMP.x, TEMP.y, 0, 0);
cout << "leftdown" << endl;
}
else
{
mouse_event(MOUSEEVENTF_LEFTUP, TEMP.x, TEMP.y, 0, 0);
mouse_event(MOUSEEVENTF_MOVE, TEMP.x, TEMP.y, 0, 0);
cout << "up" << endl;
}
hand_mode[1] = hand_mode[0];
///////////////////////////
//cout << "TEMP.x=" << TEMP.x << "\tTEMP.y=" << TEMP.y << endl;
imshow("result", show_img);
show_img.release();
if (cvWaitKey(20) == 'q')
break;
}
}
return 0;
}
//肤色提取，skinArea为二值化肤色图像
void skinExtract(const Mat &frame, Mat &skinArea)
{
Mat YCbCr;
vector<Mat> planes;
//转换为YCrCb颜色空间
cvtColor(frame, YCbCr, CV_RGB2YCrCb);
//将多通道图像分离为多个单通道图像
split(YCbCr, planes);
//运用迭代器访问矩阵元素
MatIterator_<uchar> it_Cb = planes[1].begin<uchar>(), it_Cb_end = planes[1].end<uchar>();
MatIterator_<uchar> it_Cr = planes[2].begin<uchar>();
MatIterator_<uchar> it_skin = skinArea.begin<uchar>();
//人的皮肤颜色在YCbCr色度空间的分布范围:100<=Cb<=127, 138<=Cr<=170
for (; it_Cb != it_Cb_end; ++it_Cr, ++it_Cb, ++it_skin)
{
if (138 <= *it_Cr && *it_Cr <= 170 && 100 <= *it_Cb && *it_Cb <= 127)
*it_skin = 255;
else
*it_skin = 0;
}
//膨胀和腐蚀，膨胀可以填补凹洞（将裂缝桥接），腐蚀可以消除细的凸起（“斑点”噪声）
dilate(skinArea, skinArea, Mat(5, 5, CV_8UC1), Point(-1, -1));
erode(skinArea, skinArea, Mat(5, 5, CV_8UC1), Point(-1, -1));
}
void on_mouse(int EVENT, int x, int y, int flags, void* userdata)
{
Mat hh;
hh = *(Mat*)userdata;
Point p(x, y);
switch (EVENT)
{
case EVENT_LBUTTONDOWN:
{
//cout << "h=" << hh.at<Vec3b>(p)[0] << endl;
//cout << "s=" << hh.at<Vec3b>(p)[1] << endl;
//cout << "v=" << hh.at<Vec3b>(p)[2] << endl;
printf("h=%d\t", hh.at<Vec3b>(p)[0]);
printf("s=%d\t", hh.at<Vec3b>(p)[1]);
printf("v=%d\n", hh.at<Vec3b>(p)[2]);
circle(hh, p, 2, Scalar(255), 3);
}
break;
}
}
来源：CSDN
作者：blackfly043
链接：https://blog.csdn.net/weixin_43288132/article/details/103641397