在日常生活开发中,我们时常遇到需要自动化完成的重复性任务,比如自动化测试,还记得在某银行开发某某通软件时,开发要辅助测试,每次项目上线后都要群里发100条消息,真的苦不堪言,每次发版后都要测试(因为之前出现过消息丢失),在比如游戏辅助,比如读取桌面,在桌面内进行人脸识别找到头部,然后鼠标移动到头部,按下鼠标左键进行射击(不要骂我哦,我没有开挂),再比如完成一些日常任务啥的
Java中的Robot类是用于模拟鼠标和键盘输入的工具,它主要用于自动化测试、屏幕捕捉、模拟用户交互等任务。Robot类提供了以下主要功能:
OpenCV(开源计算机视觉库)是一个开源的计算机视觉和机器学习软件库,提供了丰富的功能和工具,用于处理图像和视频数据。其主要功能包括但不限于以下几个方面:
<?xml version="1.0" encoding="UTF-8"?>
<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 https://maven.apache.org/xsd/maven-4.0.0.xsd">
<modelVersion>4.0.0</modelVersion>
<parent>
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-starter-parent</artifactId>
<version>2.2.2.RELEASE</version>
<relativePath/>
</parent>
<groupId>com.lxw</groupId>
<artifactId>robot</artifactId>
<version>0.0.1-SNAPSHOT</version>
<name>robot-study</name>
<description>robot-study</description>
<properties>
<java.version>1.8</java.version>
</properties>
<dependencies>
<dependency>
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-starter-web</artifactId>
</dependency>
<dependency>
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-starter-test</artifactId>
<scope>test</scope>
</dependency>
<dependency>
<groupId>org.openpnp</groupId>
<artifactId>opencv</artifactId>
<version>4.8.1-0</version>
</dependency>
</dependencies>
<build>
<plugins>
<plugin>
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-maven-plugin</artifactId>
</plugin>
</plugins>
</build>
</project>
public static void loadOpencv() {
// 加载opencv库
System.load(new File("src/main/resources/lib/opencv/opencv_java490.dll").getAbsolutePath());
}
mat可以理解为图片的矩阵形式
public static Mat desktopMat(Robot robot) {
BufferedImage bi = robot.createScreenCapture(new Rectangle(Toolkit.getDefaultToolkit().getScreenSize()));
int type = bi.getType();
System.out.println("type = " + type);
int width = bi.getWidth();
int height = bi.getHeight();
int[] pixels = bi.getRGB(0, 0, width, height, null, 0, width);
Mat mat = new Mat(height, width, CvType.CV_8UC3);
byte[] data = new byte[width * height * 3];
int index = 0;
for (int y = 0; y < height; y++) {
for (int x = 0; x < width; x++) {
int pixel = pixels[y * width + x];
// 提取 RGB 值并转换为 BGR
data[index++] = (byte) (pixel & 0xFF); // B
data[index++] = (byte) ((pixel >> 8) & 0xFF); // G
data[index++] = (byte) ((pixel >> 16) & 0xFF); // R
}
}
mat.put(0, 0, data);
return mat;
}
public static Mat imread(String filename) {
Mat img = Imgcodecs.imread(filename);
// 检查图像是否成功加载
if (img.empty()) {
throw new RuntimeException("读取图片失败:" + filename);
}
return img;
}
public static org.opencv.core.Point findImage(Mat src, Mat template) {
// 创建结果图像
Mat result = new Mat();
// 进行模板匹配
Imgproc.matchTemplate(src, template, result, Imgproc.TM_CCOEFF_NORMED);
// 设置匹配阈值
double threshold = 0.8;
// 在原始图像中寻找匹配结果
Core.MinMaxLocResult mmr = Core.minMaxLoc(result);
org.opencv.core.Point maxLoc = mmr.maxLoc;
if (mmr.maxVal >= threshold) {
// 计算匹配区域的右下角坐标
org.opencv.core.Point matchLoc = new org.opencv.core.Point(maxLoc.x + template.cols(), maxLoc.y + template.rows());
// 绘制矩形框 此步骤只为展示效果,可注释掉
Imgproc.rectangle(src, maxLoc, matchLoc, new Scalar(0, 255, 0), 2);
// 在匹配区域的中心点绘制红色的点 此步骤只为展示效果,可注释掉
Point center = new Point((maxLoc.x + matchLoc.x) / 2, (maxLoc.y + matchLoc.y) / 2);
Imgproc.circle(src, center, 5, new Scalar(0, 0, 255), -1);
// 显示结果 此步骤只为展示效果,可注释掉
HighGui.imshow("Match Result", src);
// 此步骤只为展示效果,可注释掉
HighGui.waitKey();
return center;
} else {
throw new RuntimeException("没有搜索到目标图标");
}
}
public void move(Robot robot,org.opencv.core.Point point){
robot.mouseMove((int)point.x,(int)point.y);
}
测试代码
public static void main(String[] args) {
try {
loadOpencv();
// 创建 Robot 对象
Robot robot = new Robot();
robot.setAutoDelay(500);
// 模拟ctrl+d快捷键切换到桌面
robot.keyPress(KeyEvent.VK_WINDOWS);
robot.keyPress(KeyEvent.VK_D);
robot.keyRelease(KeyEvent.VK_D);
robot.keyRelease(KeyEvent.VK_WINDOWS);
robot.delay(1000);
// 读取桌面截图
Mat desktop = desktopMat(robot);
// 读取 idea图片
Mat idea = Imgcodecs.imread("src/main/resources/img/find/idea.png");
// 区域找图并返回中心点
Point center = findImage(desktop, idea);
// 移动到中心点
move(robot, center);
} catch (Exception ex) {
ex.printStackTrace();
}
}
在桌面范围内寻找idea图标,找到了用绿色框框起来idea图标,用红色点标记中心位置,
Opencv库路径
Opencv类库见附件
import lombok.Data;
import lombok.extern.slf4j.Slf4j;
import org.opencv.calib3d.Calib3d;
import org.opencv.core.Core;
import org.opencv.core.CvType;
import org.opencv.core.DMatch;
import org.opencv.core.Mat;
import org.opencv.core.MatOfByte;
import org.opencv.core.MatOfDMatch;
import org.opencv.core.MatOfKeyPoint;
import org.opencv.core.MatOfPoint2f;
import org.opencv.core.Point;
import org.opencv.core.Rect;
import org.opencv.core.Scalar;
import org.opencv.core.Size;
import org.opencv.features2d.DescriptorMatcher;
import org.opencv.features2d.Feature2D;
import org.opencv.features2d.Features2d;
import org.opencv.features2d.ORB;
import org.opencv.features2d.SIFT;
import org.opencv.highgui.HighGui;
import org.opencv.imgcodecs.Imgcodecs;
import org.opencv.imgproc.Imgproc;
import java.awt.*;
import java.awt.image.BufferedImage;
import java.io.File;
import java.util.ArrayList;
import java.util.LinkedHashMap;
import java.util.LinkedList;
import java.util.List;
import java.util.Map;
@Slf4j
public class OpenCVUtil {
static {
// 加载opencv库
try {
// System.load(new File("src/main/resources/lib/opencv/opencv_java490.dll").getAbsolutePath());
System.load(new File("src/main/resources/lib/opencv/opencv_java481.dll").getAbsolutePath());
} catch (Exception e) {
log.error("加载加载opencv库跑出异常", e.getMessage());
}
}
/**
* 读取桌面为Mat
*/
public static Mat desktopMat(Robot robot) {
BufferedImage bi = robot.createScreenCapture(new Rectangle(Toolkit.getDefaultToolkit().getScreenSize()));
int type = bi.getType();
System.out.println("type = " + type);
int width = bi.getWidth();
int height = bi.getHeight();
int[] pixels = bi.getRGB(0, 0, width, height, null, 0, width);
Mat mat = new Mat(height, width, CvType.CV_8UC3);
byte[] data = new byte[width * height * 3];
int index = 0;
for (int y = 0; y < height; y++) {
for (int x = 0; x < width; x++) {
int pixel = pixels[y * width + x];
// 提取 RGB 值并转换为 BGR
data[index++] = (byte) (pixel & 0xFF); // B
data[index++] = (byte) ((pixel >> 8) & 0xFF); // G
data[index++] = (byte) ((pixel >> 16) & 0xFF); // R
}
}
mat.put(0, 0, data);
return mat;
}
/**
* 查找并返回中心点
* 效果还不错
*
* @param original 大图
* @param target 需要查找的小图
* @return 找到的小图中心点的坐标
*/
public static Point findImage(String original, String target) {
// 读取原始图像和目标图像
Mat src = imread(original);
Mat template = imread(target);
return findImage(src, template);
}
public static Point findImage(Mat src, String target) {
// 读取原始图像和目标图像
Mat template = imread(target);
return findImage(src, template);
}
public static Point findImage(Mat src, Mat template) {
// 创建结果图像
Mat result = new Mat();
// 进行模板匹配
Imgproc.matchTemplate(src, template, result, Imgproc.TM_CCOEFF_NORMED);
// 设置匹配阈值
double threshold = 0.8;
// 在原始图像中寻找匹配结果
Core.MinMaxLocResult mmr = Core.minMaxLoc(result);
Point maxLoc = mmr.maxLoc;
if (mmr.maxVal >= threshold) {
// 计算匹配区域的右下角坐标
Point matchLoc = new Point(maxLoc.x + template.cols(), maxLoc.y + template.rows());
// // 输出匹配区域的中心点坐标
// System.out.println("Match found at: (" + (matchLoc.x + maxLoc.x) / 2 + ", " + (matchLoc.y + maxLoc.y) / 2 + ")");
// 绘制矩形框
// Imgproc.rectangle(src, maxLoc, matchLoc, new Scalar(0, 255, 0), 2);
// 在匹配区域的中心点绘制红色的点
// Imgproc.circle(src, center, 5, new Scalar(0, 0, 255), -1);
// 显示结果
// HighGui.imshow("Match Result", src);
// HighGui.waitKey();
return new Point((maxLoc.x + matchLoc.x) / 2, (maxLoc.y + matchLoc.y) / 2);
} else {
throw new RuntimeException("没有搜索到目标图标");
}
}
/**
* 读取图片
*
* @param filename 图片文件全路径
* @return 图片矩阵
*/
public static Mat imread(String filename) {
Mat img = Imgcodecs.imread(filename);
// 检查图像是否成功加载
if (img.empty()) {
throw new RuntimeException("读取图片失败:" + filename);
}
return img;
}
public static Mat imread(byte[] bytes) {
MatOfByte matOfByte = new MatOfByte(bytes);
Mat img = Imgcodecs.imdecode(matOfByte, Imgcodecs.IMREAD_COLOR);
// 检查图像是否成功加载
if (img.empty()) {
throw new RuntimeException("读取图片失败");
}
return img;
}
/**
* 写图片
*
* @param filename 图片文件全路径
* @param img 图片矩阵
* @return 是否写成功
*/
public static boolean imwrite(String filename, Mat img) {
return Imgcodecs.imwrite(filename, img);
}
/**
* 生产灰度图片
*
* @param img 图片矩阵
* @return 绘图图片
*/
public static Mat cvtColor(Mat img) {
Mat gray = new Mat();
Imgproc.cvtColor(img, gray, Imgproc.COLOR_BGR2GRAY);
return gray;
}
public static Mat threshold(Mat img) {
Mat thresh = new Mat();
Imgproc.threshold(cvtColor(img), thresh, 0, 255, Imgproc.THRESH_BINARY_INV + Imgproc.THRESH_OTSU);
return thresh;
}
/**
* 用ORB获取关键的和计算描述子
* 特点:准
*
* @param img 图片矩阵
* @return 特征点和描述子
*/
public static KeyPointAndDescriptor detectAndCompute4ORB(Mat img) {
return detectAndCompute(img, ORB.create());
}
/**
* 用SIFT获取关键的和计算描述子
* 特点:快
*
* @param img 图片矩阵
* @return 特征点和描述子
*/
public static KeyPointAndDescriptor detectAndCompute4SIFT(Mat img) {
return detectAndCompute(img, SIFT.create());
}
private static KeyPointAndDescriptor detectAndCompute(Mat img, Feature2D detector) {
Mat gray = cvtColor(img);
// 检测图像中的关键点
MatOfKeyPoint keyPoint = new MatOfKeyPoint();
// 计算关键点的描述子
Mat descriptor = new Mat();
// 获取关键点并计算描述子,更常用 第二个参数 Mat mask,感兴趣的区域,空对象表示对整个图操作
detector.detectAndCompute(gray, new Mat(), keyPoint, descriptor);
return new KeyPointAndDescriptor(keyPoint, descriptor);
}
/**
* 绘制关键点
*
* @param img 图像矩阵
* @param keyPoint 关键点
* @return 包含关键点的图片
*/
private static Mat drawKeypoints(Mat img, MatOfKeyPoint keyPoint) {
// 绘制关键点
Mat imgWithKeyPoints = new Mat();
Features2d.drawKeypoints(img, keyPoint, imgWithKeyPoints);
return imgWithKeyPoints;
}
/**
* 查找图片并显示矩形和中心点
*
* @param original 大图
* @param target 小图
*/
public static void findImageWithShow(String original, String target) {
// 读取原始图像和目标图像
Mat src = imread(original);
Mat template = imread(target);
findImageWithShow(src, template);
}
public static void findImageWithShow(Mat src, Mat template) {
// 创建结果图像
Mat result = new Mat();
// 进行模板匹配
Imgproc.matchTemplate(src, template, result, Imgproc.TM_CCOEFF_NORMED);
// 设置匹配阈值
double threshold = 0.8;
// 在原始图像中寻找匹配结果
Core.MinMaxLocResult mmr = Core.minMaxLoc(result);
Point maxLoc = mmr.maxLoc;
if (mmr.maxVal >= threshold) {
// 计算匹配区域的右下角坐标
Point matchLoc = new Point(maxLoc.x + template.cols(), maxLoc.y + template.rows());
// // 输出匹配区域的中心点坐标
System.out.println("Match found at: (" + (matchLoc.x + maxLoc.x) / 2 + ", " + (matchLoc.y + maxLoc.y) / 2 + ")");
// 绘制矩形框
Imgproc.rectangle(src, maxLoc, matchLoc, new Scalar(0, 255, 0), 2);
// 在匹配区域的中心点绘制红色的点
Point center = new Point((maxLoc.x + matchLoc.x) / 2, (maxLoc.y + matchLoc.y) / 2);
Imgproc.circle(src, center, 5, new Scalar(0, 0, 255), -1);
// 显示结果
HighGui.imshow("Match Result", src);
HighGui.waitKey();
} else {
HighGui.imshow("Match Result", src);
HighGui.waitKey();
// throw new RuntimeException("没有搜索到目标图标");
}
}
/**
* 关键的和描述子
* 描述子:对关键点有贡献的像素
*/
@Data
public static class KeyPointAndDescriptor {
MatOfKeyPoint keyPoint;
Mat descriptor;
public KeyPointAndDescriptor(MatOfKeyPoint keyPoint, Mat descriptor) {
this.keyPoint = keyPoint;
this.descriptor = descriptor;
}
}
public static Mat montage(String imageName1, String imageName2) {
Mat image1 = imread(imageName1);
Mat image2 = imread(imageName2);
int cols1 = image1.cols();
int cols2 = image2.cols();
int rows1 = image1.rows();
int rows2 = image2.rows();
if (cols1 == cols2) {
return montage(image1, image2, false, 50);
}
if (rows1 == rows2) {
return montage(image1, image2, true, 50);
}
throw new RuntimeException("宽高都不相同,无法判断是上下拼接或者左右拼接");
}
public static Mat montage(String imageName1, String imageName2, boolean isLeftRight) {
return montage(imageName1, imageName2, isLeftRight, 50);
}
public static Mat montage(String imageName1, String imageName2, boolean isLeftRight, int distance) {
Mat image1 = imread(imageName1);
Mat image2 = imread(imageName2);
int cols1 = image1.cols();
int cols2 = image2.cols();
int rows1 = image1.rows();
int rows2 = image2.rows();
if (isLeftRight) {
if (rows1 != rows2) {
Size imageSize = new Size(image2.cols(), Math.max(image1.rows(), image2.rows()));
Imgproc.resize(image2, image2, imageSize);
}
} else {
if (cols1 != cols2) {
Size imageSize = new Size(image2.rows(), Math.max(image1.cols(), image2.cols()));
Imgproc.resize(image2, image2, imageSize);
}
}
return montage(image1, image2, isLeftRight, distance);
}
/**
* 拼图
*
* @param image1 左边的或者上边的
* @param image2 右边的或者下边的
* @param isLeftRight 是否为左右拼图
* @param distance 相似度 值越小要求相似度越高
*/
public static Mat montage(Mat image1, Mat image2, boolean isLeftRight, int distance) {
// 获取特征点和描述子
KeyPointAndDescriptor keyPointAndDescriptor1 = detectAndCompute4SIFT(image1);
KeyPointAndDescriptor keyPointAndDescriptor2 = detectAndCompute4SIFT(image2);
MatOfKeyPoint keyPoint1 = keyPointAndDescriptor1.getKeyPoint();
MatOfKeyPoint keyPoint2 = keyPointAndDescriptor2.getKeyPoint();
Mat descriptors1 = keyPointAndDescriptor1.getDescriptor();
Mat descriptors2 = keyPointAndDescriptor2.getDescriptor();
// 创建暴力匹配器
DescriptorMatcher matcher = DescriptorMatcher.create(DescriptorMatcher.BRUTEFORCE);
MatOfDMatch matches = new MatOfDMatch();
matcher.match(descriptors1, descriptors2, matches);
// 筛选匹配
LinkedList<DMatch> goodMatchesList = new LinkedList<>();
List<DMatch> matchesList = matches.toList();
for (DMatch match : matchesList) {
// 调整阈值以筛选好的匹配 两个特征点之间的距离(相似度).距离越短,相似度越高
if (match.distance < distance) {
goodMatchesList.addLast(match);
}
}
// 提取匹配点对
List<Point> objList = new LinkedList<>();
List<Point> sceneList = new LinkedList<>();
for (int i = 0; i < goodMatchesList.size(); i++) {
objList.add(keyPoint1.toList().get(goodMatchesList.get(i).queryIdx).pt);
sceneList.add(keyPoint2.toList().get(goodMatchesList.get(i).trainIdx).pt);
}
MatOfPoint2f obj = new MatOfPoint2f();
obj.fromList(objList);
MatOfPoint2f scene = new MatOfPoint2f();
scene.fromList(sceneList);
if (objList.size() < 4) {
throw new RuntimeException("计算单应性矩阵所需的点对数量至少为 4 对");
}
// 计算单应性矩阵
Mat homography = Calib3d.findHomography(scene, obj, Calib3d.RANSAC, 5);
Size size;
if (isLeftRight) {
size = new Size(image1.cols() + image2.cols(), image1.rows());
} else {
size = new Size(image1.cols(), image1.rows() + image2.rows());
}
// 使用单应性矩阵进行图像配准
Mat result = new Mat();
Imgproc.warpPerspective(image2, result, homography, size);
Mat roi = new Mat(result, new Rect(0, 0, image1.cols(), image1.rows()));
image1.copyTo(roi);
return result;
}
/**
* 通道分离
*
* @param mat
* @return
*/
public static List<Mat> split(Mat mat) {
List<Mat> channels = new ArrayList<>();
Core.split(mat, channels);
return channels;
}
/**
* 图片显示
*
* @param mats
*/
// public static void show(LinkedHashMap<String,Mat> mats) {
// // 获取默认工具包
// Toolkit toolkit = Toolkit.getDefaultToolkit();
// // 获取屏幕的尺寸
// Dimension screenSize = toolkit.getScreenSize();
// // 输出屏幕宽度和高度
// int i = 0;
// int columnCount = 0;
// for (Map.Entry<String, Mat> entry : mats.entrySet()) {
// Mat mat = entry.getValue();
// String winName = entry.getKey();
// System.out.println("winName = " + winName);
// HighGui.imshow(winName, mat);
// int xSpace = 10;
// int count = i / (screenSize.width / (mat.cols()+ xSpace));
// int y = count*(mat.rows()+40);
// int x = (mat.cols()+ xSpace)*columnCount;
// if(x+mat.cols() > screenSize.width){
// columnCount = 0;
// x = (mat.cols()+ xSpace)*columnCount;
// }
// columnCount++;
// HighGui.moveWindow(winName,x,y);
// i++;
// }
// }
public static void show(LinkedHashMap<String, Mat> mats) {
// 获取默认工具包
Toolkit toolkit = Toolkit.getDefaultToolkit();
// 获取屏幕的尺寸
Dimension screenSize = toolkit.getScreenSize();
// 输出屏幕宽度和高度
int i = 0;
int x = 0;
int y = 0;
int xSpacing = 10;
int ySpacing = 40;
for (Map.Entry<String, Mat> entry : mats.entrySet()) {
Mat mat = entry.getValue();
String winName = entry.getKey();
System.out.println("winName = " + winName);
HighGui.imshow(winName, mat);
if (i > 0) {
x += (mat.cols() + xSpacing);
if (x + mat.cols() > screenSize.width) {
x = 0;
y += (mat.rows() + ySpacing);
}
}
HighGui.moveWindow(winName, x, y);
i++;
}
}
/**
* 高斯降噪
*
* @param mat
*/
public static void gaussianBlur(Mat mat) {
Imgproc.GaussianBlur(mat, mat, new Size(5, 5), 0);
}
/**
* 开运算去除内部早点
*
* @param mat
*/
public static void morphOpen(Mat mat) {
Mat kernel = Imgproc.getStructuringElement(Imgproc.MORPH_RECT, new Size(5, 5));
Imgproc.morphologyEx(mat, mat, Imgproc.MORPH_OPEN, kernel);
}
public static void morphClose(Mat mat) {
Mat kernel = Imgproc.getStructuringElement(Imgproc.MORPH_RECT, new Size(5, 5));
Imgproc.morphologyEx(mat, mat, Imgproc.MORPH_CLOSE, kernel);
}
public static void morphOpenClose(Mat mat) {
morphOpen(mat);
morphClose(mat);
}
public static Mat sketch(Mat mat) {
Mat gray = cvtColor(mat);
// 高斯模糊
Mat blurredImage = new Mat();
Imgproc.GaussianBlur(gray, blurredImage, new Size(21, 21), 0);
// 融合原始灰度图像和模糊图像
Mat sketchImage = new Mat();
Core.divide(gray, blurredImage, sketchImage, 256.0);
return sketchImage;
}
}