Android 中使用 dlib+opencv 实现动态人脸检测功能

defaultConfig {
 applicationId "com.example.lightweh.facedetection"
 minSdkVersion 23
 targetSdkVersion 28
 versionCode 1
 versionName "1.0"
 testInstrumentationRunner "android.support.test.runner.AndroidJUnitRunner"
 externalNativeBuild {
 cmake {
  arguments "-DCMAKE_BUILD_TYPE=Release"
  cppFlags "-std=c++11 -frtti -fexceptions"
 }
 }
}

# 设置native目录
set(NATIVE_DIR ${CMAKE_SOURCE_DIR}/src/main/cpp)
# 设置dlib
include(${NATIVE_DIR}/dlib/cmake)
# 设置opencv include文件夹
include_directories(${NATIVE_DIR}/opencv/jni/include)
# 设置opencv的so库
add_library(
 libopencv_java3
 SHARED
 IMPORTED)
set_target_properties(
 libopencv_java3
 PROPERTIES
 IMPORTED_LOCATION
 ${NATIVE_DIR}/opencv/libs/${ANDROID_ABI}/libopencv_java3.so)
# 将jni_common目录中所有文件名，存至SRC_LIST中
AUX_SOURCE_DIRECTORY(${NATIVE_DIR}/jni_common SRC_LIST)
add_library( # Sets the name of the library.
 native-lib
 # Sets the library as a shared library.
 SHARED
 # Provides a relative path to your source file(s).
 ${SRC_LIST}
 src/main/cpp/face_detector.h
 src/main/cpp/face_detector.cpp
 src/main/cpp/native-lib.cpp)
find_library( # Sets the name of the path variable.
 log-lib
 # Specifies the name of the NDK library that
 # you want CMake to locate.
 log)
target_link_libraries( # Specifies the target library.
 native-lib
 dlib
 libopencv_java3
 jnigraphics
 # Links the target library to the log library
 # included in the NDK.
 ${log-lib})
# 指定release编译选项
set(CMAKE_C_FLAGS_RELEASE "${CMAKE_C_FLAGS_RELEASE} -s -O3 -Wall")
set(CMAKE_CXX_FLAGS_RELEASE "${CMAKE_CXX_FLAGS_RELEASE} -s -O3 -Wall")

public final class VisionDetRet {
 private int mLeft;
 private int mTop;
 private int mRight;
 private int mBottom;
 VisionDetRet() {}
 public VisionDetRet(int l, int t, int r, int b) {
 mLeft = l;
 mTop = t;
 mRight = r;
 mBottom = b;
 }
 public int getLeft() {
 return mLeft;
 }
 public int getTop() {
 return mTop;
 }
 public int getRight() {
 return mRight;
 }
 public int getBottom() {
 return mBottom;
 }
}

public class FaceDet {
 private static final String TAG = "FaceDet";
 // accessed by native methods
 @SuppressWarnings("unused")
 private long mNativeFaceDetContext;
 static {
 try {
  // 预加载native方法库
  System.loadLibrary("native-lib");
  jniNativeClassInit();
  Log.d(TAG, "jniNativeClassInit success");
 } catch (UnsatisfiedLinkError e) {
  Log.e(TAG, "library not found");
 }
 }
 public FaceDet() {
 jniInit();
 }
 @Nullable
 @WorkerThread
 public List<VisionDetRet  detect(@NonNull Bitmap bitmap) {
 VisionDetRet[] detRets = jniBitmapDet(bitmap);
 return Arrays.asList(detRets);
 }
 @Override
 protected void finalize() throws Throwable {
 super.finalize();
 release();
 }
 public void release() {
 jniDeInit();
 }
 @Keep
 private native static void jniNativeClassInit();
 @Keep
 private synchronized native int jniInit();
 @Keep
 private synchronized native int jniDeInit();
 @Keep
 private synchronized native VisionDetRet[] jniBitmapDet(Bitmap bitmap);
}

#include <jni.h 
#define CLASSNAME_VISION_DET_RET "com/lightweh/dlib/VisionDetRet"
#define CONSTSIG_VISION_DET_RET "()V"
#define CLASSNAME_FACE_DET "com/lightweh/dlib/FaceDet"
class JNI_VisionDetRet {
public:
 JNI_VisionDetRet(JNIEnv *env) {
 // 查找VisionDetRet类信息
 jclass detRetClass = env- FindClass(CLASSNAME_VISION_DET_RET);
 // 获取VisionDetRet类成员变量
 jID_left = env- GetFieldID(detRetClass, "mLeft", "I");
 jID_top = env- GetFieldID(detRetClass, "mTop", "I");
 jID_right = env- GetFieldID(detRetClass, "mRight", "I");
 jID_bottom = env- GetFieldID(detRetClass, "mBottom", "I");
 }
 void setRect(JNIEnv *env, jobject &jDetRet, const int &left, const int &top,
   const int &right, const int &bottom) {
 // 设置VisionDetRet类对象jDetRet的成员变量值
 env- SetIntField(jDetRet, jID_left, left);
 env- SetIntField(jDetRet, jID_top, top);
 env- SetIntField(jDetRet, jID_right, right);
 env- SetIntField(jDetRet, jID_bottom, bottom);
 }
 // 创建VisionDetRet类实例
 static jobject createJObject(JNIEnv *env) {
 jclass detRetClass = env- FindClass(CLASSNAME_VISION_DET_RET);
 jmethodID mid =
  env- GetMethodID(detRetClass, "<init ", CONSTSIG_VISION_DET_RET);
 return env- NewObject(detRetClass, mid);
 }
 // 创建VisionDetRet类对象数组
 static jobjectArray createJObjectArray(JNIEnv *env, const int &size) {
 jclass detRetClass = env- FindClass(CLASSNAME_VISION_DET_RET);
 return (jobjectArray) env- NewObjectArray(size, detRetClass, NULL);
 }
private:
 jfieldID jID_left;
 jfieldID jID_top;
 jfieldID jID_right;
 jfieldID jID_bottom;
};

class FaceDetector {
private:
 dlib::frontal_face_detector face_detector;
 std::vector<dlib::rectangle  det_rects;
public:
 FaceDetector();
 // 实现人脸检测算法
 int Detect(const cv::Mat &image);
 // 返回检测结果
 std::vector<dlib::rectangle  getDetResultRects();
};
FaceDetector::FaceDetector() {
 // 定义人脸检测器
 face_detector = dlib::get_frontal_face_detector();
}
int FaceDetector::Detect(const cv::Mat &image) {
 if (image.empty())
 return 0;
 if (image.channels() == 1) {
 cv::cvtColor(image, image, CV_GRAY2BGR);
 }
 dlib::cv_image<dlib::bgr_pixel  dlib_image(image);
 det_rects.clear();
 // 返回检测到的人脸矩形特征框
 det_rects = face_detector(dlib_image);
 return det_rects.size();
}
std::vector<dlib::rectangle  FaceDetector::getDetResultRects() {
 return det_rects;
}

JNI_VisionDetRet *g_pJNI_VisionDetRet;
JavaVM *g_javaVM = NULL;
// 该函数在加载本地库时被调用
JNIEXPORT jint JNI_OnLoad(JavaVM *vm, void *reserved) {
g_javaVM = vm;
JNIEnv *env;
vm- GetEnv((void **) &env, JNI_VERSION_1_6);
// 初始化 g_pJNI_VisionDetRet
g_pJNI_VisionDetRet = new JNI_VisionDetRet(env);
return JNI_VERSION_1_6;
}
// 该函数用于执行清理操作
void JNI_OnUnload(JavaVM *vm, void *reserved) {
g_javaVM = NULL;
delete g_pJNI_VisionDetRet;
}
namespace {
#define JAVA_NULL 0
using DetPtr = FaceDetector *;
// 用于存放人脸检测类对象的指针，关联Jave层对象与C++底层对象（相互对应）
class JNI_FaceDet {
public:
JNI_FaceDet(JNIEnv *env) {
jclass clazz = env- FindClass(CLASSNAME_FACE_DET);
mNativeContext = env- GetFieldID(clazz, "mNativeFaceDetContext", "J");
env- DeleteLocalRef(clazz);
}
DetPtr getDetectorPtrFromJava(JNIEnv *env, jobject thiz) {
DetPtr const p = (DetPtr) env- GetLongField(thiz, mNativeContext);
return p;
}
void setDetectorPtrToJava(JNIEnv *env, jobject thiz, jlong ptr) {
env- SetLongField(thiz, mNativeContext, ptr);
}
jfieldID mNativeContext;
};
// Protect getting/setting and creating/deleting pointer between java/native
std::mutex gLock;
std::shared_ptr<JNI_FaceDet  getJNI_FaceDet(JNIEnv *env) {
static std::once_flag sOnceInitflag;
static std::shared_ptr<JNI_FaceDet  sJNI_FaceDet;
std::call_once(sOnceInitflag, [env]() {
sJNI_FaceDet = std::make_shared<JNI_FaceDet (env);
});
return sJNI_FaceDet;
}
// 从java对象获取它持有的c++对象指针
DetPtr const getDetPtr(JNIEnv *env, jobject thiz) {
std::lock_guard<std::mutex  lock(gLock);
return getJNI_FaceDet(env)- getDetectorPtrFromJava(env, thiz);
}
// The function to set a pointer to java and delete it if newPtr is empty
// C++对象new以后，将指针转成long型返回给java对象持有
void setDetPtr(JNIEnv *env, jobject thiz, DetPtr newPtr) {
std::lock_guard<std::mutex  lock(gLock);
DetPtr oldPtr = getJNI_FaceDet(env)- getDetectorPtrFromJava(env, thiz);
if (oldPtr != JAVA_NULL) {
delete oldPtr;
}
getJNI_FaceDet(env)- setDetectorPtrToJava(env, thiz, (jlong) newPtr);
}
} // end unnamespace
#ifdef __cplusplus
extern "C" {
#endif
#define DLIB_FACE_JNI_METHOD(METHOD_NAME) Java_com_lightweh_dlib_FaceDet_##METHOD_NAME
void JNIEXPORT
DLIB_FACE_JNI_METHOD(jniNativeClassInit)(JNIEnv *env, jclass _this) {}
// 生成需要返回的结果数组
jobjectArray getRecResult(JNIEnv *env, DetPtr faceDetector, const int &size) {
// 根据检测到的人脸数创建相应大小的jobjectArray
jobjectArray jDetRetArray = JNI_VisionDetRet::createJObjectArray(env, size);
for (int i = 0; i < size; i++) {
// 对检测到的每一个人脸创建对应的实例对象，然后插入数组
jobject jDetRet = JNI_VisionDetRet::createJObject(env);
env- SetObjectArrayElement(jDetRetArray, i, jDetRet);
dlib::rectangle rect = faceDetector- getDetResultRects()[i];
// 将人脸矩形框的值赋给对应的jobject实例对象
g_pJNI_VisionDetRet- setRect(env, jDetRet, rect.left(), rect.top(),
rect.right(), rect.bottom());
}
return jDetRetArray;
}
JNIEXPORT jobjectArray JNICALL
DLIB_FACE_JNI_METHOD(jniBitmapDet)(JNIEnv *env, jobject thiz, jobject bitmap) {
cv::Mat rgbaMat;
cv::Mat bgrMat;
jniutils::ConvertBitmapToRGBAMat(env, bitmap, rgbaMat, true);
cv::cvtColor(rgbaMat, bgrMat, cv::COLOR_RGBA2BGR);
// 获取人脸检测类指针
DetPtr mDetPtr = getDetPtr(env, thiz);
// 调用人脸检测算法，返回检测到的人脸数
jint size = mDetPtr- Detect(bgrMat);
// 返回检测结果
return getRecResult(env, mDetPtr, size);
}
jint JNIEXPORT JNICALL
DLIB_FACE_JNI_METHOD(jniInit)(JNIEnv *env, jobject thiz) {
DetPtr mDetPtr = new FaceDetector();
// 设置人脸检测类指针
setDetPtr(env, thiz, mDetPtr);
return JNI_OK;
}
jint JNIEXPORT JNICALL
DLIB_FACE_JNI_METHOD(jniDeInit)(JNIEnv *env, jobject thiz) {
// 指针置0
setDetPtr(env, thiz, JAVA_NULL);
return JNI_OK;
}
#ifdef __cplusplus
}
#endif

public class BoundingBoxView extends SurfaceView implements SurfaceHolder.Callback {
protected SurfaceHolder mSurfaceHolder;
private Paint mPaint;
private boolean mIsCreated;
public BoundingBoxView(Context context, AttributeSet attrs) {
super(context, attrs);
mSurfaceHolder = getHolder();
mSurfaceHolder.addCallback(this);
mSurfaceHolder.setFormat(PixelFormat.TRANSPARENT);
setZOrderOnTop(true);
mPaint = new Paint();
mPaint.setAntiAlias(true);
mPaint.setColor(Color.RED);
mPaint.setStrokeWidth(5f);
mPaint.setStyle(Paint.Style.STROKE);
}
@Override
public void surfaceChanged(SurfaceHolder surfaceHolder, int format, int width, int height) {
}
@Override
public void surfaceCreated(SurfaceHolder surfaceHolder) {
mIsCreated = true;
}
@Override
public void surfaceDestroyed(SurfaceHolder surfaceHolder) {
mIsCreated = false;
}
public void setResults(List<VisionDetRet  detRets)
{
if (!mIsCreated) {
return;
}
Canvas canvas = mSurfaceHolder.lockCanvas();
//清除掉上一次的画框。
canvas.drawColor(Color.TRANSPARENT, PorterDuff.Mode.CLEAR);
canvas.drawColor(Color.TRANSPARENT);
for (VisionDetRet detRet : detRets) {
Rect rect = new Rect(detRet.getLeft(), detRet.getTop(), detRet.getRight(), detRet.getBottom());
canvas.drawRect(rect, mPaint);
}
mSurfaceHolder.unlockCanvasAndPost(canvas);
}
}

<?xml version="1.0" encoding="utf-8"? 
<RelativeLayout xmlns:android="http://schemas.android.com/apk/res/android"
xmlns:tools="http://schemas.android.com/tools"
android:layout_width="match_parent"
android:layout_height="match_parent"
tools:context=".CameraFragment" 
<com.lightweh.facedetection.AutoFitTextureView
android:id="@+id/textureView"
android:layout_width="wrap_content"
android:layout_height="wrap_content"
android:layout_centerVertical="true"
android:layout_centerHorizontal="true" / 
<com.lightweh.facedetection.BoundingBoxView
android:id="@+id/boundingBoxView"
android:layout_width="wrap_content"
android:layout_height="wrap_content"
android:layout_alignLeft="@+id/textureView"
android:layout_alignTop="@+id/textureView"
android:layout_alignRight="@+id/textureView"
android:layout_alignBottom="@+id/textureView" / 
</RelativeLayout 

private class detectAsync extends AsyncTask<Bitmap, Void, List<VisionDetRet   {
@Override
protected void onPreExecute() {
mIsDetecting = true;
super.onPreExecute();
}
protected List<VisionDetRet  doInBackground(Bitmap... bp) {
List<VisionDetRet  results;
// 返回检测结果
results = mFaceDet.detect(bp[0]);
return results;
}
protected void onPostExecute(List<VisionDetRet  results) {
// 绘制检测到的人脸矩形框
mBoundingBoxView.setResults(results);
mIsDetecting = false;
}
}

@Override
public void onResume() {
super.onResume();
startBackgroundThread();
mFaceDet = new FaceDet();
if (mTextureView.isAvailable()) {
openCamera(mTextureView.getWidth(), mTextureView.getHeight());
} else {
mTextureView.setSurfaceTextureListener(mSurfaceTextureListener);
}
}
@Override
public void onPause() {
closeCamera();
stopBackgroundThread();
if (mFaceDet != null) {
mFaceDet.release();
}
super.onPause();
}

@Override
public void onSurfaceTextureUpdated(SurfaceTexture texture) {
if (!mIsDetecting) {
Bitmap bp = mTextureView.getBitmap();
// 保证图片方向与预览方向一致
bp = Bitmap.createBitmap(bp, 0, 0, bp.getWidth(), bp.getHeight(), mTextureView.getTransform(null), true );
new detectAsync().execute(bp);
}
}