人脸识别系列三 | MTCNN算法详解下篇

    // 缩放到12为止
	int MIN_DET_SIZE = 12;
	// 可以检测的最小人脸
	int minsize = 40;
	float m = (float)MIN_DET_SIZE / minsize;
	minl *= m;
	float factor = 0.709;
	int factor_count = 0;
	vector<float> scales_;
	while (minl>MIN_DET_SIZE) {
		if (factor_count>0)m = m*factor;
		scales_.push_back(m);
		minl *= factor;
		factor_count++;
	}

for (size_t i = 0; i < scales_.size(); i++) {
		int hs = (int)ceil(img_h*scales_[i]);
		int ws = (int)ceil(img_w*scales_[i]);
		//ncnn::Mat in = ncnn::Mat::from_pixels_resize(image_data, ncnn::Mat::PIXEL_RGB2BGR, img_w, img_h, ws, hs);
		ncnn::Mat in;
		resize_bilinear(img_, in, ws, hs);
		//in.substract_mean_normalize(mean_vals, norm_vals);
		ncnn::Extractor ex = Pnet.create_extractor();
		ex.set_light_mode(true);
		ex.input("data", in);
		ncnn::Mat score_, location_;
		ex.extract("prob1", score_);
		ex.extract("conv4-2", location_);
		std::vector<Bbox> boundingBox_;
		std::vector<orderScore> bboxScore_;
		generateBbox(score_, location_, boundingBox_, bboxScore_, scales_[i]);
		nms(boundingBox_, bboxScore_, nms_threshold[0]);

		for (vector<Bbox>::iterator it = boundingBox_.begin(); it != boundingBox_.end(); it++) {
			if ((*it).exist) {
				firstBbox_.push_back(*it);
				order.score = (*it).score;
				order.oriOrder = count;
				firstOrderScore_.push_back(order);
				count++;
			}
		}
		bboxScore_.clear();
		boundingBox_.clear();
	}

// 根据Pnet的输出结果，由滑框的得分，筛选可能是人脸的滑框，并记录该框的位置、人脸坐标信息、得分以及编号
void mtcnn::generateBbox(ncnn::Mat score, ncnn::Mat location, std::vector<Bbox>& boundingBox_, std::vector<orderScore>& bboxScore_, float scale) {
	int stride = 2;
	int cellsize = 12;
	int count = 0;
	//score p 判定为人脸的概率
	float *p = score.channel(1);
	// 人脸框回归偏移量
	float *plocal = location.channel(0);
	Bbox bbox;
	orderScore order;
	for (int row = 0; row<score.h; row++) {
		for (int col = 0; col<score.w; col++) {
			if (*p>threshold[0]) {
				bbox.score = *p;
				order.score = *p;
				order.oriOrder = count;
				// 对应原图中的坐标
				bbox.x1 = round((stride*col + 1) / scale);
				bbox.y1 = round((stride*row + 1) / scale);
				bbox.x2 = round((stride*col + 1 + cellsize) / scale);
				bbox.y2 = round((stride*row + 1 + cellsize) / scale);
				bbox.exist = true;
				// 在原图中的大小
				bbox.area = (bbox.x2 - bbox.x1)*(bbox.y2 - bbox.y1);
				// 当前人脸框的回归坐标
				for (int channel = 0; channel<4; channel++)
					bbox.regreCoord[channel] = location.channel(channel)[0];
				boundingBox_.push_back(bbox);
				bboxScore_.push_back(order);
				count++;
			}
			p++;
			plocal++;
		}
	}
}

void mtcnn::nms(std::vector<Bbox> &boundingBox_, std::vector<orderScore> &bboxScore_, const float overlap_threshold, string modelname) {
	if (boundingBox_.empty()) {
		return;
	}
	std::vector<int> heros;
	//sort the score
	sort(bboxScore_.begin(), bboxScore_.end(), cmpScore);

	int order = 0;
	float IOU = 0;
	float maxX = 0;
	float maxY = 0;
	float minX = 0;
	float minY = 0;
	// 规则，站上擂台的擂台主，永远都是胜利者
	while (bboxScore_.size()>0) {
		order = bboxScore_.back().oriOrder; //取得分最高勇士的编号ID
		bboxScore_.pop_back(); // 勇士出列
		if (order<0)continue; //死的？下一个！（order在(*it).oriOrder = -1;改变）
		if (boundingBox_.at(order).exist == false) continue; //记录擂台主ID
		heros.push_back(order);
		boundingBox_.at(order).exist = false;//当前这个Bbox为擂台主，签订生死簿

		for (int num = 0; num<boundingBox_.size(); num++) {
			if (boundingBox_.at(num).exist) {// 活着的勇士
				//the iou
				maxX = (boundingBox_.at(num).x1>boundingBox_.at(order).x1) ? boundingBox_.at(num).x1 : boundingBox_.at(order).x1;
				maxY = (boundingBox_.at(num).y1>boundingBox_.at(order).y1) ? boundingBox_.at(num).y1 : boundingBox_.at(order).y1;
				minX = (boundingBox_.at(num).x2<boundingBox_.at(order).x2) ? boundingBox_.at(num).x2 : boundingBox_.at(order).x2;
				minY = (boundingBox_.at(num).y2<boundingBox_.at(order).y2) ? boundingBox_.at(num).y2 : boundingBox_.at(order).y2;
				//maxX1 and maxY1 reuse
				maxX = ((minX - maxX + 1)>0) ? (minX - maxX + 1) : 0;
				maxY = ((minY - maxY + 1)>0) ? (minY - maxY + 1) : 0;
				//IOU reuse for the area of two bbox
				IOU = maxX * maxY;
				if (!modelname.compare("Union"))
					IOU = IOU / (boundingBox_.at(num).area + boundingBox_.at(order).area - IOU);
				else if (!modelname.compare("Min")) {
					IOU = IOU / ((boundingBox_.at(num).area<boundingBox_.at(order).area) ? boundingBox_.at(num).area : boundingBox_.at(order).area);
				}
				if (IOU>overlap_threshold) {
					boundingBox_.at(num).exist = false; //如果该对比框与擂台主的IOU够大，挑战者勇士战死
					for (vector<orderScore>::iterator it = bboxScore_.begin(); it != bboxScore_.end(); it++) {
						if ((*it).oriOrder == num) {
							(*it).oriOrder = -1;//勇士战死标志
							break;
						}
					}
				}
				//那些距离擂台主比较远迎战者幸免于难，将有机会作为擂台主出现
			}
		}
	}
	//从生死簿上剔除，擂台主活下来了
	for (int i = 0; i<heros.size(); i++)
		boundingBox_.at(heros.at(i)).exist = true;
}

//second stage
	count = 0;
	for (vector<Bbox>::iterator it = firstBbox_.begin(); it != firstBbox_.end(); it++) {
		if ((*it).exist) {
			ncnn::Mat tempIm;
			copy_cut_border(img, tempIm, (*it).y1, img_h - (*it).y2, (*it).x1, img_w - (*it).x2);
			ncnn::Mat in;
			resize_bilinear(tempIm, in, 24, 24);
			ncnn::Extractor ex = Rnet.create_extractor();
			ex.set_light_mode(true);
			ex.input("data", in);
			ncnn::Mat score, bbox;
			ex.extract("prob1", score);
			ex.extract("conv5-2", bbox);
			if ((score[1])>threshold[1]) {
				for (int channel = 0; channel<4; channel++)
					it->regreCoord[channel] = bbox[channel];
				it->area = (it->x2 - it->x1)*(it->y2 - it->y1);
				it->score = score[1];
				secondBbox_.push_back(*it);
				order.score = it->score;
				order.oriOrder = count++;
				secondBboxScore_.push_back(order);
			}
			else {
				(*it).exist = false;
			}
		}
	}
	printf("secondBbox_.size()=%d\n", secondBbox_.size());
	if (count<1)return;
	nms(secondBbox_, secondBboxScore_, nms_threshold[1]);
	refineAndSquareBbox(secondBbox_, img_h, img_w);

count = 0;
	for (vector<Bbox>::iterator it = secondBbox_.begin(); it != secondBbox_.end(); it++) {
		if ((*it).exist) {
			ncnn::Mat tempIm;
			copy_cut_border(img, tempIm, (*it).y1, img_h - (*it).y2, (*it).x1, img_w - (*it).x2);
			ncnn::Mat in;
			resize_bilinear(tempIm, in, 48, 48);
			ncnn::Extractor ex = Onet.create_extractor();
			ex.set_light_mode(true);
			ex.input("data", in);
			ncnn::Mat score, bbox, keyPoint;
			ex.extract("prob1", score);
			ex.extract("conv6-2", bbox);
			ex.extract("conv6-3", keyPoint);
			if (score[1]>threshold[2]) {
				for (int channel = 0; channel<4; channel++)
					it->regreCoord[channel] = bbox[channel];
				it->area = (it->x2 - it->x1)*(it->y2 - it->y1);
				it->score = score[1];
				for (int num = 0; num<5; num++) {
					(it->ppoint)[num] = it->x1 + (it->x2 - it->x1)*keyPoint[num];
					(it->ppoint)[num + 5] = it->y1 + (it->y2 - it->y1)*keyPoint[num + 5];
				}

				thirdBbox_.push_back(*it);
				order.score = it->score;
				order.oriOrder = count++;
				thirdBboxScore_.push_back(order);
			}
			else
				(*it).exist = false;
		}
	}

	printf("thirdBbox_.size()=%d\n", thirdBbox_.size());