音视频开发之旅（72）- AI数字人-照片说话之SadTalker

#将人脸从图像中裁剪出来，并提取人脸的关键点以及3DMM形态模型self.preprocess_model = CropAndExtract(path_of_lm_croper, path_of_net_recon_model, dir_of_BFM_fitting, self.device)#将音频转换为控制面部表情特别是唇部运动的系数self.audio_to_coeff = Audio2Coeff(audio2pose_checkpoint, audio2pose_yaml_path,                             audio2exp_checkpoint, audio2exp_yaml_path,                             wav2lip_checkpoint, self.device)#生成口型动画视频self.animate_from_coeff = AnimateFromCoeff(free_view_checkpoint, mapping_checkpoint,                             facerender_yaml_path, self.device)

#3DMM Extractionfirst_coeff_path, crop_pic_path, crop_info =  self.preprocess_model.generate(pic_path, first_frame_dir, self.preprocess)
batch = get_data(first_coeff_path, audio_path, self.device, ref_eyeblink_coeff_path, still=still)coeff_path = self.audio_to_coeff.generate(batch, save_dir, pose_style, ref_pose_coeff_path)
data = get_facerender_data(coeff_path, crop_pic_path, first_coeff_path, audio_path,                                     batch_size, input_yaw_list, input_pitch_list, input_roll_list,                                    expression_scale=expression_scale, still_mode=still, preprocess=self.preprocess)                                    return_path = self.animate_from_coeff.generate(data, save_dir, pic_path, crop_info, \        enhancer=enhancer, background_enhancer=background_enhancer, preprocess=self.preprocess)

#内部通过dlib进行人脸检测 获取人脸68个关键点，并提取面部关键点和3DMM（三维形态模型）参数x_full_frames, crop, quad = self.croper.crop(x_full_frames_before, still=True, xsize=pic_size)clx, cly, crx, cry = croplx, ly, rx, ry = quadlx, ly, rx, ry = int(lx), int(ly), int(rx), int(ry)oy1, oy2, ox1, ox2 = cly+ly, cly+ry, clx+lx, clx+rxcrop_info = ((ox2 - ox1, oy2 - oy1), crop, quad)
#get the landmark according to the detected facelm = self.kp_extractor.extract_keypoint(frames_pil, landmarks_path)
# load 3dmm paramter generator from Deep3DFaceRecon_pytorchsavemat(coeff_path, {'coeff_3dmm': semantic_npy, 'full_3dmm': np.array(full_coeffs)[0]})

#sadtalker/src/test_audio2coeff.py
exp_pred = results_dict_exp['exp_coeff_pred']   results_dict_pose = self.audio2pose_model.test(batch) pose_pred = results_dict_pose['pose_pred'] 
coeffs_pred = torch.cat((exp_pred, pose_pred), dim=-1)            #bs T 70coeffs_pred_numpy = coeffs_pred[0].clone().detach().cpu().numpy() savemat(os.path.join(coeff_save_dir, '%s.mat'%(batch['pic_name'])),                      {'coeff_3dmm': coeffs_pred_numpy})

#sadtalker/src/facerender/animate.pydef generate(self, x, video_save_dir, pic_path, crop_info, enhancer=None, background_enhancer=None, preprocess='crop'):    #根据3dmm人脸模型数据以及音频驱动口型数据和关键点信息等 生成视频帧    predictions_video = make_animation(source_image, source_semantics, target_semantics,                                        self.generator, self.kp_extractor, self.he_estimator, self.mapping,                                         yaw_c_seq, pitch_c_seq, roll_c_seq, use_exp = True)    predictions_video = predictions_video.reshape((-1,)+predictions_video.shape[2:])    predictions_video = predictions_video[:frame_num]        ### the generated video is 256x256, so we  keep the aspect ratio,     #生成256x256的人头口型说话视频    imageio.mimsave(path, result, fps=float(25))        #对音频进行重采样到16000    sound = AudioSegment.from_mp3(audio_path)    frames = frame_num     end_time = start_time + frames*1/25*1000    word1=sound.set_frame_rate(16000)    word = word1[start_time:end_time]    word.export(new_audio_path, format="wav")        #合并音轨和视轨    save_video_with_watermark(path, new_audio_path, av_path, watermark= None)        #把裁剪人脸的生成的对口型视频再贴回到全身图生成全身视频    paste_pic(path, pic_path, crop_info, new_audio_path, full_video_path)            #对人脸进行画质增强        enhanced_images = face_enhancer(full_video_path, method=enhancer, bg_upsampler=background_enhancer)

```#src/utils/paste_pic.py
修改如下# tmp_path = str(uuid.uuid4())+'.mp4'
    # out_tmp = cv2.VideoWriter(full_video_path, cv2.VideoWriter_fourcc(*'MP4V'), fps, (frame_w, frame_h))    # for crop_frame in tqdm(crop_frames, 'seamlessClone:'):    #     p = cv2.resize(crop_frame.astype(np.uint8), (crx-clx, cry - cly)) 
    #     mask = 255*np.ones(p.shape, p.dtype)    #     location = ((ox1+ox2) // 2, (oy1+oy2) // 2)    #     gen_img = cv2.seamlessClone(p, full_img, mask, location, cv2.NORMAL_CLONE)    #     out_tmp.write(gen_img)
    # 自定义修改开始    def process_image(crop_frame):        p = cv2.resize(crop_frame.astype(np.uint8), (ox2-ox1, oy2 - oy1)) 
        mask = 255*np.ones(p.shape, p.dtype)        location = ((ox1+ox2) // 2, (oy1+oy2) // 2)        gen_img = cv2.seamlessClone(p, full_img, mask, location, cv2.NORMAL_CLONE)
        return gen_img    tmp_path = str(uuid.uuid4())+'.mp4'    out_tmp = cv2.VideoWriter(tmp_path, cv2.VideoWriter_fourcc(*'MP4V'), fps, (frame_w, frame_h))
    processed_frames = []  # 存储处理后的图像
    # 创建线程池    # 指定线程池的最大线程数    max_threads = 10
    # 创建线程池并设置max_workers参数    with futures.ThreadPoolExecutor(max_workers=max_threads) as executor:        # 提交任务并获取处理结果        processed_frames = []        for gen_img in tqdm(executor.map(process_image, crop_frames), total=len(crop_frames) ,desc='seamlessClone:'):            processed_frames.append(gen_img)
    # 一次将所有处理后的图像写入视频文件    for frame in processed_frames:        out_tmp.write(frame)    # 自定义修改结束```
感谢 https://github.com/OpenTalker/SadTalker/issues/520