Core ML简介及实时目标检测及Caffe TensorFlow coremltools模型转换

//VGG16模型的输入类
@interface VGG16Input : NSObject<MLFeatureProvider>

/*
创建输入类，需要传入一个CVPixelBufferRef类型的图像数据
类型需要为kCVPixelFormatType_32BGRA，大小为224*224像素
*/
@property (readwrite, nonatomic) CVPixelBufferRef image;
- (instancetype)init NS_UNAVAILABLE;
//直接使用该构造函数创建对象即可
- (instancetype)initWithImage:(CVPixelBufferRef)image;
@end

//VGG16模型的输出类
@interface VGG16Output : NSObject<MLFeatureProvider>

//前文讲述的1000分类的名称和置信度字典
@property (readwrite, nonatomic) NSDictionary<NSString *, NSNumber *> * classLabelProbs;
//前文讲述的置信度最高的类别名称
@property (readwrite, nonatomic) NSString * classLabel;
//一般不手动创建输出对象，通过模型对象识别时返回结果为输出对象
- (instancetype)init NS_UNAVAILABLE;
//在进行预测是，Core ML会调用该方法创建一个output对象返回给调用者
- (instancetype)initWithClassLabelProbs:(NSDictionary<NSString *, NSNumber *> *)classLabelProbs classLabel:(NSString *)classLabel;
@end

//VGG16模型接口
@interface VGG16 : NSObject
//MLModel即为Core ML抽象的模型对象
@property (readonly, nonatomic, nullable) MLModel * model;
//构造函数，可以直接使用默认构造函数，则默认加载工程文件中名称为VGG16的mlmodel模型文件
//也可以提供远程下载的功能，使用该构造函数来加载模型
- (nullable instancetype)initWithContentsOfURL:(NSURL *)url error:(NSError * _Nullable * _Nullable)error;

/*
进行预测的方法，需要传入VGG16Input对象和一个NSError指针的指针
返回结果为VGG16Ouput对象，从返回的对象中即可获取到识别的结果
*/
- (nullable VGG16Output *)predictionFromFeatures:(VGG16Input *)input error:(NSError * _Nullable * _Nullable)error;

//直接通过CVPixelBufferRef进行预测，省去了创建input对象的步骤
- (nullable VGG16Output *)predictionFromImage:(CVPixelBufferRef)image error:(NSError * _Nullable * _Nullable)error;
@end

//模型拖入工程后，使用默认构造函数进行模型加载，就会去加载同名的VGG16.mlmodel文件
VGG16 *vgg16Model = [[VGG16 alloc] init];
//加载一个需要识别图片，一定是224*224大小的，否则不能识别
UIImage *image = [UIImage imageNamed:@"test.png"];
//将图片转换为CVPixelBufferRef格式的数据
CVPixelBufferRef imageBuffer = [self pixelBufferFromCGImage:[image CGImage]];
//使用转换后的图片数据构造模型输入对象
VGG16Input *input = [[VGG16Input alloc] initWithImage:imageBuffer];
   
NSError *error;
//使用VGG16模型进行图片的识别工作        
VGG16Output *output = [vgg16Model predictionFromFeatures:input error:&error];
//根据error判断识别是否成功
if (error) {
    NSLog(@"Prediction Error %@", error);
} else {
    //识别成功，输出可能性最大的分类标签
    NSLog(@"Label: %@", output.classLabel);
}

//由于在转换UIImage到CVPixelBufferRef时，手动开辟了一个空间，因此使用完成后需要及时释放
CVPixelBufferRelease(imageBuffer);

//CGImageRef转换CVPiexlBufferRef格式数据
- (CVPixelBufferRef) pixelBufferFromCGImage: (CGImageRef) image {
    NSDictionary *options = @{
                              (NSString*)kCVPixelBufferCGImageCompatibilityKey : @YES,
                              (NSString*)kCVPixelBufferCGBitmapContextCompatibilityKey : @YES,
                              };
    
    CVPixelBufferRef pxbuffer = NULL;
    /*
    注意需要使用kCVPixelFormatType_32BGRA的格式
    深度学习常用OpenCV对图片进行处理，OpenCV使用的不是RGBA而是BGRA
    这里使用CVPixelBufferCreate手动开辟了一个空间，因此使用完成后一定要释放该空间
    */
    CVReturn status = CVPixelBufferCreate(kCFAllocatorDefault, CGImageGetWidth(image),
                                          CGImageGetHeight(image), kCVPixelFormatType_32BGRA, (__bridge CFDictionaryRef) options,
                                          &pxbuffer);
    if (status != kCVReturnSuccess) {
        NSLog(@"Operation failed");
    }
    NSParameterAssert(status == kCVReturnSuccess && pxbuffer != NULL);
    
    CVPixelBufferLockBaseAddress(pxbuffer, 0);
    void *pxdata = CVPixelBufferGetBaseAddress(pxbuffer);
    
    CGColorSpaceRef rgbColorSpace = CGColorSpaceCreateDeviceRGB();
    CGContextRef context = CGBitmapContextCreate(pxdata, CGImageGetWidth(image),
                                                 CGImageGetHeight(image), 8, 4*CGImageGetWidth(image), rgbColorSpace,
                                                 kCGImageAlphaNoneSkipFirst);
    NSParameterAssert(context);
    
    CGContextConcatCTM(context, CGAffineTransformMakeRotation(0));
    CGAffineTransform flipVertical = CGAffineTransformMake( 1, 0, 0, -1, 0, CGImageGetHeight(image) );
    CGContextConcatCTM(context, flipVertical);
    CGAffineTransform flipHorizontal = CGAffineTransformMake( -1.0, 0.0, 0.0, 1.0, CGImageGetWidth(image), 0.0 );
    CGContextConcatCTM(context, flipHorizontal);
    
    CGContextDrawImage(context, CGRectMake(0, 0, CGImageGetWidth(image),
                                           CGImageGetHeight(image)), image);
    CGColorSpaceRelease(rgbColorSpace);
    CGContextRelease(context);
    
    CVPixelBufferUnlockBaseAddress(pxbuffer, 0);
    return pxbuffer;
}

//定义一个深度学习模型执行的block，方便在一个应用内，调用不同的模型
typedef void(^machineLearningExecuteBlock)(CVPixelBufferRef imageBuffer);
//实时目标检测视图类，需要实现一个协议用于获取摄像头的输出数据
@interface RealTimeDetectionViewController() <AVCaptureVideoDataOutputSampleBufferDelegate>

//当前获取到的设备
@property (nonatomic, strong) AVCaptureDevice *device;
/*
设备的输入，表示摄像头或麦克风这样的实际物理硬件
通过AVCaptureDevice对象创建，可以控制实际的物理硬件设备
*/
@property (nonatomic, strong) AVCaptureDeviceInput *input;
//视频输出，还有音频输出等类型
@property (nonatomic, strong) AVCaptureVideoDataOutput *output;
//设备连接，用于将session会话获取的数据与output输出数据，可以同时捕获视频和音频数据
@property (nonatomic, strong) AVCaptureConnection *videoConnection;
/*
捕捉设备会话，从实际的硬件设备中获取数据流，可以从摄像头或麦克风中获取
将数据流输出到一个或数个目的地，对于图像可以预设捕捉图片的大小质量等
*/
@property (nonatomic, strong) AVCaptureSession *session;
//设备预览layer，对于图像来说，摄像头拍摄到的图像数据直接展示在该layer上
@property (nonatomic, strong) AVCaptureVideoPreviewLayer *preview;

//感兴趣的区域，即将摄像头上该区域的图像捕获去进行识别
@property (nonatomic, assign) CGRect interestRegionRect;
//目标图像的大小，针对不同模型，有不同的输入图像大小
@property (nonatomic, assign) CGSize targetSize;
//一个框，类似于扫描二维码的，提示在这个框内的图像会被用于实时检测
@property (nonatomic, strong) UIImageView *interestRegionImageView;

//session传递数据是阻塞的，使用单独的串行队列处理
@property (nonatomic) dispatch_queue_t videoCaptureQueue;
//显示识别结果的label
@property (nonatomic, strong) UILabel *titleLabel;
//退出button
@property (nonatomic, strong) UIButton *exitButton;
//切换前后摄像头button
@property (nonatomic, strong) UIButton *reverseCameraButton;
//机器学习模型识别block
@property (nonatomic, copy) machineLearningExecuteBlock executeBlock;

@end

/**
 初始化设备、界面函数

 @param interestRegion 感兴趣区域，如果为CGRectZero不创建提示imageView
 @param targetSize 目标图像大小，如果为CGSizeZero不转换图像大小
 @param executeBlock 机器学习模型执行block
 */
- (void)setUpWithInterestRegion:(CGRect)interestRegion targetSize:(CGSize)targetSize machineLearningExecuteBlock:(machineLearningExecuteBlock)executeBlock {
    
    self.view.backgroundColor = [UIColor whiteColor];
    
    //创建一个串行队列，用于处理session获取到的数据，即执行回调函数
    self.videoCaptureQueue = dispatch_queue_create("VideoCaptureQueue", DISPATCH_QUEUE_SERIAL);
    
    //获取一个默认摄像头设备，默认使用后置摄像头
    //self.device = [AVCaptureDevice defaultDeviceWithDeviceType:AVCaptureDeviceTypeBuiltInWideAngleCamera mediaType:AVMediaTypeVideo position:AVCaptureDevicePositionFront];
    //指定获取前置或后置摄像头
    self.device = [self cameraWithPosition:AVCaptureDevicePositionFront];
    //设置摄像头自动对焦
    if (self.device.isFocusPointOfInterestSupported && [self.device isFocusModeSupported:AVCaptureFocusModeContinuousAutoFocus]) {
        [self.device lockForConfiguration:nil];
        [self.device setFocusMode:AVCaptureFocusModeContinuousAutoFocus];
        [self.device unlockForConfiguration];
    }
    //创建一个捕获输入
    self.input = [AVCaptureDeviceInput deviceInputWithDevice:self.device error:nil];
    //创建捕获输出
    self.output = [[AVCaptureVideoDataOutput alloc] init];
    //设置输出数据的代理为self，代理函数执行队列为创建的串行队列
    [self.output setSampleBufferDelegate:self queue:self.videoCaptureQueue];
    //创建一个字典，用于限制输出数据类型，输出数据的格式为kCVPixelFormatType_32BGRA
    NSDictionary *outputSettings = @{(id)kCVPixelBufferPixelFormatTypeKey: [NSNumber numberWithInt:kCVPixelFormatType_32BGRA]};
    //限制输出数据类型
    [self.output setVideoSettings:outputSettings];
    //创建一个session会话
    self.session = [[AVCaptureSession alloc] init];
    //设置输出数据为高质量，输出图像大小为1080*1920
    [self.session setSessionPreset:AVCaptureSessionPresetHigh];
    //将input、output添加进session会话进行管理
    if ([self.session canAddInput:self.input])
        [self.session addInput:self.input];
    if ([self.session canAddOutput:self.output])
        [self.session addOutput:self.output];
    //创建一个connection
    self.videoConnection = [self.output connectionWithMediaType:AVMediaTypeVideo];
    self.videoConnection.enabled = NO;
    //设置拍摄图像是竖屏，1080*1920，否则默认会旋转
    [self.videoConnection setVideoOrientation:AVCaptureVideoOrientationPortrait];
    //创建一个预览layer
    self.preview = [AVCaptureVideoPreviewLayer layerWithSession:self.session];
    self.preview.videoGravity = AVLayerVideoGravityResize;
    //设置这个预览layer为全屏
    self.preview.frame = self.view.frame;
    self.preview.contentsScale = [[UIScreen mainScreen] scale];
    //将预览layer插入到view.layer的第一个，作为子layer
    [self.view.layer insertSublayer:self.preview atIndex:0];
    //判断是否有感兴趣的区域，如果有就创建一个imageView防一个框在该区域中，用于提示用户
    if (!CGRectEqualToRect(interestRegion, CGRectZero)) {
        self.interestRegionImageView = [[UIImageView alloc] initWithFrame:interestRegion];
        self.interestRegionImageView.image = [UIImage imageNamed:@"InterestRegionBG"];
        [self.view addSubview:self.interestRegionImageView];
        self.interestRegionRect = interestRegion;
    }
    //创建显示结果的label
    self.titleLabel = [[UILabel alloc] initWithFrame:CGRectMake(0, self.view.frame.size.height - 50, self.view.frame.size.width, 50)];
    self.titleLabel.textAlignment = NSTextAlignmentCenter;
    self.titleLabel.textColor = [UIColor redColor];
    [self.view addSubview:self.titleLabel];
    //创建退出button
    self.exitButton = [UIButton buttonWithType:UIButtonTypeCustom];
    self.exitButton.titleLabel.textColor = [UIColor redColor];
    [self.exitButton setTitle:@"Exit" forState:UIControlStateNormal];
    self.exitButton.frame = CGRectMake(20, 20, 80, 40);
    [self.exitButton addTarget:self action:@selector(exitButtonClickedHandler) forControlEvents:UIControlEventTouchUpInside];
    [self.view addSubview:self.exitButton];
    //创建摄像头翻转butotn
    self.reverseCameraButton = [UIButton buttonWithType:UIButtonTypeCustom];
    self.reverseCameraButton.titleLabel.textColor = [UIColor redColor];
    [self.reverseCameraButton setTitle:@"ReverseCamera" forState:UIControlStateNormal];
    self.reverseCameraButton.frame = CGRectMake(self.view.frame.size.width - 20 - 200, 20, 200, 40);
    [self.reverseCameraButton addTarget:self action:@selector(reverseCameraButtonClickedHandler) forControlEvents:UIControlEventTouchUpInside];
    [self.view addSubview:self.reverseCameraButton];
    //记录targetsize和block
    self.targetSize = targetSize;
    self.executeBlock = executeBlock;
    //启动session，打开摄像头，开始收数据了
    [self.session startRunning];
    self.videoConnection.enabled = YES;
}

/**
 初始化函数

 @return return value description
 */
- (instancetype)initWithVGG16 {
    if (self = [super init]) {
        //加载本地的VGG16模型
        VGG16 *vgg16Model = [[VGG16 alloc] init];
        //获取一个weakSelf，在block中使用，防止引用循环
        __weak typeof(self) weakSelf = self;
        /*
         调用setUpWithInterestRegion:targetSize:machineLearningExecuteBlock函数进行初始化操作
         感兴趣区域放在屏幕正中央，大小为224*224，目标大小也为224*224
         */
        [self setUpWithInterestRegion:CGRectMake((self.view.frame.size.width - 224) / 2.0, (self.view.frame.size.height - 224) / 2.0, 224, 224) targetSize:CGSizeMake(224, 224) machineLearningExecuteBlock:^(CVPixelBufferRef imageBuffer) {
            //使用block的参数构造VGG16的输入对象
            VGG16Input *input = [[VGG16Input alloc] initWithImage:imageBuffer];
            NSError *error;
            //使用VGG16模型识别imageBuffer获取输出数据
            VGG16Output *output = [vgg16Model predictionFromFeatures:input error:&error];
            //获取一个strong的self
            __strong typeof(weakSelf) strongSelf = weakSelf;
            //如果识别失败，在主队列即主线程中修改titleLbale
            if (error) {
                dispatch_async(dispatch_get_main_queue(), ^{
                    strongSelf.titleLabel.text = @"Error!!!";
                });
            } else {
                //识别成功，在主队列即主线程中修改titleLabel
                dispatch_async(dispatch_get_main_queue(), ^{
                    strongSelf.titleLabel.text = output.classLabel;
                });
            }
        }];
    }
    return self;
}

/**
 AVCaptureVideoDataOutputSampleBufferDelegate代理方法，摄像头获取到数据后就会回调该方法
 默认是30FPS，每秒获取30张图像

 @param output output description
 @param sampleBuffer 摄像头获取到的图像数据格式
 @param connection connection description
 */
- (void)captureOutput:(AVCaptureOutput *)output didOutputSampleBuffer:(CMSampleBufferRef)sampleBuffer fromConnection:(AVCaptureConnection *)connection {

    //需要将CMSampleBufferRef转换为CVPixelBufferRef才能交给机器学习模型识别
    CVPixelBufferRef buffer;
    //首先判断是否需要将图像转换为目标大小
    if (!CGSizeEqualToSize(self.targetSize, CGSizeZero)) {
        //将CMSampleBufferRef转换为UIImage类型的对象
        UIImage *image = [self convertImage:sampleBuffer];
        //截图，获取感兴趣区域的图像，然后转换为目标大小
        image = [self getSubImage:self.interestRegionRect image:image targetSize:self.targetSize];
        //将CGImageRef图像转换为CVPixelBufferRef，使用CVPixelBufferCreate手动开辟的空间，使用完成后需要释放
        buffer = [self pixelBufferFromCGImage:[image CGImage]];
    } else {
        //不需要转化，则直接将CMSampleBufferRef转换为CVPixelBufferRef类型
        //这个函数虽然是叫get ImageBuffer，其实CVPixelBufferRef是CVImageBufferRef的别称
        //不是手动开辟的空间，随着弹栈自动释放内存
        buffer = CMSampleBufferGetImageBuffer(sampleBuffer);
    }
    //如果executeBlock不为空，传递CVPixelBufferRef数据，执行识别
    if (self.executeBlock) {
        self.executeBlock(buffer);
    }
    
    //自己创建的buffer自己清理，不是自己创建的交由系统清理
    if (!CGSizeEqualToSize(self.targetSize, CGSizeZero)) {
        CVPixelBufferRelease(buffer);
    }
}


/**
 截图函数

 @param rect 感兴趣区域
 @param image 要截的图
 @param targetSize 目标大小
 @return 截过的图片
 */
- (UIImage*)getSubImage:(CGRect)rect image:(UIImage*)image targetSize:(CGSize)targetSize {
    
    CGFloat screenWidth = [[UIScreen mainScreen] bounds].size.width;
    CGFloat screenHeight = [[UIScreen mainScreen] bounds].size.height;
    //因为输出图像是1080*1920，屏幕大小比这个小，所以需要根据比例将感兴趣区域换算到1080*1920大小的图像上
    CGRect imageRegion = CGRectMake((rect.origin.x / screenWidth) * image.size.width, (rect.origin.y / screenHeight) * image.size.height, (rect.size.width / screenWidth) * image.size.width, (rect.size.height / screenHeight) * image.size.height);
    //使用C函数直接截取感兴趣区域的图像
    CGImageRef imageRef = image.CGImage;
    CGImageRef imagePartRef = CGImageCreateWithImageInRect(imageRef, imageRegion);
    //获取到截取的UIImage对象
    UIImage *cropImage = [UIImage imageWithCGImage:imagePartRef];
    //释放创建的CGImageRef数据
    CGImageRelease(imagePartRef);
    //开启一个绘制图像的上下文，设置大小为目标大小
    UIGraphicsBeginImageContext(targetSize);
    //将前面裁剪的感兴趣的图像绘制在目标大小上
    [cropImage drawInRect:CGRectMake(0, 0, targetSize.width, targetSize.height)];
    //获取绘制完成的UIImage对象
    UIImage *resImage = UIGraphicsGetImageFromCurrentImageContext();
    //释放内存
    UIGraphicsEndImageContext();
    
    return resImage;
}
 
/**
 官方给的转换代码

 @param sampleBuffer sampleBuffer description
 @return return value description
 */
- (UIImage*)convertImage:(CMSampleBufferRef)sampleBuffer {

    // Get a CMSampleBuffer's Core Video image buffer for the media data
    CVImageBufferRef imageBuffer = CMSampleBufferGetImageBuffer(sampleBuffer);
    // Lock the base address of the pixel buffer
    CVPixelBufferLockBaseAddress(imageBuffer, 0);
    
    // Get the number of bytes per row for the pixel buffer
    void *baseAddress = CVPixelBufferGetBaseAddress(imageBuffer);
    
    // Get the number of bytes per row for the pixel buffer
    size_t bytesPerRow = CVPixelBufferGetBytesPerRow(imageBuffer);
    // Get the pixel buffer width and height
    size_t width = CVPixelBufferGetWidth(imageBuffer);
    size_t height = CVPixelBufferGetHeight(imageBuffer);
    
    // Create a device-dependent RGB color space
    CGColorSpaceRef colorSpace = CGColorSpaceCreateDeviceRGB();
    
    // Create a bitmap graphics context with the sample buffer data
    CGContextRef context = CGBitmapContextCreate(baseAddress, width, height, 8,
                                                 bytesPerRow, colorSpace, kCGBitmapByteOrder32Little | kCGImageAlphaPremultipliedFirst);
    // Create a Quartz image from the pixel data in the bitmap graphics context
    CGImageRef quartzImage = CGBitmapContextCreateImage(context);
    // Unlock the pixel buffer
    CVPixelBufferUnlockBaseAddress(imageBuffer,0);
    
    // Free up the context and color space
    CGContextRelease(context);
    CGColorSpaceRelease(colorSpace);
    
    // Create an image object from the Quartz image
    //UIImage *image = [UIImage imageWithCGImage:quartzImage];
    UIImage *image = [UIImage imageWithCGImage:quartzImage scale:1.0f orientation:UIImageOrientationUp];
    
    // Release the Quartz image
    CGImageRelease(quartzImage);
    
    return image;
}

#pragma mark touchsbegin
- (void)touchesBegan:(NSSet<UITouch *> *)touches withEvent:(UIEvent *)event {
    for (UITouch *touch in touches) {
        CGPoint point = [touch locationInView:self.view];
        //触摸屏幕，实现对焦
        [self focusOnPoint:point];
    }
}

#pragma mark avfoundation function

/**
 根据deviceDiscoverySession类型，获取前置或后置摄像头

 @param position position description
 @return return value description
 */
- (AVCaptureDevice*)cameraWithPosition:(AVCaptureDevicePosition)position {
    //iOS10以后使用该类方法获取想要的设备
    AVCaptureDeviceDiscoverySession *deviceDiscoverySession = [AVCaptureDeviceDiscoverySession discoverySessionWithDeviceTypes:@[AVCaptureDeviceTypeBuiltInWideAngleCamera] mediaType:AVMediaTypeVideo position:position];
    //遍历获取到的所有设备，返回需要的类型
    for (AVCaptureDevice* device in deviceDiscoverySession.devices) {
        if(device.position == position)
            return device;
    }
    return nil;
}


/**
 手动对焦

 @param point 需要对焦的区域点
 */
- (void)focusOnPoint:(CGPoint)point {
    if ([self.device isFocusPointOfInterestSupported] && [self.device lockForConfiguration:nil]) {
        self.device.focusPointOfInterest = point;
        [self.device unlockForConfiguration];
    }
}

#pragma mark - UIButton target-action handler
- (void)exitButtonClickedHandler {
    [self dismissViewControllerAnimated:YES completion:nil];
}

/**
 前后摄像头翻转
 */
- (void)reverseCameraButtonClickedHandler {
    //创建一个新的设备
    AVCaptureDevice *newCamera;
    //根据当前设备类型，获取翻转后的摄像头设备
    if (self.device.position == AVCaptureDevicePositionBack) {
        newCamera = [self cameraWithPosition:AVCaptureDevicePositionFront];
    } else {
        newCamera = [self cameraWithPosition:AVCaptureDevicePositionBack];
    }
    //根据设备创建一个新的input
    AVCaptureDeviceInput *newInput = [AVCaptureDeviceInput deviceInputWithDevice:newCamera error:nil];
    self.device = newCamera;
    //会话开启一个配置阶段
    [self.session beginConfiguration];
    //删除旧设备input
    [self.session removeInput:self.input];
    //添加新设备input
    [self.session addInput:newInput];
    self.input = newInput;
    //关闭connection，重新创建一个，否则切换摄像头时输出的图片又默认旋转了
    self.videoConnection.enabled = NO;
    self.videoConnection = [self.output connectionWithMediaType:AVMediaTypeVideo];
    self.videoConnection.enabled = NO;
    [self.videoConnection setVideoOrientation:AVCaptureVideoOrientationPortrait];
    self.videoConnection.enabled = YES;
    //提交配置，自动更新
    [self.session commitConfiguration];
}

>>> import coremltools
>>> model = coremltools.converters.caffe.convert(('age_net.caffemodel', 'deploy_age.prototxt'), image_input_names='data', class_labels='age_labels.txt')
>>> model.save('AgePredictionNet.mlmodel')

0-2
4-6
8-13
15-20
25-32
38-43
48-53
60

def convert(model, image_input_names=[], is_bgr=False,
            red_bias=0.0, blue_bias=0.0, green_bias=0.0, gray_bias=0.0,
            image_scale=1.0, class_labels=None, predicted_feature_name=None):

    tf_model_dir = '/Users/chenyoake/Desktop/MLModel/inception_resnet'
    tf_model_path = os.path.join(tf_model_dir, 'inception_resnet_v2_2016_08_30_frozen.pb')

    mlmodel_path = os.path.join(tf_model_dir, 'InceptionResnetV2.mlmodel')
    mlmodel = tf_converter.convert(
        tf_model_path = tf_model_path,
        mlmodel_path = mlmodel_path,
        output_feature_names = ['InceptionResnetV2/Logits/Predictions:0'],
        input_name_shape_dict = {'input:0':[1,299,299,3]},
        image_input_names = ['input:0'],
        red_bias = -1, 
        green_bias = -1, 
        blue_bias = -1, 
        image_scale = 2.0/255.0,
        class_labels = '/Users/chenyoake/Desktop/MLModel/inception_resnet/imagenet_slim_labels.txt')

- (void)classification {
    //加载一个mlmodel
    GoogLeNetPlaces *model = [[GoogLeNetPlaces alloc] init];
    NSError *error;
    //通过mlmodel模型创建一个Vision能够使用的VNCoreMLModel
    VNCoreMLModel *visionModel = [VNCoreMLModel modelForMLModel:model.model error:&error];
    //根据转换后的模型创建一个机器学习模型识别的request
    VNCoreMLRequest *request = [[VNCoreMLRequest alloc] initWithModel: visionModel completionHandler:^(VNRequest * _Nonnull request, NSError * _Nullable error) {
        //机器学习模型运行完成后的回调块，在request中可以查看到相关信息
        CGFloat confidence = 0.f;
        VNClassificationObservation *tempClassification = nil;
        //GoogLeNetPlaces输出多个结果，遍历查找置信度最高的
        for (VNClassificationObservation *classification in request.results) {
            if (classification.confidence > confidence) {
                confidence = classification.confidence;
                tempClassification = classification;
            }
        }
        //输出结果
        NSLog(@"Classification Value %@ %@ %lf", tempClassification, tempClassification.identifier, tempClassification.confidence);

    }];

    //创建一个请求的handler
    VNImageRequestHandler *handler = [[VNImageRequestHandler alloc] initWithCGImage:[[UIImage imageNamed:@"a.jpeg"] CGImage] options:nil];
    //执行请求
    [handler performRequests:@[request] error:&error];

    if (error) {
        NSLog(@"ERROR CLASSIFICATION");
    }
    
}

- (instancetype)initWithFaceDetection {
    if (self = [super init]) {
        //Vision人脸检测的输入图片大小没有限制，所以可以全屏检测
        [self setUpWithInterestRegion:CGRectZero targetSize:CGSizeZero machineLearningExecuteBlock:^(CVPixelBufferRef imageBuffer) {
                   
            /*
            首先创建一个人脸检测的请求Requst，通过名称可以知道
            这是一个人脸矩形框检测的请求，返回值是一个bounding box bbox
            */
            VNDetectFaceRectanglesRequest *request = [[VNDetectFaceRectanglesRequest alloc] initWithCompletionHandler:^(VNRequest * _Nonnull request, NSError * _Nullable error) {
                //遍历计算结果
                for (VNFaceObservation *faceObservation in request.results) {
                    //拿到bbox
                    CGRect boundingBox = faceObservation.boundingBox;
                    //获取图像大小
                    CGSize imageSize = CGSizeMake(CVPixelBufferGetWidth(imageBuffer), CVPixelBufferGetHeight(imageBuffer));
                    //bbox给的是在图片中的比例，所以需要自己转换
                    CGFloat width = boundingBox.size.width * imageSize.width;
                    CGFloat height = boundingBox.size.height * imageSize.height;
                    CGFloat x = boundingBox.origin.x * imageSize.width;
                    CGFloat y = imageSize.height - (boundingBox.origin.y * imageSize.height) - height;
                    
                    CGFloat screenWidth = [[UIScreen mainScreen] bounds].size.width;
                    CGFloat screenHeight = [[UIScreen mainScreen] bounds].size.height;
                    //在屏幕上计算真实的位置
                    CGRect faceRegion = CGRectMake((x / imageSize.width) * screenWidth, (y / imageSize.height) * screenHeight, (width / imageSize.width) * screenWidth, (height / imageSize.height) * screenHeight);
                    
                    NSLog(@"FACE REGION %lf %lf %lf %lf %@", faceRegion.origin.x, faceRegion.origin.y, faceRegion.size.width, faceRegion.size.height, faceObservation.uuid);
                    
            }];
            //使用CVPixelBufferRef创建一个handler
            VNImageRequestHandler *handler = [[VNImageRequestHandler alloc] initWithCVPixelBuffer:imageBuffer options:@{}];
            NSError *error;
            //执行请求
            [handler performRequests:@[request] error:&error];
            
            if (error) {
                NSLog(@"ERRROROROROR %@", error);
            }
            
        }];
    }
    return self;
}

/**
 目标追踪

 @return return value description
 */
- (instancetype)initWithObjectTracking {
    if (self = [super init]) {
        /*
         @property (nonatomic, strong) NSMutableDictionary<NSString*, VNDetectedObjectObservation*> *trackingObservationDict;
         @property (nonatomic, strong) NSMutableDictionary<NSString*, CALayer*> *trackingBBoxDict;
         */
        //每一个检测的结果observation都有一个uuid用于区分，构造一个字典，用于记录要追踪的目标observation
        self.trackingObservationDict = [[NSMutableDictionary alloc] init];
        //要追踪目标的bbox的layer，手动画上去的
        self.trackingBBoxDict = [[NSMutableDictionary alloc] init];
        //weak一下防止引用循环
        __weak typeof(self) weakSelf = self;
        //不需要限制输入大小
        [self setUpWithInterestRegion:CGRectZero targetSize:CGSizeZero machineLearningExecuteBlock:^(CVPixelBufferRef imageBuffer) {
            //strong一下
            __strong typeof(weakSelf) strongSelf = weakSelf;
            //首先需要查看要追踪的observation是否为空，如果为空，就需要先去找一个要追踪的目标
            if (strongSelf.trackingObservationDict.count == 0) {
                //查找初始要追踪的目标
                [strongSelf detectTrackingFace:imageBuffer];
                return ;
            }
            //每一个目标的observation进行追踪都需要一个单独的request，创建一个集合来保存
            NSMutableArray<VNTrackObjectRequest*> *trackingRequestArray = [[NSMutableArray alloc] init];
            //遍历要追踪目标observation的字典
            for (NSString *uuid in strongSelf.trackingObservationDict.allKeys) {
                //根据uuid获取到这个observation
                VNDetectedObjectObservation *faceObservation = strongSelf.trackingObservationDict[uuid];
                //创建一个追踪目标的请求，需要传入要追踪目标的observation，内部应该有一个反馈的操作
                VNTrackObjectRequest *trackingRequest = [[VNTrackObjectRequest alloc] initWithDetectedObjectObservation:faceObservation completionHandler:^(VNRequest * _Nonnull request, NSError * _Nullable error) {
                    
                    //判断是否有错，追踪目标的结果数量是不是大于0
                    if (error == nil && request.results.count > 0) {
                        //获取追踪后的第一个observation
                        VNFaceObservation *observation = request.results.firstObject;
                        //更新追踪observation的字典
                        [strongSelf.trackingObservationDict setObject:observation forKey:uuid];
                        //如果置信度小于0.5就抛弃掉
                        if (observation.confidence < 0.5) {
                            [strongSelf.trackingObservationDict removeObjectForKey:uuid];
                            dispatch_async(dispatch_get_main_queue(), ^{
                                CALayer *layer = strongSelf.trackingBBoxDict[uuid];
                                [layer removeFromSuperlayer];
                                [strongSelf.trackingBBoxDict removeAllObjects];
                            });
                            return ;
                        }
                        //置信度满足要求，获取bbox
                        CGRect boundingBox = observation.boundingBox;
                        //一系列换算
                        CGSize imageSize = CGSizeMake(CVPixelBufferGetWidth(imageBuffer), CVPixelBufferGetHeight(imageBuffer));
                        CGFloat width = boundingBox.size.width * imageSize.width;
                        CGFloat height = boundingBox.size.height * imageSize.height;
                        CGFloat x = boundingBox.origin.x * imageSize.width;
                        CGFloat y = imageSize.height - (boundingBox.origin.y * imageSize.height) - height;
                        
                        CGFloat screenWidth = [[UIScreen mainScreen] bounds].size.width;
                        CGFloat screenHeight = [[UIScreen mainScreen] bounds].size.height;
                        //获取到在屏幕上的region rect
                        CGRect faceRegion = CGRectMake((x / imageSize.width) * screenWidth, (y / imageSize.height) * screenHeight, (width / imageSize.width) * screenWidth, (height / imageSize.height) * screenHeight);
                        //画一个layer 打框
                        CALayer *layer = strongSelf.trackingBBoxDict[uuid];
                        if (!layer) {
                            layer = [[CALayer alloc] init];
                            layer.frame = faceRegion;
                            layer.borderWidth = 2;
                            layer.cornerRadius = 3;
                            layer.borderColor = [UIColor redColor].CGColor;
                            [strongSelf.trackingBBoxDict setObject:layer forKey:uuid];
                            dispatch_async(dispatch_get_main_queue(), ^{
                                [strongSelf.view.layer addSublayer:layer];
                            });
                        } else {
                            dispatch_async(dispatch_get_main_queue(), ^{
                                layer.frame = faceRegion;
                            });
                        }
                        
                    } else {
                        //有问题，就清除所有的bbox layer
                        [strongSelf.trackingObservationDict removeObjectForKey:uuid];
                        dispatch_async(dispatch_get_main_queue(), ^{
                            CALayer *layer = strongSelf.trackingBBoxDict[uuid];
                            [layer removeFromSuperlayer];
                            [strongSelf.trackingBBoxDict removeAllObjects];
                        });
                    }
                    
                }];
                //设置tracking以准确率为主
                trackingRequest.trackingLevel = VNRequestTrackingLevelAccurate;
                //添加请求到数组中
                [trackingRequestArray addObject:trackingRequest];
            }
            
            NSError *error = nil;
            //需要保留这个sequenceRequestHandler，每次追踪都需要使用这个，否则结果不正确
            if (!strongSelf.sequenceRequestHandler) {
                strongSelf.sequenceRequestHandler = [[VNSequenceRequestHandler alloc] init];
            }
            //执行一系列的请求
            [strongSelf.sequenceRequestHandler performRequests:trackingRequestArray onCVPixelBuffer:imageBuffer error:&error];
            if (error) {
                //出错就删除框
                [strongSelf.trackingObservationDict removeAllObjects];
                dispatch_async(dispatch_get_main_queue(), ^{
                    for (NSString *uuid in strongSelf.trackingBBoxDict.allKeys) {
                        CALayer *layer = strongSelf.trackingBBoxDict[uuid];
                        [layer removeFromSuperlayer];
                    }
                    [strongSelf.trackingBBoxDict removeAllObjects];
                });
            }
            
        }];
        
    }
    return self;
}


/**
 初始的目标检测，检测需要跟踪的目标

 @param imageBuffer imageBuffer description
 */
- (void)detectTrackingFace:(CVPixelBufferRef)imageBuffer {
    //创建一个人脸检测的请求
    VNDetectFaceRectanglesRequest *request = [[VNDetectFaceRectanglesRequest alloc] initWithCompletionHandler:^(VNRequest * _Nonnull request, NSError * _Nullable error) {
        for (VNFaceObservation *faceObservation in request.results) {
            //由于tracking的时候只能使用VNDetectedObjectObservation，而VNDetectFaceRectanglesRequest是其子类
            //所以作者使用检测到的人脸的区域来创建一个observation想让Vision追踪人脸，但失败了。。
            VNDetectedObjectObservation *observation = [VNDetectedObjectObservation observationWithBoundingBox:faceObservation.boundingBox];
            [self.trackingObservationDict setObject:observation forKey:observation.uuid.UUIDString];
        }
    }];
    //读者可以试验，检测目标的持续追踪
//    VNDetectRectanglesRequest *request = [[VNDetectRectanglesRequest alloc] initWithCompletionHandler:^(VNRequest * _Nonnull request, NSError * _Nullable error) {
//        for (VNDetectedObjectObservation *observation in request.results) {
//            [self.trackingObservationDict setObject:observation forKey:observation.uuid.UUIDString];
//        }
//    }];
    //创建一个检测的handler
    VNImageRequestHandler *handler = [[VNImageRequestHandler alloc] initWithCVPixelBuffer:imageBuffer options:@{}];
    NSError *error;
    //执行请求
    [handler performRequests:@[request] error:&error];
    
    if (error) {
        //出错就清空所有数据
        [self.trackingObservationDict removeAllObjects];
        dispatch_async(dispatch_get_main_queue(), ^{
            for (NSString *uuid in self.trackingBBoxDict.allKeys) {
                CALayer *layer = self.trackingBBoxDict[uuid];
                [layer removeFromSuperlayer];
            }
            [self.trackingBBoxDict removeAllObjects];
        });
        
    }
}