论文实践学习 - Deep Metric Learning via Lifted Structured Feature Embedding

image_id class_id super_class_id path
1 1 1 bicycle_final/111085122871_0.JPG
2 1 1 bicycle_final/111085122871_1.JPG
3 1 1 bicycle_final/111085122871_2.JPG
......

function conf = config()

conf = struct();

%% Directories

conf.root_path = '/path/to/Deep-Metric-Learning-CVPR16/code/ebay/';
conf.cache_path = '/path/to/Deep-Metric-Learning-CVPR16/code/ebay/cache';
conf.image_path = '/path/to/Ebay_Dataset/';

%% Training parameters
conf.preprocessing.crop_padding = 15;
conf.preprocessing.square_size = 256;
conf.preprocessing.num_to_load = 255;
conf.preprocessed_image_file = [conf.cache_path, '/training_images.mat'];

path_triplet = '/path/to/Deep-Metric-Learning-CVPR16/code/ebay/cache';

% for multilabel pairs batchsize = 64
conf.training_set_path_multilabel_m64 = [path_triplet, '/training_set_ebay_multilabel_m64.lmdb'];

% for multilabel pairs batchsize = 128
conf.training_set_path_multilabel_m128 = [path_triplet, '/training_set_ebay_multilabel_m128.lmdb'];

% for multilabel pairs batchsize = 128*2 = 256
conf.training_set_path_multilabel_m256 = [path_triplet, '/training_set_ebay_multilabel_m256.lmdb'];

% for multilabel pairs batchsize = 128*3 = 384
conf.training_set_path_multilabel_m384 = [path_triplet, '/training_set_ebay_multilabel_m384.lmdb'];

% for debuggin,
conf.training_imageset_path = [path_triplet, '/training_imageset_ebay.lmdb'];

conf.training_set_path_triplet = [path_triplet, '/training_set_triplet.lmdb'];
conf.validation_set_path_triplet = [path_triplet, '/validation_set_triplet.lmdb'];
conf.validation_set_path = [path_triplet, '/validation_set_cars196.lmdb'];

function gen_splits

conf = config;

%% generate splits
[image_ids, class_ids, superclass_ids, path_list] = ...
    textread('/path/to/Ebay_train.txt', '%d %d %d %s',...
    'headerlines', 1); % train.txt 路径
......

[image_ids, class_ids, superclass_ids, path_list] = ...
    textread('/path/to/Ebay_test.txt', '%d %d %d %s',...
    'headerlines', 1); % test.txt 路径
......

savepath = [conf.root_path, 'splits.mat'];
save(savepath, 'train_images', 'val_images', 'dict');

mex imageset_to_leveldb.cpp -lcaffe -lglog -lgflags -lprotobuf -lleveldb -llmdb ...
 -L/path/to/caffe/build/lib ...
 -I/path/to/caffe/build/src/caffe/proto

mex serialized_pairs_to_leveldb.cpp -lcaffe -lglog -lgflags -lprotobuf -lleveldb -llmdb ...
 -L/path/to/caffe/build/lib ...
 -I/path/to/caffe/build/src/caffe/proto

layer {
  name: "data"
  type: "Data"
  top: "data"
  top: "label"
  include {
    phase: TEST
  }
  transform_param {
    crop_size: 227
    mean_file: "/path/to/ebay/cache/imagenet_mean.binaryproto"
  }
  data_param {
    source: "/path/to/ebay/cache/validation_set_ebay.lmdb"
    batch_size: 26
    backend: LMDB
  }
}

#!/usr/bin/python
# coding: utf-8

import numpy as np
import matplotlib.pyplot as plt
import caffe
import scipy.io as io
import sys

embedding_dimension = 64 # 提取特征维度
print 'Embedding dim: %d' % embedding_dimension

MODEL_FILE = 'model/extract_googlenet_ebay_feature_embed%d.prototxt' % (embedding_dimension)

PRETRAINED = 'snapshot_ebay_googlenet_finetune_liftedstructsim_softmax_pair_m128_multilabel_embed%d_baselr_1E4_iter_20000.caffemodel' % embedding_dimension

MEAN_FILE = 'cache/imagenet_mean.binaryproto'

caffe.set_device(0)
caffe.set_mode_gpu()

net = caffe.Net(MODEL_FILE, PRETRAINED, caffe.TEST)

import lmdb
LMDB_FILENAME = '/path/to/ebay/cache/validation_set_ebay.lmdb'
lmdb_env = lmdb.open(LMDB_FILENAME)
lmdb_txn = lmdb_env.begin()
lmdb_cursor = lmdb_txn.cursor()

num_imgs = 0
for key in lmdb_cursor:
    num_imgs += 1
print 'Num images: %d' %num_imgs

batchsize = 26
num_batches = num_imgs / batchsize 
print 'Num batches: %d' %num_batches

# Store fc features for all images
fc_feat_dim = embedding_dimension

feat_matrix = np.zeros((num_imgs, fc_feat_dim), dtype=np.float32)
filename_list = []

filename_idx = 0
for batch_id in range(num_batches):
    batch = net.forward()
    fc = net.blobs['fc_embedding'].data.copy()
    for i in range(batchsize):
        feat_matrix[filename_idx+i, :] = fc[i,:]

    filename_idx += batchsize


a = {}
a['fc_embedding'] = feat_matrix
io.savemat('features/validation_googlenet_feat_matrix_liftedstructsim_softmax_pair_m128_multilabel_embed%d_baselr_1E4_gaussian2k.mat' % embedding_dimension, a)

print "all zeros?: ", np.allclose(feat_matrix, 0)

function evaluate_recall(embedding_dimension)

fprintf('embedding_dimension %d\n', embedding_dimension); % 这里 embedding_dimension=64

name = 'liftedstructsim_softmax_pair_m128_multilabel';
feature_filename = ...
    sprintf('/path/to/features/validation_googlenet_feat_matrix_%s_embed%d_baselr_1E4_gaussian2k.mat', ...
    name, embedding_dimension);

try
    load(feature_filename, 'fc_embedding');
catch
    error('filename is non existent.\n');
end

features = fc_embedding;

dims = size(features);
assert(dims(2) == embedding_dimension);
assert(dims(1) == 60502);

%D = squareform(pdist(features, 'euclidean'));
D2 = distance_matrix(features); % 此处比较耗费内存和时间，本机32G内存加 swap 32G 仅能针对 64 维特征进行距离矩阵计算.

[image_ids, class_ids, superclass_ids, path_list] = ...
    textread('/path/to/Ebay_test.txt', '%d %d %d %s',...
    'headerlines', 1); 

% set diagonal to very high number
assert(dims(1) == length(class_ids));
num = dims(1);
D = sqrt(abs(D2));
D(1:num+1:num*num) = inf;

for K = [1, 10, 100, 1000]
    compute_recall_at_K(D, K, class_ids, num);
end

disp('done');

% compute pairwise distance matrix
function D = distance_matrix(X)

m = size(X, 1);
t = ones(m, 1);
x = zeros(m, 1);
for i = 1:m
    n = norm(X(i,:));
    x(i) = n * n;
end

D = x * t' + t * x' - 2 * X * X'; %%%%%%

% compute recall@K
function recall = compute_recall_at_K(D, K, class_ids, num)

num_correct = 0;
for i = 1 : num
    this_gt_class_idx = class_ids(i);
    this_row = D(i,:);
    [~, inds] = sort(this_row, 'ascend');
    knn_inds = inds(1:K);

    knn_class_inds = class_ids(knn_inds);

    if sum(ismember(knn_class_inds, this_gt_class_idx)) > 0
        num_correct = num_correct + 1;
    end
end
recall = num_correct / num;
fprintf('K: %d, Recall: %.3f\n', K, recall);