深度学习中超大规模数据集的处理

class HDF5DatasetWriter:
 def __init__(self, dims, output_path, data_key="images", buf_size=1000):
   # check to see if the output path exists, and if so, raise an exception
   if os.path.exists(output_path):
     raise ValueError("the supplied `output_path` already exists and cannot be overwritten.", output_path)   # open the HDF5 database for writing and create two datasets: one to store images/features
   # and another to store the class labels
   self.db = h5py.File(output_path, "w")
   self.data = self.db.create_dataset(data_key, dims, dtype="float")
   self.labels = self.db.create_dataset("labels", (dims[0],), dtype="int")   self.buf_size = buf_size
   self.buffer = {"data": [], "labels": []}
   self.idx = 0 def add(self, rows, labels):
   self.buffer["data"].extend(rows)
   self.buffer["labels"].extend(labels)   if len(self.buffer["data"]) >= self.buf_size:
     self.flush() def flush(self):
   i = self.idx + len(self.buffer["data"])
   self.data[self.idx:i] = self.buffer["data"]
   self.labels[self.idx:i] = self.buffer["labels"]
   self.idx = i
   self.buffer = {"data": [], "labels": []} def store_class_labels(self, class_labels):
   dt = h5py.special_dtype(vlen=str)
   labelset = self.db.create_dataset("label_names", (len(class_labels),), dtype=dt)
   labelset[:] = class_labels def close(self):
   if len(self.buffer["data"]) > 0:
     self.flush()   self.db.close()

kaggle_dogs_vs_cats/train/cat.11866.jpg
...
kaggle_dogs_vs_cats/train/dog.11046.jpg

train_paths = list(paths.list_images(config.IMAGES_PATH))
train_labels = [p.split(os.path.sep)[-1].split(".")[0] for p in train_paths]

split = train_test_split(train_paths, train_labels, test_size=config.NUM_TEST_IMAGES, stratify=train_labels,
                        random_state=42)
(train_paths, test_paths, train_labels, test_labels) = splitsplit = train_test_split(train_paths, train_labels, test_size=config.NUM_VAL_IMAGES, stratify=train_labels,
                        random_state=42)
(train_paths, val_paths, train_labels, val_labels) = split

datasets = [
 ("train", train_paths, train_labels, config.TRAIN_HDF5),
 ("val", val_paths, val_labels, config.VAL_HDF5),
 ("test", test_paths, test_labels, config.TEST_HDF5)
]aap = AspectAwarePreprocessor(256, 256)
(R, G, B) = ([], [], [])for (dtype, paths, labels, output_path) in datasets:
 writer = HDF5DatasetWriter((len(paths), 256, 256, 3), output_path) # loop over the image paths
 for (i, (path, label)) in enumerate(zip(paths, labels)):
   image = cv2.imread(path)
   image = aap.preprocess(image)   if dtype == "train":
     (b, g, r) = cv2.mean(image)[:3]
     R.append(r)
     G.append(g)
     B.append(b)   writer.add([image], [label]) writer.close()

D = {"R": np.mean(R), "G": np.mean(G), "B": np.mean(B)}
f = open(config.DATASET_MEAN, "w")
f.write(json.dumps(D))
f.close()

-rw-rw-r-- 1 alex alex  3932182048 Feb 18 11:33 test.hdf5
-rw-rw-r-- 1 alex alex 31457442048 Feb 18 11:31 train.hdf5
-rw-rw-r-- 1 alex alex  3932182048 Feb 18 11:32 val.hdf5