【转载】PyTorch系列 (二)：pytorch数据读取

1
class torch.utils.data.Dataset

class Dataset(object):
    # 强制所有的子类override getitem和len两个函数，否则就抛出错误；
    # 输入数据索引，输出为索引指向的数据以及标签；
    def __getitem__(self, index):
        raise NotImplementedError
    
    # 输出数据的长度
    def __len__(self):
        raise NotImplementedError
        
    def __add__(self, other):
        return ConcatDataset([self, other])

class torch.utils.data.TensorDataset(*tensors)

class TensorDataset(Dataset):
    def __init__(self, *tensor):
        assert all(tensors[0].size(0) == tensor.size(0) for tensor in tensors)
        self.tensors = tensors
        
    def __getitem__(self, index):
        return tuple(tensor[index] for tensor in tensors
        
    def __len__(self):
        return self.tensors[0].size(0)

class torch.utils.data.ConcatDateset(datasets)

class ConcatDataset(Dataset):
    @staticmethod
    def cumsum(sequence):
        # sequence是一个列表，e.g. [[1,2,3], [a,b], [4,h]]
        # return 一个数据大小列表，[3, 5, 7], 明显看的出来包含数据多少，第一个代表第一个数据的大小，第二个代表第一个+第二数据的大小，最后代表所有的数据大学；
    ...
    def __getitem__(self, idx):
        # 主要是这个函数，通过bisect的类实现了任意索引数据的输出；
        dataset_idx = bisect.bisect_right(self.cumulative_size, idx)
        if dataset_idx == 0:
            sample_idx == idx
        else:
            sample_idx = idx - self.cumulative_sizes[dataset_idx -1]
        return self.datasets[dataset_idx][sample_idx]
    ...

class torch.utils.data.Subset(dataset, indices)

class torch.utils.data.random_split(dataset, lengths):

class torch.utils.data.DataLoader(dataset, batch_size=1, shuffle=False, sampler=None, batch_sampler=None, num_workers=0, collate_fn=<function default_collate>, pin_memory=False, drop_last=False, timeout=0, worker_init_fn=None)

class _DataLoaderIter(loader)

class torch.utils.data.sampler.Sampler(data_source)

class Sampler(object):
    def __init__(self, data_source):
        pass
        
    def __iter__(self):
        raise NotImplementedError
        
    def __len__(self):
        raise NotImplementedError

class torch.utils.data.SequentialSampler(data_source)

class torch.utils.data.RandomSampler(data_source, replacement=False, num_samples=None)

class torch.utils.data.SubsetRandomSampler(indices)

class torch.utils.data.BatchSampler(sampler, batch_size, drop_last)

class torch.utils.data.WeightedRandomSampler(weights, num_samples, replacement=True)

class torch.utils.data.distributed.DistributedSampler(dataset, num_replicas=None, rank=None)

import os

import numpy as np
from torch.utils.data import Dataset
from PIL import Image

np.random.seed(0)

class MyDataset(Dataset):
    
    def __init__(self, root, size=229, ):
        """
        Initialize the data producer
        """
        self._root = root
        self._size = size
        self._num_image = len(os.listdir(root))
        self._img_name = os.listdir(root)
    
    def __len__(self):
        return self._num_image
        
    def __getitem__(self, index):
        img = Image.open(os.path.join(self._root, self._img_name[index]))
        
        # PIF image: H × W × C
        # torch image: C × H × W
        img = np.array(img, dtype-np.float32).transpose((2, 0, 1))
        
        return img

from torch.utils.data import DataLoader

# 实例化MyData
dataset_tumor_train = MyDataset(root=/img/train/tumor/)
dataset_normal_train = MyDataset(root=/img/train/normal/)
dataset_tumor_validation = MyDataset(root=/img/validation/tumor/)
dataset_normal_validation = MyDataset(root=/img/validation/normal/)

# 封装到loader
dataloader_tumor_train = DataLoader(dataset_tumor_train, batch_size=10)
dataloader_normal_train = DataLoader(dataset_normal_train, batch_size=10)
dataloader_tumor_validation = DataLoader(dataset_tumor_validation, batch_size=10)
dataloader_normal_validation = DataLoader(dataset_normal_validation, batch_size=10)

def train_epoch(model, loss_fn, optimizer, dataloader_tumor, dataloader_normal):
    model.train()
    
    # 由于tumor图像和normal图像一样多，所以将tumor，normal连接起来，steps=len(tumor_loader)=len(normal_loader)
    steps = len(dataloader_tumor)
    batch_size = dataloader_tumor.batch_size
    dataiter_tumor = iter(dataloader_tumor)
    dataiter_normal = iter(dataloader_normal)
    
    for step in range(steps):
        data_tumor = next(dataiter_tumor)
        target_tumor = [1, 1,..,1] # 和data_tumor长度相同的tensor
        data_tumor = Variable(data_tumor.cuda(async=True))
        target_tumor = Variable(target_tumor.cuda(async=True))
         
        data_normal = next(dataiter_normal)
        target_normal = [0, 0,..,0] # 
        data_normal = Variable(data_normal.cuda(async=True))
        target_normal = Variable(target_normal.cuda(async=True))
        
        idx_rand = Variable(torch.randperm(batch_size*2).cuda(async=True))
        
        data = torch.cat([data_tumor, data_normal])[idx_rand]
        target = torch.cat([target_tumor, target_normal])[idx_rand]
        output = model(data)
        loss = loss_fn(output, target)
        
        optimizer.zero_grad()
        loss.backward()
        optimizer.step()
        
        probs = output.sigmoid()