基于PyTorch深度学习框架的序列图像数据装载器

import os
from PIL import Image
import torch
from torch.utils.data import DataLoader, Dataset
import torchvision.transforms as transforms
import torch.nn as nn

class data(Dataset):
   def __init__(self, param1, param2):
        # 函数在此处初始化

   def __len__(self):
        # 函数返回数据的长度

   def __getitem__(self, index):
        # 一次提供一个项目

data/
   - dog_1.jpg
   - dog_2.jpg
    ...
    ...
    ...
   - cat_1.jpg
   - cat_2.jpg
    ...
    ...
    ...

class data(Dataset):   
   def __init__(self, path, transform):
        self.files = os.listdir(path)
        self.transform = transform
        self.path = path   def __len__(self):
        return len(self.files)   def __getitem__(self, index):
       filename = self.files[index]
       input = Image.open(os.path.join(self.path, filename))
       label = 0 if filename.find("dog")>=0 else 1
       img_as_tensor = self.transform(input)
       return img_as_tensor, labeltransformations = transforms.Compose(
         [transforms.Resize((224,224)),transforms.ToTensor()]
                 )

path = "./data"
train_dataset = data(path, transformations)
dataloader = DataLoader(train_dataset, batch_size=Train_Batch_Size, shuffle=True)

import os
import gensim
from collections import Counter
import json

train_path = "./aclImdb/train"
test_path = "./aclImdb/test"

# simple函数从目录读取数据并返回数据和标签
# 你可以为其他数据集制作自己的读取器。
def reader(path):
    pos_path = os.path.join(path, "pos")
    neg_path = os.path.join(path, "neg")
    data = []
    label = []
    for file in os.listdir(pos_path):
        f = open(os.path.join(pos_path, file))
        data.append(f.read())
        label.append(1)
    for file in os.listdir(neg_path):
        f = open(os.path.join(neg_path, file))
        data.append(f.read())
        label.append(0)
    # print(data[:1])
    return data, label

def build_vocab(data, min_word_count = 5):
    counter = Counter()
    for line in data:
        l = gensim.utils.simple_preprocess(line)
        counter.update(l)
    # 初始化一个字典或查找表
    word2id = {}
    word2id['<pad>'] = 0
    word2id['<unk>'] = 1
    # 只包括那些在字典中出现超过min次的单词。
    words = [word for word, count in counter.items() if count>min_word_count]

    for i, word in enumerate(words):
        word2id[word] = i+2

    with open("word2id.json", 'w') as f:
        json.dump(word2id, f)
    return word2id

data, label = reader(train_path)
word2id = build_vocab(data)
print("Dictionary Formed and saved. The length of dictionary is-: ", len(word2id))

import torch
from torch.utils.data import Dataset, DataLoader
import numpy as np
import os
import gensim

class Dataset_seq(Dataset):
    def __init__(self, word2id, train_path):
        self.word2id = word2id
        self.train_path = train_path
        # 读取数据和标签
        self.data, self.label = reader(train_path)

    def __getitem__(self, index):
        # 返回seq和标签
        seq = self.preprocess(self.data[index])
        label = self.label[index]
        return seq, label

    def __len__(self):
        return(len(self.data))

    def preprocess(self, text):
        # 用于将line转换为token，然后使用word2id将其转换为相应的数字值
        line = gensim.utils.simple_preprocess(text)
        seq = []
        for word in line:
            if word in self.word2id:
                seq.append(self.word2id[word])
            else:
                seq.append(self.word2id['<unk>'])
        # 将list转换成张量
        seq = torch.from_numpy(np.array(seq))
        return seq

train_dataset = Dataset_seq(word2id, train_path)
train_dataloader = DataLoader(dataset=train_dataset, batch_size=batch_size, shuffle=True,collate_fn=collate_fn)

def collate_fn(data):
    '''  
    我们应该构建一个自定义的collate_fn，而不是使用默认的collate_fn，
    因为每个句子的大小不同，并且默认不支持合并序列。
    Args:
        data: 元组列表 (training sequence, label)
    Return:
        padded_seq - 填充序列，形状 (batch_size, padded_length)
        length - 每个序列的原始长度(没有填充), 形状(batch_size)
        label - 张量形状 (batch_size)
    '''

    data.sort(key=lambda x: len(x[0]), reverse=True)
    sequences, label = zip(*data)
    length = [len(seq) for seq in sequences]
    padded_seq = torch.zeros(len(sequences), max(length)).long()
    for i, seq in enumerate(sequences):
        end = length[i]
        padded_seq[i,:end] = seq
    return padded_seq, torch.from_numpy(np.array(length)), torch.from_numpy(np.array(label))