我有如下数据组织如下:所以,内部培训或验证,我有2个文件夹,一个消极和积极的一个。我有标题ValueError: num_samples should be a positive integer value, but got num_samples=0
的错误,因为基本上我是在/dataset/train_或_validation中,但是接下来需要访问文件夹neg或pos。图像采用这种格式: MCUCXR_0000_1.png表示正图像,MCUCXR_0000_0.png表示负类。我正在考虑从文件夹中提取所有的图像,以便拥有/dataset/train_或_validation/Images.png,但是在本例中,我如何指定哪个类是类呢?或者,我如何遍历正/负文件夹?这是我的密码:
"""Montgomery Shard Descriptor."""
import logging
import os
from typing import List
from torchvision.datasets import ImageFolder
from torch.utils.data import DataLoader
from pathlib import Path
import numpy as np
import requests
from openfl.interface.interactive_api.shard_descriptor import ShardDataset
from openfl.interface.interactive_api.shard_descriptor import ShardDescriptor
from torchvision import transforms
# Compose transformations
train_transform = transforms.Compose([
transforms.RandomHorizontalFlip(),
transforms.RandomVerticalFlip(),
transforms.Resize((512, 512)),
transforms.ToTensor(),
])
test_transform = transforms.Compose([
transforms.Resize((512, 512)),
transforms.ToTensor(),
])
logger = logging.getLogger(__name__)
class MontgomeryShardDataset(ShardDataset):
"""Montgomery Shard dataset class."""
def __init__(self, dataset_dir: Path, dataset_type: str,):
"""Initialize MontgomeryDataset."""
self.data_type = dataset_type
self.dataset_dir = dataset_dir
print(self.dataset_dir)
self.imgs_path = list(dataset_dir.glob('*.png'))
def __getitem__(self, index: int):
"""Return an item by the index."""
img_path = self.imgs_path[index]
img = Image.open(img_path)
return img
def __len__(self):
"""Return the len of the dataset."""
return len(self.imgs_path)
class MontgomeryShardDescriptor(ShardDescriptor):
"""Montgomery Shard descriptor class."""
def __init__(
self,
data_folder: str = 'montgomery_data',
**kwargs
):
"""Initialize MontgomeryShardDescriptor."""
#print("Path at terminal when executing this file")
print(os.getcwd() + "\n")
#print(self.common_data_folder)
self.data_folder = data_folder
self.dataset_dir = Path.cwd() / data_folder
trainset, testset = self.get_data()
print("IO SONO" + "\n")
print(self.dataset_dir)
self.data_by_type = {
'train': self.dataset_dir / 'TRAIN',
'val': self.dataset_dir / 'TEST'
}
def get_shard_dataset_types(self) -> List[str]:
"""Get available shard dataset types."""
return list(self.data_by_type)
def get_dataset(self, dataset_type='train'):
"""Return a shard dataset by type."""
print("Path at terminal when executing this file")
print(os.getcwd() + "\n")
#os.chdir("/home/lmancuso/openfl/openfl-tutorials/interactive_api/OPENLAB/envoy")
if dataset_type not in self.data_by_type:
raise Exception(f'Wrong dataset type: {dataset_type}')
return MontgomeryShardDataset(
dataset_dir=self.data_by_type[dataset_type],
dataset_type=dataset_type,
)
@property
def sample_shape(self):
"""Return the sample shape info."""
return ['3', '512', '512']
@property
def target_shape(self):
"""Return the target shape info."""
return ['3', '512', '512']
@property
def dataset_description(self) -> str:
"""Return the dataset description."""
return (f'Montgomery dataset, shard number')
def get_data(self):
root_dir = "montgomery_data"
#train_set = ImageFolder(os.path.join(root_dir, "TRAIN"), transform=train_transform)
#test_set = ImageFolder(os.path.join(root_dir, "TEST"), transform=test_transform)
train_set = os.path.join(root_dir, "TRAIN")
test_set = os.path.join(root_dir, "TEST")
print('Montgomery data was loaded!')
return train_set, test_set
我正在使用由英特尔,OpenFL开发的联邦学习框架。正如您所看到的,我也尝试使用ImageFolder,因为我认为它在本例中是有用的。
使用完整的跟踪编辑:
new_state[k] = pt.from_numpy(tensor_dict.pop(k)).to(device)
ERROR Collaborator failed with error: num_samples should be a positive integer value, but got num_samples=0: envoy.py:93
Traceback (most recent call last):
File "/home/lmancuso/openfl/openfl/component/envoy/envoy.py", line 91, in run
self._run_collaborator()
File "/home/lmancuso/openfl/openfl/component/envoy/envoy.py", line 164, in _run_collaborator
col.run()
File "/home/lmancuso/openfl/openfl/component/collaborator/collaborator.py", line 145, in run
self.do_task(task, round_number)
File "/home/lmancuso/openfl/openfl/component/collaborator/collaborator.py", line 259, in do_task
**kwargs)
File "/home/lmancuso/openfl/openfl/federated/task/task_runner.py", line 117, in collaborator_adapted_task
loader = self.data_loader.get_train_loader()
File "/tmp/ipykernel_8572/1777129341.py", line 35, in get_train_loader
File "/home/lmancuso/bruno/lib/python3.7/site-packages/torch/utils/data/dataloader.py", line 262, in __init__
sampler = RandomSampler(dataset, generator=generator) # type: ignore
File "/home/lmancuso/bruno/lib/python3.7/site-packages/torch/utils/data/sampler.py", line 104, in __init__
"value, but got num_samples={}".format(self.num_samples))
ValueError: num_samples should be a positive integer value, but got num_samples=0
INFO Send WaitExperiment request director_client.py:80
INFO WaitExperiment response has received director_client.py:82
发布于 2022-04-14 08:52:24
问题是数据集是空的。数据路径可能是错误的,或者预处理可能导致问题,最终在Dataset对象中没有对象。
发布于 2022-08-05 20:00:53
这是因为我们的图像或标签列表是空的。
考虑到上述问题中的dataset类,
self.imgs_path = list(dataset_dir.glob('*.png'))
是空的。因此
def __len__(self):
"""Return the len of the dataset."""
return len(self.imgs_path)
此方法返回len = 0
。
因此,请确保数据被正确读取。通过打印此len
来验证它
https://stackoverflow.com/questions/71615089
复制相似问题