StreamDataset¶

class StreamDataset[源代码]¶

一个流数据的抽象类。

__iter__ 方法是额外需要的。

实际案例

from megengine.data.dataset import StreamDataset
from megengine.data.dataloader import DataLoader, get_worker_info
from megengine.data.sampler import StreamSampler

class MyStream(StreamDataset):
    def __init__(self):
        self.data = [iter([1, 2, 3]), iter([4, 5, 6]), iter([7, 8, 9])]
    def __iter__(self):
        worker_info = get_worker_info()
        data_iter = self.data[worker_info.idx]
        while True:
            yield next(data_iter)

dataloader = DataLoader(
    dataset = MyStream(),
    sampler = StreamSampler(batch_size=2),
    num_workers=3,
    parallel_stream = True,
)

for step, data in enumerate(dataloader):
    print(data)

ArrayDataset

VisionDataset