import pickle
import numpy as np
import glob, gzip
import multiprocessing as mp
from functools import partial
import tqdm
def daemon(paths, idx):
item = paths[idx].split(' ')
data = pickle.load(gzip.open(item[0], 'rb'))
images = data.astype(np.float32)
print(item[0], images.shape)
root = '/data/ninja/workspace/data/boyshome/train.txt'
ifile = open(root)
files = ifile.readlines()
indices = list(range(len(files)))
worker = partial(daemon, files)
pool = mp.Pool(24)
tqdm.tqdm(pool.map(func=worker, iterable=indices))
pool.close()
pool.join()
No comments:
Post a Comment