Thursday, 29 April 2021

How to do multiprocessing in python?

import pickle
import numpy as np
import glob, gzip
import multiprocessing as mp
from functools import partial
import tqdm

def daemon(paths, idx):
    item = paths[idx].split(' ')
    data = pickle.load(gzip.open(item[0], 'rb'))
    images = data.astype(np.float32)
    print(item[0], images.shape)

root = '/data/ninja/workspace/data/boyshome/train.txt'
ifile = open(root)
files = ifile.readlines()
indices = list(range(len(files)))

worker = partial(daemon, files)
pool = mp.Pool(24)
tqdm.tqdm(pool.map(func=worker, iterable=indices))
pool.close()
pool.join()

No comments:

Post a Comment