Thursday 29 April 2021

How to do multiprocessing in python?

import pickle
import numpy as np
import glob, gzip
import multiprocessing as mp
from functools import partial
import tqdm

def daemon(paths, idx):
    item = paths[idx].split(' ')
    data = pickle.load(gzip.open(item[0], 'rb'))
    images = data.astype(np.float32)
    print(item[0], images.shape)

root = '/data/ninja/workspace/data/boyshome/train.txt'
ifile = open(root)
files = ifile.readlines()
indices = list(range(len(files)))

worker = partial(daemon, files)
pool = mp.Pool(24)
tqdm.tqdm(pool.map(func=worker, iterable=indices))
pool.close()
pool.join()

No comments:

Post a Comment