python多进程

import glob
import tqdm
from multiprocessing import Process, Queue
import time
import math
import alog, os
from skimage import io
import argparse

# 在主进程中创建特殊字典
m = Manager()
d = m.dict()

# split a list into evenly sized chunks
def chunks(l, n):
    return [l[i:i+n] for i in range(0, len(l), n)]

def do_job(job_id, data_slice):
    for filename in (data_slice):
        try:
            # print(filename)
            if not os.path.exists(filename):
                print("not exist")
            img = io.imread(filename)
        except:
            print(filename)

def dispatch_jobs(data, job_number):
    total = len(data)
    chunk_size = math.ceil(total / job_number)
    slice = chunks(data, chunk_size)
    jobs = []

    for i, s in enumerate(slice):
        j = Process(target=do_job, args=(i, s))
        j.daemon = True
        jobs.append(j)
    for j in jobs:
        j.start()
    exit_codes = [p.join() for p in jobs]
    print("finish")
    return exit_codes

if __name__ == "__main__":
    parser = argparse.ArgumentParser(description='Process some integers.')
    parser.add_argument('--path', type=str, 
                    help='path')

    args = parser.parse_args()

    print("waiting...")
    img_path = os.path.join(args.path, "*.jpg")
    data = [x for x in glob.glob(img_path)]

    dispatch_jobs(data, 40)