Python中运行并行任务技巧

798次阅读  |  发布于5年以前

示例

标准线程多进程,生产者/消费者示例:
Worker越多,问题越大

复制代码 代码如下:

-- coding: utf8 --

import os
import time
import Queue
import threading
from PIL import Image

def createthumbnail(filename, size=(128, 128)):
try:
fp, fmt = filename.rsplit('.', 1)
im = Image.open(filename)
im.thumbnail(size, Image.ANTIALIAS)
im.save((fp + '
'+'x'.join(str(i) for i in size) + '.'+fmt), im.format)
return '%s thumbnail success!' % filename
except Exception:
return '%s thumbnail failed!' % filename

def get_image_paths(folder):
return [os.path.join(folder, f) for f in os.listdir(folder) if 'png' in f]

class Consumer(threading.Thread):
def init(self, queue):
threading.Thread.init(self)
self._queue = queue

def run(self):  
    while True:  
        content = self._queue.get()  
        if isinstance(content, str) and content == 'quit':  
            break  
        respone = create_thumbnail(content)  
    print 'Bye bye!'

def Producer():
filenames = get_image_paths('images')
queue = Queue.Queue()
worker_threads = build_worker_pool(queue, 4)
start_time = time.time()

for filename in filenames:  
    queue.put(filename)  
for worker in worker_threads:  
    queue.put('quit')  
for worker in worker_threads:  
    worker.join()

print time.time() - start_time

def build_workerpool(queue, size):
workers = []
for
in range(size):
worker = Consumer(queue)
worker.start()
workers.append(worker)
return workers

if name == 'main':
Producer()

Copyright© 2013-2020

All Rights Reserved 京ICP备2023019179号-8