python基于queue和threading实现多线程下载实例

本文实例讲述了python基于queue和threading实现多线程下载的方法，分享给大家供大家参考。具体方法如下：

主代码如下：

      #download worker 
      queue_download = Queue.Queue(0) 
      DOWNLOAD_WORKERS = 20 
      for i in range(DOWNLOAD_WORKERS): 
        DownloadWorker(queue_download).start() #start a download worker 

      for md5 in MD5S: 
        queue_download.put(md5) 
      for i in range(DOWNLOAD_WORKERS): 
        queue_download.put(None) 

其中downloadworkers.py
类继承 threading.Thread,重载run方法..在init中调用threading.Thread.init(self),
在run方法中实现耗时的操作

    import threading 
    import Queue 
    import md5query 
    import DOM 
    import os,sys 

    class DownloadWorker(threading.Thread): 
      """""" 


      def __init__(self, queue): 
        """Constructor""" 
        self.__queue = queue 
        threading.Thread.__init__(self) 


      def run(self): 
        while 1: 
          md5 = self.__queue.get() 
          if md5 is None: 
            break #reached end of queue 
          #this is a time-cost produce 
          self._down(md5) 

          print "task:", md5, "finished" 

      def _down(self, md5): 
        config = { 
          'input':sys.stdin,  
          'output':'./samples',  
          'location':'xxx',  
          'has-fn':False,  
          'options':{'connect.timeout':60, 'timeout':3600},  
          'log':file('logs.txt', 'w'),  
        } 
        print 'download %s...' % (md5) 
        try: 
          data = downloadproc(config['location'], config['options'])#我的下载过程 
          if data: 
            dom, fileData = md5query.splited(data) 
            filename = md5 
            if config['has-fn']: 
              filename = '%s_%s' % (md5, dom.nodeValue2('xxxxxxx', '').encode('utf-8'))#这是我的下载的方法 
            f = file(os.path.join(config['output'], filename), 'w') 
            f.write(fileData) 
            f.close() 

            print '%s\tok' % (md5) 
          else: 
            print>>config['log'], '%s\t%s' % (md5, 'failed') 
        except Exception, e: 
          print>>config['log'], '%s\t%s' % (md5, str(e))

希望本文所述对大家的Python程序设计有所帮助。