Python实现批量下载文件

Python实现批量下载文件

    #!/usr/bin/env python
    # -*- coding:utf-8 -*-

    from gevent import monkey
    monkey.patch_all()
    from gevent.pool import Pool
    import requests
    import sys
    import os

    def download(url):
     chrome = 'Mozilla/5.0 (X11; Linux i86_64) AppleWebKit/537.36 ' + 
     '(KHTML, like Gecko) Chrome/41.0.2272.101 Safari/537.36'
     headers = {'User-Agent': chrome}
     filename = url.split('/')[-1].strip()
     r = requests.get(url.strip(), headers=headers, stream=True)
     with open(filename, 'wb') as f:
     for chunk in r.iter_content(chunk_size=1024):
     if chunk:
    f.write(chunk)
    f.flush()
     print filename,"is ok"

    def removeLine(key, filename):
     os.system('sed -i /%s/d %s' % (key, filename))

    if __name__ =="__main__":
     if len(sys.argv) == 2:
     filename = sys.argv[1]
     f = open(filename,"r")
     p = Pool(4)
     for line in f.readlines():
     if line:
     p.spawn(download, line.strip())
     key = line.split('/')[-1].strip()
     removeLine(key, filename)
    f.close()
    p.join()
    else:
     print 'Usage: python %s urls.txt' % sys.argv[0]

其他网友的方法：

    from os.path import basename
    from urlparse import urlsplit
    def url2name(url):
      return basename(urlsplit(url)[2])

    def download(url, localFileName = None):
      localName = url2name(url)
      req = urllib2.Request(url)
      r = urllib2.urlopen(req)
      if r.info().has_key('Content-Disposition'):
        # If the response has Content-Disposition, we take file name from it
        localName = r.info()['Content-Disposition'].split('filename=')[1]
        if localName[0] == '"' or localName[0] == "'":
          localName = localName[1:-1]
      elif r.url != url:
        # if we were redirected, the real file name we take from the final URL
        localName = url2name(r.url)
      if localFileName:
        # we can force to save the file as specified name
        localName = localFileName
      f = open(localName, 'wb')
      f.write(r.read())
      f.close()

    download(r'你要下载的python文件的url地址')

以上便是本文给大家分享的全部内容了，小伙伴们可以测试下哪种方法效率更高呢。