使用Python3编写抓取网页和只抓网页图片的脚本

最基本的抓取网页内容的代码实现：

    #!/usr/bin/env python 

    from urllib import urlretrieve 

    def firstNonBlank(lines): 
      for eachLine in lines: 
        if not eachLine.strip(): 
          continue 
        else: 
          return eachLine 

    def firstLast(webpage): 
      f = open(webpage) 
      lines = f.readlines() 
      f.close() 
      print firstNonBlank(lines), 
      lines.reverse() 
      print firstNonBlank(lines), 

    def download(url='http://www',process=firstLast): 
      try: 
        retval = urlretrieve(url)[0] 
      except IOError: 
        retval = None 
      if retval: 
        process(retval) 

    if __name__ == '__main__': 
      download() 

利用urllib模块，来实现一个网页中针对图片的抓取功能：

    import urllib.request 
    import socket 
    import re 
    import sys 
    import os 
    targetDir = r"C:\Users\elqstux\Desktop\pic" 
    def destFile(path): 
      if not os.path.isdir(targetDir): 
        os.mkdir(targetDir) 
      pos = path.rindex('/') 
      t = os.path.join(targetDir, path[pos+1:]) 
      return t 

    if __name__ == "__main__": 
      hostname = "http://www.douban.com" 
      req = urllib.request.Request(hostname) 
      webpage = urllib.request.urlopen(req) 
      contentBytes = webpage.read() 
      for link, t in set(re.findall(r'(http:[^\s]*?(jpg|png|gif))', str(contentBytes))): 
        print(link) 
        urllib.request.urlretrieve(link, destFile(link)) 

    import urllib.request 
    import socket 
    import re 
    import sys 
    import os 
    targetDir = r"H:\pic" 
    def destFile(path): 
      if not os.path.isdir(targetDir): 
        os.mkdir(targetDir) 
      pos = path.rindex('/') 
      t = os.path.join(targetDir, path[pos+1:]) #会以/作为分隔 
      return t 

    if __name__ == "__main__": 
      hostname = "http://www.douban.com/" 
      req = urllib.request.Request(hostname) 
      webpage = urllib.request.urlopen(req) 
      contentBytes = webpage.read() 
      match = re.findall(r'(http:[^\s]*?(jpg|png|gif))', str(contentBytes) )#r'(http:[^\s]*?(jpg|png|gif))'中包含两层圆括号，故有两个分组， 
                                 #上面会返回列表，括号中匹配的内容才会出现在列表中 
      for picname, picType in match: 
        print(picname) 
        print(picType) 


    ''''' 
    输出： 
    http://img3.douban.com/pics/blank.gif 
    gif 
    http://img3.douban.com/icon/g111328-1.jpg 
    jpg 
    http://img3.douban.com/pics/blank.gif 
    gif 
    http://img3.douban.com/icon/g197523-19.jpg 
    jpg 
    http://img3.douban.com/pics/blank.gif 
    gif 
    ... 
    '''