Python实现抓取网页并且解析的实例

本文以实例形式讲述了Python实现抓取网页并解析的功能。主要解析问答与百度的首页。分享给大家供大家参考之用。

主要功能代码如下：

    #!/usr/bin/python
    #coding=utf-8

    import sys 
    import re
    import urllib2
    from urllib import urlencode
    from urllib import quote
    import time
    maxline = 2000

    wenda = re.compile("href=\"http://wenda.so.com/q/.+\?src=(.+?)\"")
    baidu = re.compile("<a href=\"http://www.baidu.com/link\?url=.+\".*?>更多知道相关问题.*?</a>")
    f1 = open("baidupage.txt","w")
    f2 = open("wendapage.txt","w")

    for line in sys.stdin:
      if maxline == 0:
        break
      query = line.strip();
      time.sleep(1);
      recall_url = "http://www.so.com/s?&q;=" + query;
      response = urllib2.urlopen(recall_url);
      html = response.read();                                                   
      f1.write(html)
      m = wenda.search(html);
      if m:
        if m.group(1) == "110":
          print query + "\twenda\t0";
        else:
          print query + "\twenda\t1";
      else:
        print query + "\twenda\t0";
      recall_url = "http://www.baidu.com/s?wd=" + query +"&ie;=utf-8";
      response = urllib2.urlopen(recall_url);
      html = response.read();
      f2.write(html)
      m = baidu.search(html);
      if m:
        print query + "\tbaidu\t1";
      else:
        print query + "\tbaidu\t0";
      maxline = maxline - 1;
    f1.close()
    f2.close()

希望本文所述对大家Python程序设计的学习有所帮助。