Python制作爬虫采集小说

开发工具：python3.4
操作系统：win8
主要功能：去指定小说网页爬小说目录，按章节保存到本地，并将爬过的网页保存到本地配置文件。
被爬网站：http://www.cishuge.com/
小说名称：灵棺夜行
代码出处：本人亲自码的

    import urllib.request
    import http.cookiejar

    import socket
    import time
    import re

    timeout = 20
    socket.setdefaulttimeout(timeout)

    sleep_download_time = 10
    time.sleep(sleep_download_time)

    def makeMyOpener(head = {
     'Connection': 'Keep-Alive',
     'Accept': 'text/html, application/xhtml+xml, */*',
     'Accept-Language': 'en-US,en;q=0.8,zh-Hans-CN;q=0.5,zh-Hans;q=0.3',
     'User-Agent': 'Mozilla/5.0 (Windows NT 6.3; WOW64; Trident/7.0; rv:11.0) like Gecko'
    }):
     cj = http.cookiejar.CookieJar()
     opener = urllib.request.build_opener(urllib.request.HTTPCookieProcessor(cj))
     header = []
     for key, value in head.items():
      elem = (key, value)
      header.append(elem)
     opener.addheaders = header
     return opener

    def saveFile(save_path,txts):
     f_obj = open(save_path,'w+')
     for item in txts:
      f_obj.write(item+'\n')
     f_obj.close()

    #get_code_list
    code_list='http://www.cishuge.com/read/0/771/'
    oper = makeMyOpener()
    uop = oper.open(code_list,timeout=1000)
    data = uop.read().decode('gbk','ignore')

    pattern = re.compile('<li><a href="(.*?)".*?>(.*?)</a></li>',re.S)

    items = re.findall(pattern,data)

    print ('获取列表完成')
    url_path='url_file.txt'

    url_r=open(url_path,'r')
    url_arr=url_r.readlines(100000)
    url_r.close()
    print (len(url_arr))

    url_file=open(url_path,'a')

    print ('获取已下载网址')

    for tmp in items:
     save_path = tmp[1].replace(' ','')+'.txt'
     url = code_list+tmp[0]
     if url+'\n' in url_arr:
      continue
     print('写日志：'+url+'\n')
     url_file.write(url+'\n')
     opene = makeMyOpener()
     op1 = opene.open(url,timeout=1000)
     data = op1.read().decode('gbk','ignore')
     opene.close()
     pattern = re.compile('    (.*?)<br />',re.S)
     txts = re.findall(pattern,data)
     saveFile(save_path,txts)

    url_file.close()

虽然代码还是有点瑕疵，还是分享给大家，一起改进