python使用scrapy解析js示例

复制代码 代码如下:

from selenium import selenium

class MySpider(CrawlSpider):
name = 'cnbeta'
allowed_domains = ['cnbeta.com']
start_urls = ['http://www.jb51.net']

rules = (  
    # Extract links matching 'category.php' (but not matching 'subsection.php')  
    # and follow links from them (since no callback means follow=True by default).  
    Rule(SgmlLinkExtractor(allow=('/articles/.*\\.htm', )),  
         callback='parse_page', follow=True),

    # Extract links matching 'item.php' and parse them with the spider's method parse_item  
)

def __init__(self):  
    CrawlSpider.__init__(self)  
    self.verificationErrors = []  
    self.selenium = selenium("localhost", 4444, "*firefox", "http://www.jb51.net")  
    self.selenium.start()

def __del__(self):  
    self.selenium.stop()  
    print self.verificationErrors  
    CrawlSpider.__del__(self)


def parse_page(self, response):  
    self.log('Hi, this is an item page! %s' % response.url)  
    sel = Selector(response)  
    from webproxy.items import WebproxyItem

    sel = self.selenium  
    sel.open(response.url)  
    sel.wait_for_page_to_load("30000")  
    import time

    time.sleep(2.5)