bladedragon

pip3 install lxml

pip3 install wheel

pip3 install pyOpenSSL-19.1.0-py2.py3-none-any.whl 

pip3 install Twisted-20.3.0-cp37-cp37m-win_amd64.whl

pip3 install Scrapy

scrapy startproject xxx

import scrapy

#这里我们需要排名、标题、收藏人数、上映时间和分数
class SpiderDemoItem(scrapy.Item):
    # define the fields for your item here like:
    # name = scrapy.Field()
    index = scrapy.Field()
    title = scrapy.Field()
    star = scrapy.Field()
    releasetime = scrapy.Field()
    score = scrapy.Field()

import scrapy
from spider_demo.items import SpiderDemoItem


class MaoyanSpider(scrapy.Spider):
    #这是爬虫启动的名称，之后启动爬虫就需要用到这个名称
    name = "maoyan"
    #可以爬取的域名可选列表
    allowed_domains = ["maoyan.com"]
    #目标爬取的网址
    start_urls = [
        "http://maoyan.com/board/7/", 
        "http://maoyan.com/board/4/",
    ]

    #处理已经下载的页面
    def parse(self, response):
        
        dl = response.css(".board-wrapper dd")
        #通过解析得到具体的数据存到容器中
        for dd in dl:
            item = SpiderDemoItem()
            item["index"] = dd.css(".board-index::text").extract_first()
            item["title"] = dd.css(".name a::text").extract_first()
            item["star"] = dd.css(".star::text").extract_first()
            item["releasetime"] = dd.css(".releasetime::text").extract_first()
            score = dd.css('.integer::text').extract_first()
            if score is not None:
                item["score"] = score  + dd.css('.fraction::text').extract_first()
            else:
                item["score"] = 0

            #通过yield将结果返回
            yield item

#如果没有自动生成UA，就需要手动定义，但是每次爬取都是同样的UA容易出现验证操作，因此后面还会介绍一种随机生成UA的方法
USER_AGENT = 'Mozilla/5.0 (Windows NT 6.1; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/75.0.3770.100 Safari/537.36'

#允许机器人协议，关于机器人协议的具体内容可以自行上网查找
ROBOTSTXT_OBEY = False

import json
import codecs


class SpiderDemoPipeline:
    def process_item(self, item, spider):
        return item


class JsonPipline(object):
    def __init__(self):
        print("打开文件，准备写入....")
        self.file = codecs.open("maoyan.json", "wb", encoding='utf-8')

    def process_item(self, item, spider):
        print("准备写入...")
        line = json.dumps(dict(item), ensure_ascii=False) + "\n"
        self.file.write(line)
        return item

    def close_spider(self, spider):
        print("写入完毕，关闭文件")
        self.file.close

ITEM_PIPELINES = {
    #    'spider_demo.pipelines.SpiderDemoPipeline': 300,
    'spider_demo.pipelines.JsonPipline': 200
}

from scrapy import signals
import random
from fake_useragent import UserAgent


class RandomUserAgentMiddleware(object):
    # 随机更换user-agent
    def __init__(self, crawler):
        super(RandomUserAgentMiddleware, self).__init__()
        self.ua = UserAgent()
        self.ua_type = crawler.settings.get("RANDOM_UA_TYPE", "random")

    @classmethod
    def from_crawler(cls, crawler):
        return cls(crawler)

    def process_request(self, request, spider):
        def get_ua():
            return getattr(self.ua, self.ua_type)

        request.headers.setdefault('User-Agent', get_ua())

DOWNLOADER_MIDDLEWARES = {
    'spider_demo.middlewares.SpiderDemoDownloaderMiddleware': 543,
     'spider_demo.middlewares.RandomUserAgentMiddleware': 400,
    'scrapy.downloadermiddleware.useragent.UserAgentMiddleware': None,
}

# 随机选择UA
#这个是自己设置的，依赖于fake-useragent
RANDOM_UA_TYPE = 'random'