python爬虫使用scrapy框架

阅读原文时间：2022年01月12日阅读：3

关注公众号“轻松学编程”了解更多

1、创建启动爬虫脚本

在项目目录下创建start.py文件：

添加代码：

#以后只要运行start.py就可以启动爬虫
import scrapy.cmdline

def main():
    #mytencent为当前项目爬虫名
    scrapy.cmdline.execute(['scrapy', 'crawl', 'mytencent'])

if __name__ == '__main__':
    main()

2、自动爬取多页

在spiders文件夹下的mytencent.py中MytencentSpider类要继承CrawlSpider，然后添加规则即可：

import scrapy
from tencent.items import TencentItem

from scrapy.spiders import CrawlSpider, Rule  # 爬取规则
from scrapy.linkextractors import LinkExtractor  # 提取链接

#爬虫类继承CrawlSpider
class MytencentSpider(CrawlSpider):
    name = 'mytencent'
    allowed_domains = ['hr.tencent.com']
    start_urls = ['https://hr.tencent.com/position.php?keywords=&tid=0&start=10#a']

    #添加爬取url规则，url符合正则start=(\d+)#a")就爬取
    rules = (Rule(LinkExtractor(allow=("start=(\d+)#a")), callback='get_parse', follow=True),)

    # 一定不能用parse()
    def get_parse(self, response):
        jobList = response.xpath('//tr[@class="even"] | //tr[@class="odd"]')

        # 存储对象
        item = TencentItem()

        for job in jobList:
            # .extract()提取文本
            jobName = job.xpath('./td[1]/a/text()').extract()[0]
            jobType = job.xpath('./td[2]/text()').extract()[0]
            item['jobName'] = jobName
            item['jobType'] = jobType

            yield item

3、使用框架自带的Request()构建请求

在spiders文件夹下的mysina.py中:

import scrapy
from scrapy.spiders import CrawlSpider,Rule #爬取规则
from scrapy.linkextractor import LinkExtractor #提取链接

class MysinaSpider(CrawlSpider):
    name = 'mysina'
    allowed_domains = ['sina.com.cn']
    start_urls = ['http://roll.news.sina.com.cn/news/gnxw/gdxw1/index_1.shtml']
    #设置爬取规则,可迭代对象，可设置多个规则
    rules = [Rule(LinkExtractor(allow=("index_(\d+).shtml")),callback='get_parse',follow=True)]

    def get_parse(self, response):
        newsList = response.xpath('//ul[@class="list_009"]/li')
        for news in newsList:
            # 新闻标题
            title = news.xpath('./a/text()').extract()[0]
            # 新闻时间
            newsTime = news.xpath('./span/text()').extract()[0]
            # print('***********',title,'****',newsTime)
            #获取正文的url
            contentsUrl = news.xpath('./a/@href').extract()[0]
            #使用框架自带的Request()构建请求,使用meta传递参数
            '''
            scrapy.Request()参数列表:
            url,
            callback=None, 回调函数
            meta=None, 数据传递
            '''
            request = scrapy.Request(url=contentsUrl,callback=self.get_article,)
            # 使用meta传递参数 是一个字典, 只能传递一层
            request.meta['title'] = title
            request.meta['newsTime'] = newsTime

            yield request

    def get_article(self,response):
        contents = response.xpath('//div[@id="article"]//text()')
        #新闻内容
        newsContent = ""
        for content in contents:
            newsContent += content.extract().strip()+'\n'
        print('*****新闻正文*****',newsContent,'*****新闻正文*****')
        item = SinaItem()
        # 从meta中获取参数
        item['title'] = response.meta['title']
        item['newsTime'] = response.meta['newsTime']

        item['newsContent'] = newsContent

        yield item

4、保存进MySQL数据库模板

在MySQL中建立数据库，表，然后在pipelines.py中编写代码如下：

import pymysql

class TencentPipeline(object):
    def __init__(self):
        #连接数据库
        self.conn = None
        #游标
        self.cur = None

    # 打开爬虫时调用，只调用一次
    def open_spider(self,spider):
        self.conn = pymysql.connect(host='127.0.0.1',
                                    user='root',
                                    password="123456",
                                    database='tjob', #数据库为tjob
                                    port=3306,
                                    charset='utf8')
        self.cur = self.conn.cursor()

    def process_item(self, item, spider):
        clos,value = zip(*item.items())
        sql = "INSERT INTO `%s`(%s) VALUES (%s)" % ('tencentjob', #表名为tencentjob
                                                    ','.join(clos),
                                                    ','.join(['%s'] * len(value)))

        self.cur.execute(sql, value)

        self.conn.commit()
        return item

    def close_spider(self, spider):
        self.cur.close()
        self.conn.close()

settings.py中要开启

ITEM_PIPELINES = {
   'tencent.pipelines.TencentPipeline': 300,
}

5、使用中间件做UA代理，IP代理

在middlewares.py中添加：

from scrapy import signals
import random

#ip代理
from scrapy.downloadermiddlewares.httpproxy import HttpProxyMiddleware
#UA代理
from scrapy.downloadermiddlewares.useragent import UserAgentMiddleware

from weixinsougou.settings import USER_AGENTS,PROXIES

class RandomUAMiddleware(UserAgentMiddleware):
    '''
    随机UA代理，中间件
    '''
    def process_request(self, request, spider):
        '''
        所有的请求都会经过process_request
        :param request:请求
        :param spider:爬虫名
        :return:
        '''
        ua = random.choice(USER_AGENTS)
        request.headers.setdefault("User-Agent", ua)

class RandomIPMiddleware(HttpProxyMiddleware):
    '''
    随机IP代理
    '''

    def process_request(self, request, spider):
        proxy = random.choice(PROXIES)
        request.meta['proxy'] = 'http://' + proxy['ip_port']

#class RandomCookieMiddleware(CookiesMiddleware):
#     '''
#     随机cookie池
#     '''
#
#     def process_request(self, request, spider):
#         cookie = random.choice(COOKIES)
#         request.cookies = cookie

在settings.py中添加：

# -*- coding: utf-8 -*-

# Obey robots.txt rules
ROBOTSTXT_OBEY = False

# Disable cookies (enabled by default)
COOKIES_ENABLED = False

# Override the default request headers:
DEFAULT_REQUEST_HEADERS = {
  'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',
  'Accept-Language': 'en',
  'User-Agent':'Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/55.0.2883.87 Safari/537.36',
}

# Enable or disable downloader middlewares
# See https://doc.scrapy.org/en/latest/topics/downloader-middleware.html
#启用中间件
DOWNLOADER_MIDDLEWARES = {
   # 'weixinsougou.middlewares.WeixinsougouDownloaderMiddleware': 543,
   'weixinsougou.middlewares.RandomUAMiddleware': 543,
   'weixinsougou.middlewares.RandomIPMiddleware': 544,

}

#UA池
USER_AGENTS = [
    "Mozilla/5.0 (compatible; MSIE 9.0; Windows NT 6.1; Win64; x64; Trident/5.0; .NET CLR 3.5.30729; .NET CLR 3.0.30729; .NET CLR 2.0.50727; Media Center PC 6.0)",
    "Mozilla/5.0 (compatible; MSIE 8.0; Windows NT 6.0; Trident/4.0; WOW64; Trident/4.0; SLCC2; .NET CLR 2.0.50727; .NET CLR 3.5.30729; .NET CLR 3.0.30729; .NET CLR 1.0.3705; .NET CLR 1.1.4322)",
    "Mozilla/4.0 (compatible; MSIE 7.0b; Windows NT 5.2; .NET CLR 1.1.4322; .NET CLR 2.0.50727; InfoPath.2; .NET CLR 3.0.04506.30)",
    "Mozilla/5.0 (Windows; U; Windows NT 5.1; zh-CN) AppleWebKit/523.15 (KHTML, like Gecko, Safari/419.3) Arora/0.3 (Change: 287 c9dfb30)",
    "Mozilla/5.0 (X11; U; Linux; en-US) AppleWebKit/527+ (KHTML, like Gecko, Safari/419.3) Arora/0.6",
    "Mozilla/5.0 (Windows; U; Windows NT 5.1; en-US; rv:1.8.1.2pre) Gecko/20070215 K-Ninja/2.1.1",
    "Mozilla/5.0 (Windows; U; Windows NT 5.1; zh-CN; rv:1.9) Gecko/20080705 Firefox/3.0 Kapiko/3.0",
    "Mozilla/5.0 (X11; Linux i686; U;) Gecko/20070322 Kazehakase/0.4.5"
]

#IP池
PROXIES = [
    {'ip_port': '171.38.85.93:8123'},
    {'ip_port': '113.67.227.143:8118'},
    {'ip_port': '101.236.19.165:8866'},
    {'ip_port': '101.236.21.22:8866'},
]

#cookle池
COOKIES = []

# 默认线程数量 10
REACTOR_THREADPOOL_MAXSIZE = 20

# 并发 默认16
CONCURRENT_REQUESTS = 16

# pipelines同时处理数量 默认100
CONCURRENT_ITEMS = 50

# scrapy 深度爬取，默认0 不做深度限制
DEPTH_LIMIT = 4
# 下载超时
DOWNLOAD_TIMEOUT = 180

#####6、使用redis实现分布式爬取

https://blog.csdn.net/lm_is_dc/article/details/81866275

#####7、部署

https://blog.csdn.net/lm_is_dc/article/details/81869508