今日内容：

一、爬虫三部曲：

1、发送请求

2.解析数据

3.保存数据

4、解析详情页，获取视频地址

mport requests
import re #正则模块
import uuid
#uuid.uuid4(),根据时间戳，生成一段世界上惟一的字符串
#导入线程池模块

#爬虫三部曲
#1.发送请求
def get_page(url):
response=requests.get(url)
return response
#2.解析数据
def parse_index(text):
res = re.findall('<a href="video_(.*?)"', text, re.S)
#print(res)
detail_url_list=[]
for m_id in res:
# 拼接详情页url
detail_url = "https://www.pearvideo.com/video_" + m_id
detail_url_list.append(detail_url)
return detail_url_list

#4.解析详情页，获取视频url
def parse_detail(text):
movie_url=re.findall('srcUrl="(.*?)"',text,re.S)[0]
return movie_url

#3.保存数据

def save_movie(movie_url):

response=requests.get(movie_url)

with open(f'{uuid.uuid4()}.mp4','wb')as f:

f.write(response.content)

f.flush()

if __name__ == '__main__':
#main+回车键

index\_res=get\_page(url='https://www.pearvideo.com/')

detail\_url\_list=parse\_index(index\_res.text)  
#print(detail\_url\_list)

for detail\_url in detail\_url\_list:  
    detail\_res=get\_page(url=detail\_url)  
    movie\_url=parse\_detail(detail\_res.text)  
    print(movie\_url)

    # save\_movie(movie\_url)

二、带参数的爬虫

# '''

知乎添加了反爬功能，需要添加参数使用user-agent

'''

import requests
headers={'user-agent': 'Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/74.0.3729.169 Safari/537.36'
}

#在get请求内添加user-agent

response=requests.get(url='https://www.zhihu.com/explore',headers=headers)

with open('zhihu.html','w',encoding='utf-8') as f:

f.write(response.text)

'''
params参数
访问百度搜查朴灿烈url
'''
from urllib.parse import urlencode

url='https://www.baidu.com/s?ie=utf-8&f=3&rsv_bp=1&tn=02003390_3_hao_pg&wd=%E6%9C%B4%E7%81%BF%E7%83%88'

url='https://www.baidu.com/s?'+urlencode({'wd':'朴灿烈'})
print(url)

response=requests.get(url,headers=headers,params={'百度':'安徽工程大学'})

with open('gongcheng.html','w',encoding='utf-8')as f:

f.write(response.text)

三、爬取梨视频

'''
爬取梨视频（下面三条信息从浏览器中查看）

Request URL：
https://www.pearvideo.com/

Request Method:
GET

Request headers：
user-agent: ***********

'''

import requests
import re # 正则模块
import uuid # uuid.uuid4() 可以根据当前时间戳随机产生成唯一的字符串

爬虫三部曲

1. 发送请求

def get_page(index_url):
response1 = requests.get(index_url)
return response1

2. 解析数据

解析主页获取视频页详情ID

def main_index(text):

# 获取主页视频详情页ID  
res1 = re.findall('<a href="video\_(.\*?)"', text, re.S)

detail\_list1 = \[\]

for m\_id in res1:

    # 拼接详情页url  
    url1 = 'https://www.pearvideo.com/video\_' + m\_id  
    detail\_list1.append(url1)

return detail\_list1

3. 保存数据

def save_data(v_url):
response2 = requests.get(v_url)

with open('{}.mp4'.format(uuid.uuid4()), 'wb') as f:  
    f.write(response2.content)  
    f.flush()

解析详情页获取视频url

def video_detail(text):

'''

    (.\*?) : 提取括号的内容  
    .\*? : 直接匹配  
    <video.\*?src="(.\*?)"

    srcUrl="(.\*?)"  
'''

video\_url1 = re.findall('srcUrl="(.\*?)"', text, re.S)\[0\]  
return video\_url1

if __name__ == '__main__':

# 对主页发送请求  
response = get\_page('https://www.pearvideo.com/')

# 对主页进行解析、获取详情页ID  
detail\_list = main\_index(response.text)

# 对每个详情页url发送请求  
for url in detail\_list:  
    res = get\_page(url)  
    # print(res.text)

    #  解析详情页获取视频ID  
    video\_url = video\_detail(res.text)  
    print(video\_url)

    # 保存视频  
    save\_data(video\_url)

四、今日作业：

今日作业:
爬取豆瓣TOP250部电影信息:
    https://movie.douban.com/top250

    PS: 提取电影详情页url、图片链接、电影名称、
    电影评分、评价人数、导演、主演、电影上映时间、简介

import requests
import re
headers = {'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/74.0.3729.157 Safari/537.36'}
zhengze = '

response = requests.get(url, headers=headers)

movie\_content = re.findall(zhengze, response.text, re.S)

for every\_movie in movie\_content:  
    movie\_url, movie\_jpg, name, director, time, point, num, abstract = every\_movie

    director1 = director.split('&n')\[0\]  
    star = director.split('&n')\[-1\].split('主')\[-1\].split('演')\[-1\].split(':')\[-1\].lstrip()

    data = '名称：{}, 电影链接：{}, 图片：{}, 导演：{}, 主演：{}, 时间：{}, 得分：{},评论人数：{}， 简介：{}'.format(name, movie\_url, movie\_jpg, director1, star, time\[-4:\], point, num, abstract )

    with open('douban.txt', 'a', encoding='utf-8') as f:  
        f.write(data + '\\n')

for i in range(0, 226, 25):  
    url = f'https://movie.douban.com/top250?start={i}&filter='  
    get\_every\_page(url)