const phantom = require('phantom')
const express = require('express');
const app = express();
const axios = require('axios');
const isPlainObject = require('lodash/isPlainObject');
const qs = require('qs')
const Cookies = require('js-cookie')
const fs = require('fs');
const cheerio = require('cheerio');
const request = require('request');
//express服务器,
let server = app.listen(2000, function () {
let host = server.address().address;
let port = server.address().port;
console.log('Your App is running at http://%s:%s', host, port);
});
//封装axios
const http = axios.create({
timeout: 1000 * 180,
withCredentials: true
})
/**
* 请求拦截
*/
http.interceptors.request.use(config => {
config.headers['Accept-Language'] = Cookies.get('language') || 'zh-CN'
// console.log('token',Cookies.get('token'))
// config.headers['Authorization'] = Cookies.get('token') || '' //加token
// if(config.headers['Authorization'] == '') {
// clearLoginInfo()
// router.replace({ name: 'login' })
// }
// 默认参数
var defaults = {}
// 防止缓存,GET请求默认带_t参数
if (config.method === 'get') {
config.params = {
…config.params,
…{'_t': new Date().getTime()}
}
}
if (isPlainObject(config.params)) {
config.params = {
…defaults,
…config.params
}
}
if (isPlainObject(config.data)) {
config.data = {
…defaults,
…config.data
}
if (/^application\/x-www-form-urlencoded/.test(config.headers['content-type'])) {
config.data = qs.stringify(config.data)
}
}
return config
}, error => {
return Promise.reject(error)
})
/**
* 响应拦截
*/
http.interceptors.response.use(response => {
return response;
}, error => {
console.error(error)
return Promise.reject(error)
})
class stealData {
constructor() {
this.base\_url = 'https://pvp.qq.com/web201605/wallpaper.shtml#%23%23'//爬取网址
this.listAll = \[\]
}
async init() {
try {
await this.geData()//打开网页
// await this.getUrl()//打开网页
} catch (e) {
console.log(e);
}
}
async geData() {
await this.getUrl(0)
}
async getUrl(i) {
console.log(i)
let list = \[\]
let params = {
page: i
}
http.get('https://apps.game.qq.com/cgi-bin/ams/module/ishow/V1.0/query/workList\_inc.cgi?activityId=2735&sVerifyCode=ABCD&sDataType=JSON&iListNum=20&totalpage=0&iOrder=0&iSortNumClose=1&jsoncallback=jQuery17107547180061056307\_1599185179163&iAMSActivityId=51991&\_everyRead=true&iTypeId=2&iFlowId=267733&iActId=2735&iModuleId=2735&\_=', {params: params}).then(res => {
let a = JSON.parse(res.data.match(/({(\\S\*)})/)\[1\])
this.down(a.List,i)
})
}
sleep(time) {
return new Promise((resolve) => {
console.log(\`自动睡眠中,${time / 1000}秒后重新发送请求......\`)
setTimeout(() => {
resolve();
}, time);
});
}//请求延迟时间,防止ip被封
async down(src,index) {
for (let i = 0;i<src.length;i++){
let url = decodeURIComponent(src\[i\].sProdImgNo\_7).replace("/200","/0")
console.log('url',url)
let ext = url.split('.').pop().substr(0,3)
try {
console.log(\`开始写入第${i+index\*20+1}张\`)
await request(url).pipe(fs.createWriteStream(\`./Wangzhe/${new Date().getTime()}.${ext}\`));
await this.sleep(3000)//防止被封
console.log(\`写入成功\`)
}catch (e) {
console.log('下载出错:',e)
}
}
if(index<22){
await this.getUrl(++index)
}
}
}
const thief = new stealData('xxx_url');
thief.init();
手机扫一扫
移动阅读更方便
你可能感兴趣的文章