Node.js中理解asyncmap函数 ,爬取王者荣耀荣耀官网壁纸400多张
阅读原文时间:2023年07月08日阅读:3

async/mapLimit函数理解

const phantom = require('phantom')
const express = require('express');
const app = express();
const axios = require('axios');
const isPlainObject = require('lodash/isPlainObject');
const qs = require('qs')
const Cookies = require('js-cookie')
const fs = require('fs');
const cheerio = require('cheerio');
const request = require('request');

//express服务器,
let server = app.listen(2000, function () {
let host = server.address().address;
let port = server.address().port;
console.log('Your App is running at http://%s:%s', host, port);
});

//封装axios
const http = axios.create({
timeout: 1000 * 180,
withCredentials: true
})

/**
* 请求拦截
*/
http.interceptors.request.use(config => {
config.headers['Accept-Language'] = Cookies.get('language') || 'zh-CN'
// console.log('token',Cookies.get('token'))
// config.headers['Authorization'] = Cookies.get('token') || '' //加token
// if(config.headers['Authorization'] == '') {
// clearLoginInfo()
// router.replace({ name: 'login' })
// }
// 默认参数
var defaults = {}
// 防止缓存,GET请求默认带_t参数
if (config.method === 'get') {
config.params = {
…config.params,
…{'_t': new Date().getTime()}
}
}
if (isPlainObject(config.params)) {
config.params = {
…defaults,
…config.params
}
}
if (isPlainObject(config.data)) {
config.data = {
…defaults,
…config.data
}
if (/^application\/x-www-form-urlencoded/.test(config.headers['content-type'])) {
config.data = qs.stringify(config.data)
}
}
return config
}, error => {
return Promise.reject(error)
})

/**
* 响应拦截
*/
http.interceptors.response.use(response => {
return response;
}, error => {
console.error(error)
return Promise.reject(error)
})

class stealData {

constructor() {  
    this.base\_url = 'https://pvp.qq.com/web201605/wallpaper.shtml#%23%23'//爬取网址  
    this.listAll = \[\]  
}

async init() {  
    try {  
        await this.geData()//打开网页  
        // await this.getUrl()//打开网页  
    } catch (e) {  
        console.log(e);  
    }  
}

async geData() {  
    await this.getUrl(0)  
}

async getUrl(i) {  
    console.log(i)  
    let list = \[\]  
    let params = {  
        page: i  
    }  
    http.get('https://apps.game.qq.com/cgi-bin/ams/module/ishow/V1.0/query/workList\_inc.cgi?activityId=2735&sVerifyCode=ABCD&sDataType=JSON&iListNum=20&totalpage=0&iOrder=0&iSortNumClose=1&jsoncallback=jQuery17107547180061056307\_1599185179163&iAMSActivityId=51991&\_everyRead=true&iTypeId=2&iFlowId=267733&iActId=2735&iModuleId=2735&\_=', {params: params}).then(res => {  
        let a = JSON.parse(res.data.match(/({(\\S\*)})/)\[1\])  
        this.down(a.List,i)

    })

}  
sleep(time) {  
    return new Promise((resolve) => {  
        console.log(\`自动睡眠中,${time / 1000}秒后重新发送请求......\`)  
        setTimeout(() => {  
            resolve();  
        }, time);  
    });  
}//请求延迟时间,防止ip被封  
async down(src,index) {

    for (let i = 0;i<src.length;i++){  
        let url = decodeURIComponent(src\[i\].sProdImgNo\_7).replace("/200","/0")  
        console.log('url',url)  
        let ext = url.split('.').pop().substr(0,3)  
        try {  
            console.log(\`开始写入第${i+index\*20+1}张\`)  
            await request(url).pipe(fs.createWriteStream(\`./Wangzhe/${new Date().getTime()}.${ext}\`));  
            await this.sleep(3000)//防止被封  
            console.log(\`写入成功\`)

        }catch (e) {  
            console.log('下载出错:',e)  
        }  
    }  
    if(index<22){  
        await this.getUrl(++index)  
    }

}

}

const thief = new stealData('xxx_url');
thief.init();