node_puppeteer无界爬虫
阅读原文时间:2023年07月09日阅读:2

环境:node----v14.5.0

vscode----2019

依赖库

   (需要自行设置好目录结构,否则会报目录错误)

const puppeteer = require("puppeteer")
const axios = require("axios")
const fs = require("fs")
const url = require("url")
const path = require("path")
//配置无界浏览器的启动参数
let options = {
defaultViewport: {
width: 1536,
height: 824
},
headless: false,
ignoreDefaultArgs: "--enable-automation",
// ignoreHTTPSErrors: true, // 忽略证书错误
// devtools: true, // 开发者控制台
}
//删除目录文件(更新)
function delDir(path) {
let files = [];
if (fs.existsSync(path)) {
files = fs.readdirSync(path);
files.forEach((file, index) => {
let curPath = path + "/" + file;
if (fs.statSync(curPath).isDirectory()) {
delDir(curPath); //递归删除文件夹
console.log("删除文件夹成功")
} else {
fs.unlinkSync(curPath); //删除文件
console.log("删除文件成功")
}
});
fs.rmdirSync(path);
}

}
//下载作用
async function downMusic(eles) {
let res = await axios.get(eles.src, { responseType: 'stream' });
let artistIcon = path.extname(eles.src)
let ws = fs.createWriteStream("./music/" + eles.alt + "/" + eles.alt + artistIcon, { flags: 'w' });
res.data.pipe(ws)
res.data.on("close", function() {
console.log("下载完成!")
ws.close()
})

}

//主函数
async function openChrome() {
delDir("./music")
fs.mkdir("./music", (err) => { if (err) { console.log(err) } else { console.log("创建成功:", "./music") } })
let browser = await puppeteer.launch(options)
let page = await browser.newPage()
page.on('requestfailed', function() {
console.log("连接超时")
})
page.on('console', function(args) {
console.log(args._text)
})
await page.goto("https://music.migu.cn/v3/music/artist?tagId=1&type=A&firstLetter=1&page=1")
let eles = await page.$$eval(".thumb-img.lazy-image", (elements) => {
let eles = []
elements.forEach(function(item, i) {
let info = {
alt: item.alt,
src: item.src
}
eles.push(info)
})

    return eles;

})  
console.log(eles)  
eles.forEach((item, i) => {  
    fs.mkdir("./music/" + item.alt, function(err) {  
        if (err) { console.log(err) } else {

            downMusic(item)

            fs.mkdir("./music/" + item.alt + "/song", (err) => { if (err) { console.log(err) } else { console.log("创建成功:", "./music/" + item.alt + "/song") } })  
        }  
    })  
})  
let song\_list = \[\]  
let artist\_a = await page.$$eval(".thumb-link", (elements) => {  
    elements.forEach((element) => {  
        console.log(element.href)

    })  
})

}
openChrome()

效果图如下: