腾讯云TCB云函数抓取微信订阅号话题标签文章

薛定喵君

发布于 2020-11-23 12:52:13

1.8K0

发布于 2020-11-23 12:52:13

文章被收录于专栏：薛定喵君

本文记录如何使用云函数实现微信公众号话题标签文章列表的抓取

# 实验准备

获取订阅号话题标签链接

# 核心代码

需求是获取文章的标题、跳转链接、发布时间及文章配图的缩略图就可以了，具体代码如下。

'use strict';
const puppeteer = require('puppeteer')
exports.main = async (event, context) => {
    const browser = await puppeteer.launch({
        headless: true,
        args: ['--no-sandbox', '--disable-setuid-sandbox'],
        dumpio: false,
    })
    const page = await browser.newPage()
    page.setUserAgent(
        'Mozilla/5.0 (Linux; Android 10; Redmi K30 Pro Build/QKQ1.191117.002; wv) AppleWebKit/537.36 (KHTML, like Gecko) Version/4.0 Chrome/78.0.3904.62 XWEB/2581 MMWEBSDK/200801 Mobile Safari/537.36'
    )
    await page.goto('获取到的话题标签链接', {
        waitUntil: 'networkidle0',
    })

    const articleInfo = await page.evaluate(() => {
        const element = document.querySelector(".album__list");
        let items = element.getElementsByClassName("album__list-item");
        let title = [];
        for (let i = 0; i < items.length; i++) {
            let cover = items[i]
                .getElementsByClassName("album__item-img")
                .item(0)
                .getAttribute("style").replace("background-image: url(", "");
            cover = cover.substring(0, cover.length - 2);
            title.push({
                title: items[i].getAttribute("data-title"),
                url: items[i].getAttribute("data-link"),
                date: items[i].getElementsByClassName("album__item-info-item").item(0)
                    .innerHTML,
                pic: cover,
            });
        }

        return title;
    });
    await browser.close();
    return articleInfo
};