js 页面抓取

const axios = require('axios');
const cheerio = require('cheerio');

async function scrapePage(url) {
  try {
    const response = await axios.get(url);
    const html = response.data;
    const $ = cheerio.load(html);

    // 提取标题
    const title = $('title').text();
    console.log('Title:', title);

    // 提取所有链接
    $('a').each((index, element) => {
      console.log($(element).attr('href'));
    });
  } catch (error) {
    console.error('Error fetching the page:', error);
  }
}

scrapePage('https://example.com');

const puppeteer = require('puppeteer');

async function scrapeDynamicPage(url) {
  const browser = await puppeteer.launch();
  const page = await browser.newPage();
  await page.goto(url);

  // 等待页面加载完成
  await page.waitForSelector('#dynamic-content');

  // 提取动态内容
  const content = await page.$eval('#dynamic-content', el => el.innerHTML);
  console.log('Dynamic Content:', content);

  await browser.close();
}

scrapeDynamicPage('https://example.com');