如何使用axios和cheerio实现多页抓取？

npm install axios cheerio

const axios = require('axios');
const cheerio = require('cheerio');

async function getPage(url) {
  try {
    const response = await axios.get(url);
    return response.data;
  } catch (error) {
    console.error(`Failed to fetch page: ${url}`, error);
    return null;
  }
}

function parsePage(html) {
  const $ = cheerio.load(html);
  
  // 根据HTML结构和数据定位元素，并提取数据
  const title = $('h1').text();
  const content = $('#content').text();
  
  return { title, content };
}

async function scrapePages() {
  const urls = ['https://example.com/page1', 'https://example.com/page2', 'https://example.com/page3'];

  for (const url of urls) {
    const html = await getPage(url);
    
    if (html) {
      const data = parsePage(html);
      console.log(data);
    }
  }
}

scrapePages();