自动化网页数据抓取中的动态交互与分页处理：以地区分页数据为例

const reg = ['河北省', '天津市', '钓鱼岛']; // 定义地区数组
let moreButtonClicked = false; // 标记“更多”按钮是否已经点击过

function getNextReg(currentRegion) {
    const currentIndex = reg.indexOf(currentRegion);
    return currentIndex < reg.length - 1 ? reg[currentIndex + 1] : null;
}

async function getPage(page, region) {
    console.log(`开始处理地区: ${region}`);

    if (!moreButtonClicked) {
        await waitForElement('div.read-more__toggle__text');
        const moreButtonContainers = document.querySelectorAll('div.read-more__toggle__text');
        for (const div of moreButtonContainers) {
            const span = div.querySelector('span.left-text');
            if (span && span.innerText.trim() === "更多") {
                span.click();
                console.log('已点击“更多”按钮');
                moreButtonClicked = true;
                await new Promise(resolve => setTimeout(resolve, 1000)); // 确保点击生效
                break;
            }
        }
    }

    // 等待页面元素加载
    await waitForElement('input[type="radio"]');
    await waitForElement('.checkbox-content-text .checkbox-name');
    await waitForElement('.card_box');

    // 设置为第二个单选项
    const radioOptions = document.querySelectorAll('input[type="radio"]');
    if (radioOptions.length >= 2 && !radioOptions[1].checked) {
        radioOptions[1].checked = true;
        radioOptions[1].dispatchEvent(new Event('change', { bubbles: true }));
    }

    // 选择地区
    let labelElement = Array.from(document.querySelectorAll('.checkbox-content-text .checkbox-name'))
        .find(el => el.textContent.includes(region));
    if (labelElement) {
        const checkbox = labelElement.closest('.cascader-menu-item').querySelector('input[type="checkbox"]');
        if (!checkbox.closest('.el-checkbox__input').classList.contains('is-checked')) {
            checkbox.click();
            console.log(`${region} 已被选中`);
            await new Promise(resolve => setTimeout(resolve, 1000));
        }
    }

    const search_btn = document.querySelector('.search-btn');
    if (search_btn) {
        search_btn.click();
        console.log('点击搜索按钮');
        await new Promise(resolve => setTimeout(resolve, 1000));
    }

    await retrieveCardData(page);

    const clearButton = document.querySelector('.filter-clear');
    if (clearButton) {
        clearButton.click();
        console.log('已点击清空按钮，重置筛选条件');
        await new Promise(resolve => setTimeout(resolve, 1000));
    }

    const nextRegion = getNextReg(region);
    if (nextRegion) {
        console.log(`准备处理下一个地区: ${nextRegion}`);
        await getPage(1, nextRegion); 
    }
}

async function retrieveCardData(page) {
    while (true) {
        if (page >= 20) break;

        console.log(`当前是第 ${page} 页`);
        await waitForElement('.card_box');

        const cardItems = document.querySelectorAll('.card_box');
        const cardDataList = Array.from(cardItems).map(card => ({
            招采单位: card.querySelector('.field-item .company-name .content-wrap')?.innerText.trim() || '未找到',
            标的物: card.querySelector('.subject-matter-list-item')?.innerText.trim() || '未找到',
            发布时间: card.querySelector('.properties .value')?.innerText.trim() || '未找到',
            联系方式: card.querySelector('.field-item.field-card .value')?.innerText.trim() || '未找到'
        }));
        console.log(`当前页面的卡片数量: ${cardItems.length}`);
        console.log(cardDataList);

        const pageButtons = document.querySelectorAll('.el-pager .number');
        let needClick = Array.from(pageButtons).find(button => button.textContent.trim() === `${page + 1}`);
        if (needClick) {
            needClick.click();
            console.log(`切换到第 ${needClick.textContent.trim()} 页`);
            page += 1;
            await new Promise(resolve => setTimeout(resolve, 1000)); // 等待翻页完成
        } else {
            break;
        }
    }
}

async function waitForElement(selector, timeout = 5000) {
    const startTime = Date.now();
    return new Promise((resolve, reject) => {
        const interval = setInterval(() => {
            if (document.querySelector(selector)) {
                clearInterval(interval);
                resolve();
            } else if (Date.now() - startTime > timeout) {
                clearInterval(interval);
                reject(`等待元素 ${selector} 超时`);
            }
        }, 500);
    });
}