最新puppeteer爬虫boss直聘招聘公司及职位信息

// 城市与城市编码的转换
function main(city, job){

  // 汉字 转 拼音
  // var pinyin = require('fast-pinyin');
  // var output = pinyin(city);
  // var firstChar = output[0][0]
  // console.log("城市：" + output[0][0]);


  const originData = require('./city.json');

  let city_code = null

  if (city == "全国")
    { 
      city_code = "100010000";
    }
  else
    for (var province_info of originData){ 
        var subLevelModelList = province_info["subLevelModelList"]
        for (var city_info of subLevelModelList){ 
          if (city_info["name"] == city)
            city_code = city_info["code"];
          }
  }

  console.log("找到城市代码：" + city_code);
  get_company(city_code, job)

}

var next_page;
async function get_company(city_code, key_words){
    console.log('crawler start to visit the target address');

    /* 爬虫的目标链接地址: boss*/
    var url = `https://www.zhipin.com/c${city_code}/?query=${key_words}&page=1&ka=page-1`;

  /* dumpio 是否将浏览器进程stdout和stderr导入到process.stdout和process.stderr中 */
  const browser = await puppeteer.launch({
    // headless:false,
    args: ['--no-sandbox'],
    dumpio: false,
    // args:['--proxy-server=http://47.98.154.206:3008']

  });
  const page = await browser.newPage();
  await page.evaluateOnNewDocument(() => {
         Object.defineProperty(navigator, 'webdriver', {
         get: () => undefined,
    });
 });

 // 该地区 该类别的所有公司
 var company_info = []

 while (url != null && url.length > 20){

    await page.goto(url, {
      waitUntil: 'networkidle2'
    });
    await sleep(0);

    const result = await page.evaluate(() => {
      let data = []; 
      let elements = document.querySelectorAll('#main > div > div.job-list > ul > li'); //获取所有的li 
      for (var element of elements){ // 循环 
          let title = element.querySelector('div > div.info-primary > div.info-company > div > h3 > a').innerHTML; 
          let url = element.querySelector('div > div.info-primary > div.info-company > div > h3 > a').href; //抓取链接（href）属性 

          data.push({title, url}); // 存入数组
      }

      return data;
      });

      for (var temp of result){ // 循环 
        company_info.push(temp); // 存入数组
      }

    //下一页
    let next_page;

    if (result.length < 30) 
      next_page = null;
    else
      var element_next_page = await page.$("#main > div > div.job-list > div.page > a.next");
      if (element_next_page)
        next_page = await page.$eval('#main > div > div.job-list > div.page > a.next', ele=>ele.href);
      else
        next_page = null

    url = next_page
 }

//  page.close();

// 获取搜索到的所有企业信息
  // 数组去重
  let new_company_info = deduplication(company_info)

company_all_job = company_url_value.replace("gongsi", "gongsir")
  company_all_job = company_all_job.split("?")[0] + "?ka=company-jobs"

  var company_job_urls = []

  while ( company_all_job != null && company_all_job.length > 19){

    await page2.goto(company_all_job, {
      waitUntil: 'networkidle2'
    });
    await sleep(0);

    job_count = await page2.$eval('#main > div.company-banner > div > div.company-tab > a.cur', ele=>ele.innerText);

    if (job_count.includes("(0)"))
     {
      console.log("在招职位为0")
      company_all_job = null
    }
    else
    {
      const jobs_urls = await page2.evaluate(() => {
        let data = []; 
        let elements = document.querySelectorAll('div.job-list >ul >li'); //获取所有的li 
        for (var element of elements){ // 循环 
            let url = element.querySelector('a').href; //抓取链接（href）属性 
            data.push(url); // 存入数组
        }

        return data;
        });

        for (var temp of jobs_urls){ // 循环 
          company_job_urls.push(temp); // 存入数组
        }

      //下一页
      let next_job;
      var job_next_page = null;
      job_next_page = await page2.$("#main > div.job-box.company-job > div.inner.home-inner > div.job-list > div > a.next");
      if (job_next_page)
        next_job = await page2.$eval('#main > div.job-box.company-job > div.inner.home-inner > div.job-list > div > a.next', ele=>ele.href);
      else
        next_job = null

      company_all_job = next_job
    }
     }

    console.log("职位信息：\n", company_job_urls)
    console.log("职位信息个数：\n", company_job_urls.length)

    page2.close()```