基于大数据架构实现景点游客数据分析平台，全国景点游客数据管理系统

  /**
     * 景点数据采集
     */
    @Scheduled(fixedDelay = 1000 * 60 * 1)
    public void jingdianResTask() {
        if (!detailOpen) return;
        Date start = new Date();
        log.info("采集任务开始:{}", start);
        try {
            String querySql = "select id,url from t_jingdian_res where deal=0 limit 1000;";
            List<Map<String, Object>> oplogs = jdbcTemplate.queryForList(querySql);
            if (CollectionUtils.isEmpty(oplogs)) {
                log.info("任务结束，暂无企业责任数据处理");
                return;
            }
            List<List<Map<String, Object>>> lists = ListCF.subList(oplogs, 1000);
            CountDownLatch latch = new CountDownLatch(lists.size());
            for (List<Map<String, Object>> sublist : lists) {
                CompletableFuture.runAsync(() -> {
                    sublist.forEach(task -> {
                        Integer id = Integer.valueOf(String.valueOf(task.get("id")));
                        String url = String.valueOf(task.get("url"));
                        try {
                            Document doc = null;
                            for (int p = 0; p < 3; p++) {
                                try {
                                    doc = Jsoup.connect(url).ignoreContentType(true).ignoreHttpErrors(true).timeout(5000).get();
                                    if (null != doc) break;
                                } catch (Exception e) {
                                    e.printStackTrace();
                                    log.error("地址：{}，打开失败，重试次数：{}", url, p);
                                }
                            }
                            if (null == doc) return;
                            String gdzr_mx = doc.select("body > div.wrap > div.article.w680 > div:nth-child(2)").text();
                            String ygzr_mx = doc.select("body > div.wrap > div.article.w680 > div:nth-child(3)").text();
                            String gkzzr_mx = doc.select("body > div.wrap > div.article.w680 > div:nth-child(4)").text();
                            String hjzr_mx = doc.select("body > div.wrap > div.article.w680 > div:nth-child(5)").text();
                            String shzr_mx = doc.select("body > div.wrap > div.article.w680 > div:nth-child(6)").text();
                            String shzr_img = doc.select("body > div.wrap > div.article.w680 > div.chart > div > img").attr("abs:src");
                            if (StringUtils.isEmpty(gdzr_mx) && StringUtils.isEmpty(ygzr_mx) && StringUtils.isEmpty(gkzzr_mx) && StringUtils.isEmpty(hjzr_mx) && StringUtils.isEmpty(shzr_mx) && StringUtils.isEmpty(shzr_img))
                                return;
                            String sql = "UPDATE `yk_data_view`.`t_jingdian_res` SET `gdzr_mx`=?, `ygzr_mx`=?, `gkxzr_mx`=?, `hjzr_mx`=?, `shzr_mx`=?, `shzr_img`=?, `deal`='1',`update_time`=NOW() WHERE `id`=?;";
                            jdbcTemplate.update(sql, gdzr_mx, ygzr_mx, gkzzr_mx, hjzr_mx, shzr_mx, shzr_img, id);
                            log.info("地址：{}，爬取成功", url);
                        } catch (Exception e) {
                            e.printStackTrace();
                            log.error("地址：{}，爬取失败", url);
                        }
                    });
                    latch.countDown();
                }, executorPool);
            }
            try {
                latch.await();
            } catch (InterruptedException e) {
                e.printStackTrace();
            }
            Date end = new Date();
            log.info("任务结束，耗时：{}ms，数量：{}", (end.getTime() - start.getTime()), oplogs.size());
        } catch (Exception e) {
            e.printStackTrace();
            log.error("任务错误，错误信息：{}", e.getCause());
        }
    }