OkHttp的特性优点及爬虫示例

// 导入所需的库
import com.squareup.okhttp.*;
import org.jsoup.Jsoup;
import org.jsoup.nodes.Document;
import org.jsoup.nodes.Element;
import org.jsoup.select.Elements;

import java.io.IOException;
import java.util.ArrayList;
import java.util.List;
import java.util.concurrent.CountDownLatch;

// 定义一个景点类，用于存储景点的信息
class ScenicSpot {
    private String name; // 景点名称
    private String url; // 景点链接
    private String intro; // 景点简介
    private List<String> comments; // 景点评论

    public ScenicSpot(String name, String url) {
        this.name = name;
        this.url = url;
        this.intro = "";
        this.comments = new ArrayList<>();
    }

    public String getName() {
        return name;
    }

    public String getUrl() {
        return url;
    }

    public String getIntro() {
        return intro;
    }

    public void setIntro(String intro) {
        this.intro = intro;
    }

    public List<String> getComments() {
        return comments;
    }

    public void addComment(String comment) {
        this.comments.add(comment);
    }

    @Override
    public String toString() {
        return "ScenicSpot{" +
                "name='" + name + '\'' +
                ", url='" + url + '\'' +
                ", intro='" + intro + '\'' +
                ", comments=" + comments +
                '}';
    }
}

// 定义一个回调接口，用于异步处理响应
interface CallbackHandler {
    void handle(Response response) throws IOException;
}

// 定义一个爬虫类，用于采集大众点评的景点信息
class DianpingCrawler {
    // 大众点评的网址
    private static final String BASE_URL = "http://www.dianping.com";     
    //亿牛云动态转发代理IP 爬虫加强版 代理服务地址
    private static final String PROXY_HOST = "www.16yun.cn"; 
    //亿牛云动态转发代理IP 爬虫加强版 代理端口号
    private static final int PROXY_PORT = 8080; 
    //亿牛云动态转发代理IP 爬虫加强版 代理用户名
    private static final String PROXY_USERNAME = "16YUN"; 
    //亿牛云动态转发代理IP 爬虫加强版 代理密码
    private static final String PROXY_PASSWORD = "16IP";

    private OkHttpClient client; // OkHttp客户端对象

    public DianpingCrawler() {
        // 创建一个OkHttpClient对象，并配置代理和认证信息
        client = new OkHttpClient();
        Proxy proxy = new Proxy(Proxy.Type.HTTP, new InetSocketAddress(PROXY_HOST, PROXY_PORT));
        Authenticator authenticator = new Authenticator() {
            @Override
            public Request authenticate(Proxy proxy, Response response) throws IOException {
                String credential = Credentials.basic(PROXY_USERNAME, PROXY_PASSWORD);
                return response.request().newBuilder()
                        .header("Proxy-Authorization", credential)
                        .build();
            }

            @Override
            public Request authenticateProxy(Proxy proxy, Response response) throws IOException {
                return null;
            }
        };
        client.setProxy(proxy);
        client.setAuthenticator(authenticator);
    }

    // 发送一个GET请求，并使用回调接口处理响应
    public void get(String url, CallbackHandler callbackHandler) throws IOException {
        Request request = new Request.Builder()
                .url(url)
                .header("User-Agent", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/96.0.4664.110 Safari/537.36")
                .build();
        client.newCall(request).enqueue(new Callback() {
            @Override
            public void onFailure(Request request, IOException e) {
                e.printStackTrace();
            }

            @Override
            public void onResponse(Response response) throws IOException {
                callbackHandler.handle(response);
            }
        });
    }

    // 根据城市和关键词获取景点列表的网址
    public String getScenicListUrl(String city, String keyword) {
        return BASE_URL + "/search/keyword/" + city + "/0_" + keyword;
    }

 // 根据景点列表的网址获取景点对象的列表
    public List<ScenicSpot> getScenicSpots(String scenicListUrl) throws IOException {
        List<ScenicSpot> scenicSpots = new ArrayList<>();
        // 发送一个GET请求，获取景点列表的网页内容
        get(scenicListUrl, response -> {
            // 解析网页内容，提取景点的名称和链接
            Document document = Jsoup.parse(response.body().string());
            Elements elements = document.select(".shop-list .shop-name");
            for (Element element : elements) {
                String name = element.text();
                String url = BASE_URL + element.attr("href");
                // 创建一个景点对象，并添加到列表中
                ScenicSpot scenicSpot = new ScenicSpot(name, url);
                scenicSpots.add(scenicSpot);
            }
        });
        return scenicSpots;
    }

    // 根据景点对象获取景点的简介和评论
    public void getScenicInfo(ScenicSpot scenicSpot) throws IOException {
        // 发送一个GET请求，获取景点的网页内容
        get(scenicSpot.getUrl(), response -> {
            // 解析网页内容，提取景点的简介和评论
            Document document = Jsoup.parse(response.body().string());
            String intro = document.select(".shop-intro").text();
            Elements elements = document.select(".comment-list .content");
            List<String> comments = new ArrayList<>();
            for (Element element : elements) {
                String comment = element.text();
                comments.add(comment);
            }
            // 设置景点对象的简介和评论
            scenicSpot.setIntro(intro);
            scenicSpot.getComments().addAll(comments);
        });
    }

    // 根据城市和关键词获取附近TOP20的景点介绍以及每个景点TOP的评价
    public List<ScenicSpot> getTopScenicSpots(String city, String keyword) throws IOException, InterruptedException {
        // 获取景点列表的网址
        String scenicListUrl = getScenicListUrl(city, keyword);
        // 获取景点对象的列表
        List<ScenicSpot> scenicSpots = getScenicSpots(scenicListUrl);
        // 创建一个倒计时锁，用于等待所有异步请求完成
        CountDownLatch countDownLatch = new CountDownLatch(scenicSpots.size());
        // 遍历每个景点对象，获取其简介和评论，并在完成后减少倒计时锁的计数
        for (ScenicSpot scenicSpot : scenicSpots) {
            getScenicInfo(scenicSpot);
            countDownLatch.countDown();
        }
        // 等待所有异步请求完成
        countDownLatch.await();
        // 返回前20个景点对象的列表
        return scenicSpots.subList(0, 20);
    }

    // 测试方法，打印结果
    public static void main(String[] args) throws IOException, InterruptedException {
        DianpingCrawler crawler = new DianpingCrawler();
        List<ScenicSpot> scenicSpots = crawler.getTopScenicSpots("1", "景点");
        for (ScenicSpot scenicSpot : scenicSpots) {
            System.out.println(scenicSpot);
        }
    }
}