HttpClient-代理IP、模拟浏览器

wangmcn

发布于 2022-07-25 17:13:29

6630

发布于 2022-07-25 17:13:29

代理IP、模拟浏览器

package com.test.demo;

import org.apache.http.client.methods.HttpGet;

import java.io.IOException;

import org.apache.http.HttpEntity;
import org.apache.http.HttpHost;
import org.apache.http.client.ClientProtocolException;
import org.apache.http.client.config.RequestConfig;
import org.apache.http.client.methods.CloseableHttpResponse;
import org.apache.http.impl.client.CloseableHttpClient;
import org.apache.http.impl.client.HttpClients;
import org.apache.http.util.EntityUtils;

/**
 * 代理IP
 *
 * @author wangmcn
 *
 */
public class Proxy {

       public static void main(String[] args) throws ClientProtocolException, IOException {

              // 创建CloseableHttpClient对象
              CloseableHttpClient httpclient = HttpClients.createDefault();
              // 创建HttpGet对象
              HttpGet httpget = new HttpGet("https://www.baidu.com/");

              // 创建HttpHost对象，设置代理IP
              HttpHost proxy = new HttpHost("114.113.126.82", 80);
              RequestConfig requestConfig = RequestConfig.custom()
                            .setProxy(proxy)
                            .build();
              httpget.setConfig(requestConfig);

              // 执行Get请求
              CloseableHttpResponse response = httpclient.execute(httpget);
              // 获取响应状态
              System.out.println("获取响应状态: " + response.getStatusLine().getStatusCode());
              // 获取返回实体
              HttpEntity entity = response.getEntity();
              // 获取网页源码
              System.out.println("获取网页源码：" + EntityUtils.toString(entity, "utf-8"));
              // 获取响应内容类型
              System.out.println("获取响应内容类型：" + entity.getContentType().getValue());
              // 关闭流和释放系统资源
              response.close();
              // 关闭客户端
              httpclient.close();

       }

}

2、运行结果：

2、模拟浏览器

模拟浏览器操作需要在Header里添加User-Agent参数与参数值，那么这个参数值怎么获取呢？

如图所示：

打开Fiddler，之后打开浏览器随便访问一个地址（如https://www.baidu.com/），此时Fiddler抓取请求头信息里就可以看到User-Agent的参数与参数值，将此参数与参数值添加到脚本的Header里即可。

1、创建Browser类。

模拟浏览器setHeader("User-Agent", "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/68.0.3440.106 Safari/537.36")

脚本代码：

package com.test.demo;

import java.io.IOException;

import org.apache.http.Header;
import org.apache.http.HttpEntity;
import org.apache.http.client.ClientProtocolException;
import org.apache.http.client.methods.CloseableHttpResponse;
import org.apache.http.client.methods.HttpGet;
import org.apache.http.impl.client.CloseableHttpClient;
import org.apache.http.impl.client.HttpClients;
import org.apache.http.util.EntityUtils;

/**
 * 模拟浏览器
 *
 * @author wangmcn
 *
 */
public class Browser {

       public static void main(String[] args) throws ClientProtocolException, IOException {
             
              // 创建CloseableHttpClient对象
              CloseableHttpClient httpclient = HttpClients.createDefault();
              // 创建HttpGet对象
              HttpGet httpget = new HttpGet("http://localhost:8083/getdemo");
             
              // 设置请求头信息
              httpget.setHeader("User-Agent", "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/68.0.3440.106 Safari/537.36");
             
              // 执行Get请求
              CloseableHttpResponse response = httpclient.execute(httpget);
              // 获取响应状态
              System.out.println("获取响应状态: " + response.getStatusLine().getStatusCode());
              // 获取返回实体
              HttpEntity entity = response.getEntity();
              // 获取网页源码
              System.out.println("获取网页源码：" + EntityUtils.toString(entity, "utf-8"));
              // 获取响应内容类型
              System.out.println("获取响应内容类型：" + entity.getContentType().getValue());
             
              // 获取请求头
              Header requestHeader[] = httpget.getAllHeaders();
              for (Header header : requestHeader) {
                     System.out.println("请求头: " + header.getName() + ": " + header.getValue());
              }
              // 获取响应头
              Header responseHeader[] = response.getAllHeaders();
              for (Header header : responseHeader) {
                     System.out.println("响应头: " + header.getName() + ": " + header.getValue());
              }
             
              // 关闭流和释放系统资源
              response.close();
              // 关闭客户端
              httpclient.close();

       }

}

2、运行结果：