爬取北京摩拜单车信息（附分析过程和详细代码）

d＝111.12cos{1/[sinΦAsinΦB十cosΦAcosΦBcos(λB—λA)]}

@Test
public void testGetMoBike() throws IOException {
    OkHttpClient client = new OkHttpClient();
    RequestBody body = RequestBody.create(GSON, "{}");

    double longitudeMin = 116.2875;
    double longitudeMax = 116.4787;
    double latitudeMin = 39.8402;
    double latitudeMax = 39.9778;

    double step = 0.003;

    for (double longitude = longitudeMin; longitude <= longitudeMax; longitude += step) {
        for (double latitude = latitudeMin; latitude <= latitudeMax; latitude += step) {

            // 获取对应经纬度的结果
            List<BikeDo> bikeDos = getBikeDos(client, body, longitude, latitude);

            // 将每辆单车信息都存入redis
            restoreBikeInfo(bikeDos);

            // 随机休眠，防止被封
            sleepRandomSeconds(2);

            System.out.println("经度：" + longitude + " 纬度：" + latitude + " 结束，爬取总个数：" + jedis.hlen(MAP_KEY));
        }
    }
}

private static Jedis jedis = RedisUtil.getJedis();
private static final String MAP_KEY = "mobike";
private static final Random RANDOM = new Random();

private void restoreBikeInfo(List<BikeDo> bikeDos) {
    for (BikeDo bikeDo : bikeDos) {
        // 将每一个 bike 存入 redis 中
        if (jedis.hexists(MAP_KEY, bikeDo.getDistId())) {
            continue;
        }
        jedis.hset(MAP_KEY, bikeDo.getDistId(), JSON.toJSONString(bikeDo));
    }
}

private void sleepRandomSeconds(int seconds) {
    try {
        Thread.sleep(RANDOM.nextInt(seconds) * 1000);
    } catch (InterruptedException e) {
        e.printStackTrace();
    }
}

private List<BikeDo> getBikeDos(OkHttpClient client, RequestBody body, double longitude, double latitude) throws IOException {
    Request request = createRequest(body, longitude, latitude);

    Response response = client.newCall(request).execute();
    String responseJson = response.body().string();
    JSONArray parse = (JSONArray) JSONObject.parseObject(responseJson).get("bike");
    return JSON.parseArray(parse.toJSONString(), BikeDo.class);
}

private Request createRequest(RequestBody body, double longitude, double latitude) {
    return new Request.Builder()
            .url("http://app.mobike.com/api/nearby/v3/nearbyBikeInfo?scope=700&sign=&client_id=android&biketype=0&longitude=" + longitude + "&latitude=" + latitude + "&userid=&bikenum=600")
            .post(body)
            .addHeader("version", "6.7.1")
            .addHeader("versioncode", "1649")
            .addHeader("platform", "1")
            .addHeader("mainsource", "4002")
            .addHeader("subsource", "3")
            .addHeader("os", "25")
            .addHeader("lang", "zh")
            .addHeader("time", DateTime.now().getMillis() + "")
            .addHeader("country", "0")
            .addHeader("eption", "685f8")
            .addHeader("deviceresolution", "1080X1920")
            .addHeader("utctime", DateTime.now().getMillis() / 1000 + "")
            .addHeader("mobileno", "")
            .addHeader("accesstoken", "")
            .addHeader("uuid", "")
            .addHeader("longitude", longitude + "")
            .addHeader("latitude", latitude + "")
            .addHeader("content-type", "application/x-www-form-urlencoded")
            .addHeader("content-length", "203")
            .addHeader("host", "app.mobike.com")
            .addHeader("accept-encoding", "gzip")
            .addHeader("user-agent", "okhttp/3.9.1")
            .addHeader("connection", "Keep-Alive")
            .addHeader("cache-control", "no-cache")
            .addHeader("postman-token", "fc17ae5a-32e9-d2bb-eb1f-43dd6760aea6")
            .build();
}

public class BikeDo {
    private String distId;
    private double distX;
    private double distY;
    private int distNum;
    private int distance;
    private int biketype;
    private int type;

@Test
public void testMobikeHeat() {
    String REDIS_KEY = "mobike";
    Map<String, String> bikesMap = jedis.hgetAll(REDIS_KEY);
    Multiset<String> multiset = HashMultiset.create();
    for (String key : bikesMap.keySet()) {
        BikeDo bike = JSON.parseObject(bikesMap.get(key), BikeDo.class);
        String mixKey = String.format("%.6f", bike.getDistX()) + String.format("%.6f", bike.getDistY());
        multiset.add(mixKey);
    }
    
    File file = new File("mobike.txt");
    BufferedWriter out = null;
    try {
        out = new BufferedWriter(new OutputStreamWriter(new FileOutputStream(file, true)));
        for (String ele : multiset.elementSet()) {
            out.write("{\"lng\":"+ele.substring(0, 10)+",\"lat\":"+ele.substring(10)+",\"count\":"+multiset.count(ele)+"}," + "\r\n");
        }
        
    } catch (Exception e) {
        e.printStackTrace();
    } finally {
        try {
            out.close();
        } catch (IOException e) {
            e.printStackTrace();
        }
    }
}