记一次失败的爬虫

package com.fun

import com.fun.frame.Save
import com.fun.frame.httpclient.FanLibrary
import com.fun.utils.Regex
import com.fun.utils.WriteRead
import net.sf.json.JSONObject
import org.apache.http.client.methods.HttpGet
import org.apache.http.client.methods.HttpPost

class sd extends FanLibrary {

    static names = []

    public static void main(String[] args) {

        test2(1)
//        52.times {
//            test(it+1)
//        }
//        Save.saveStringList(names,"hospital")

        def line = WriteRead.readTxtFileByLine(LONG_Path + "hospital")
        line.each {
            def s = it.split(" ", 3)[1]
            output(s)
            it += test2(changeStringToInt(s))
            names << it
        }
        Save.saveStringList(names,"address.txt")

        testOver()
    }

    static def test(int i) {
        String url = "http://ybj.***.gov.cn/ddyy/ddyy2/list";

        HttpPost httpPost = getHttpPost(url, getJson("page=" + i));

        JSONObject response = getHttpResponse(httpPost);
        def all = response.getString("content").replaceAll("\\s", EMPTY)

        def infos = Regex.regexAll(all, "<tr>.*?</tr>")
        infos.remove(0)
        output(infos)
        output(infos.size())
        infos.each {x ->
            names << x.substring(4).replaceAll("<.*?>", SPACE_1)
        }
        output(names.size())
    }
    static def test2(int i) {
        String url = "http://ybj.***.gov.cn/ddyy/ddyy2/findByName?id="+i;

        HttpGet httpGet = getHttpGet(url);

        JSONObject response = getHttpResponse(httpGet);
        def all = response.getString("content").replaceAll("\\s", EMPTY)

        def infos = Regex.regexAll(all, "<tr>.*?</tr>")
//        infos.remove(0)
        output(infos)
//        output(infos.size())
        def address = EMPTY
        infos.each {x ->
//            names << x.substring(4).replaceAll("<.*?>", SPACE_1)
           address += x.substring(4).replaceAll("<.*?>", SPACE_1)
        }
        output(address)
        return address
    }
}