Java爬虫（3）——拼接url抓取“加载更多”内容

package edu.nju.opsource.vnexpress.linktype;
import java.io.InputStream;
import java.util.Date;
import java.util.LinkedList;
import org.jsoup.Jsoup;
import org.jsoup.nodes.Document;
import org.jsoup.nodes.Element;
import org.jsoup.select.Elements;
import com.onezetta.downloader.EasyHttpDownloader;
import edu.nju.opensource.common.beans.ELinkState;
import edu.nju.opensource.common.beans.Link;
import edu.nju.opensource.common.beans.LinkType;
import edu.nju.opensource.common.beans.Site;
import edu.nju.opsource.vnexpress.utils.IdXmlUtil;
public class LinkType_NewsList extends LinkType {
private Document doc = null;
private Site site = null;
private LinkedList<Link> newItems = null;
private Link nextPage = null;
public LinkType_NewsList() {
super();
// TODO Auto-generated constructor stub
}
public LinkType_NewsList(String linkTypeName) {
super.setLinkTypeName(linkTypeName);
}
public LinkType_NewsList(Site site) {
super.setLinkTypeName("vnexpress.newslist");
this.site = site;
}
public void get(String url) {
super.get(url);
String content = new EasyHttpDownloader(url).run();
if (content != null) {
this.doc = Jsoup.parse(content, this.site.getUrl().getUrl());
System.out.println(" ... has Crawled.");
} else {
setState(ELinkState.CRAWLFAILED);
System.out.println(" ... crawled failed.");
}
}
// 把新闻列表条目的链接插入表
// 在总项目中被调用的核心函数handle()
@Override
public boolean handle(Link parentlink) throws Exception {
if (getState() == ELinkState.CRAWLFAILED)
return false;
Elements news = this.doc.select("div.list_news_folder.col_720 h4.title_news_site");
this.newItems = new LinkedList();
boolean flag = false;
for (Element newItem : news) {
Elements tmp = newItem.select("a");
if ((tmp != null) && (tmp.size() != 0)) {
Link link = new Link(tmp.first().attr("abs:href"), new LinkType_News(), ELinkState.UNCRAWL, parentlink,
this.site).setLinkDate( new java.sql.Date(new Date().getTime()));
// System.out.println(tmp.first().attr("abs:href"));
int rst = link.insert();
if (rst == -1)
flag = true;// link exist
}
}
if (flag) {
setState(ELinkState.CRAWLED);
return false;
}
Elements nextPageE = this.doc.select("a#vnexpress_folder_load_more");
           String url = getNextPageUrl();
if ((nextPageE != null) && (nextPageE.size() != 0)) {
System.out.println(url);
this.nextPage = new Link(url, new LinkType_NewsList(this.site), ELinkState.UNCRAWL, parentlink, this.site).setLinkDate( new java.sql.Date(new Date().getTime()));
this.nextPage.insert();
}
else {
IdXmlUtil.setIdByName("news", 2 + "");
}
setState(ELinkState.CRAWLED);
return true;
}
public static String getNextPageUrl() {
int id = IdXmlUtil.getIdByName("news");
// IdXmlUtil用来存储当前已抓取的pageid其实这个工具完全没必要，因为数据库的//url有此参数啊。。。当时脑抽没想到，搞麻烦了。。。
IdXmlUtil.setIdByName("news", (id + 1) + "");
//此文章的精髓
String url = "http://e.vnexpress.net/news/news?cate_id=1003894&page=" + id;
return url;
}
@Override
public String getLinkTextData() {
// TODO Auto-generated method stub
return null;
}
@Override
public InputStream getLinkBinaryData() {
// TODO Auto-generated method stub
return null;
}
@Override
public String getLinkUniqeID(String paramString) {
// TODO Auto-generated method stub
return null;
}
@Override
public Date getLinkDate() {
// TODO Auto-generated method stub
return null;
}
public Site getSite() {
return this.site;
}
@Override
public void setSite(Site site) {
// TODO Auto-generated method stub
this.site = site;
}
public LinkedList<Link> getNewItems() {
return this.newItems;
}
public void setNewItems(LinkedList<Link> newItems) {
this.newItems = newItems;
}
public Link getNextPage() {
return this.nextPage;
}
public void setNextPage(Link nextPage) {
this.nextPage = nextPage;
}
@Override
public String toString() {
return "LinkType_newsList [doc=" + doc + ", site=" + site + ", newItems=" + newItems + ", nextPage="
+ nextPage + "]";
}
}