Java微博搜索关键字采集

import java.io.File;
import java.io.FileInputStream;
import java.io.FileNotFoundException;
import java.io.FileOutputStream;
import java.io.IOException;
import java.io.ObjectInputStream;
import java.io.UnsupportedEncodingException;
import java.net.MalformedURLException;
import java.text.SimpleDateFormat;
import java.util.List;
import java.util.Random;
import java.util.concurrent.Callable;

import org.apache.http.client.CookieStore;
import org.apache.log4j.Logger;

import com.gargoylesoftware.htmlunit.BrowserVersion;
import com.gargoylesoftware.htmlunit.FailingHttpStatusCodeException;
import com.gargoylesoftware.htmlunit.WebClient;
import com.gargoylesoftware.htmlunit.html.HtmlPage;
import com.gargoylesoftware.htmlunit.util.Cookie;

public class SinaSearchCrawlerCommand implements Callable<Object> {
private static Logger logger = Logger.getLogger(SinaSearchCrawlerCommand.class);
private static String word="如家";
private static String cookiePath="E:\\学习\\微博爬虫\\cookie\\cookie.file";
private static String outputpath="E:\\学习\\微博爬虫\\";
//public Object call(){
public static void main(String[] args){
try {
word= java.net.URLEncoder.encode(word, "utf-8");
} catch (UnsupportedEncodingException e2) {
// TODO Auto-generated catch block
e2.printStackTrace();
}
WebClient webClient = new WebClient(BrowserVersion.FIREFOX_17);
webClient.getCookieManager().setCookiesEnabled(true);
for(int i=1;i<=100;i++){
System.out.println(cookiePathAppendRandom());
File file = new File(cookiePathAppendRandom());
if (file.exists()) {
FileInputStream fin = null;
try {
fin = new FileInputStream(file);
} catch (FileNotFoundException e1) {
e1.printStackTrace();
}
CookieStore cookieStore = null;
ObjectInputStream in;
try {
in = new ObjectInputStream(fin);
cookieStore = (CookieStore) in.readObject();
in.close();
} catch (IOException e) {
logger.error(e);
} catch (ClassNotFoundException e) {
logger.error(e);
}

List<org.apache.http.cookie.Cookie> l = cookieStore.getCookies();
for (org.apache.http.cookie.Cookie temp : l) {
Cookie cookie = new Cookie(temp.getDomain(), temp.getName(),
temp.getValue(), temp.getPath(), temp.getExpiryDate(),
false);
webClient.getCookieManager().addCookie(cookie);
}
/*HtmlPage page = null;
try {
page = webClient.getPage("http://weibo.cn/search/?tf=5_012");
} catch (FailingHttpStatusCodeException e) {
logger.error(e);
} catch (MalformedURLException e) {
logger.error(e);
} catch (IOException e) {
logger.error(e);
}
HtmlForm form = page.getForms().get(0);
HtmlSubmitInput button = form.getInputByName("smblog");
form.getInputByName("keyword").setValueAttribute(word);
logger.info("search:" + word);
try {
page = button.click();
} catch (IOException e1) {
logger.error(e1);
}*/

HtmlPage page = null;
try {
//logger.info("execution:"+this);
page = webClient.getPage("http://weibo.cn/search/mblog?hideSearchFrame=&keyword="+word+"&page="+i);
} catch (FailingHttpStatusCodeException e) {
logger.error(e);
} catch (MalformedURLException e) {
logger.error(e);
} catch (IOException e) {
logger.error(e);
}

SimpleDateFormat dayformat = new SimpleDateFormat("yyyyMMdd");
long start = System.currentTimeMillis();
start = System.currentTimeMillis();
String path = null;
File file2 = null;
path = new String(outputpath + "/" + dayformat.format(start)
+ "/" + System.currentTimeMillis() + file.getName()+".html" );
file2 = new File(outputpath + "/" + dayformat.format(start));
if (!file2.exists())
file2.mkdirs();
file2 = new File(path);
System.out.println("当前页"+i+",采集至"+path);
if (file2.exists())
logger.warn("outfile exit!");
else {
FileOutputStream outputStream;
try {
outputStream = new FileOutputStream(file2);
outputStream.write(page.getWebResponse().getContentAsString().getBytes());
outputStream.close();
} catch (FileNotFoundException e) {
logger.error(e);
} catch (IOException e) {
logger.error(e);
}
}
webClient.closeAllWindows();
} else {
logger.warn("CookiePath doesn`t exit !!!");
}

logger.info("execution:");
try {
Thread.sleep(10000);
} catch (InterruptedException e) {
logger.error(e);
return;
}
}
return;

}

private static String cookiePathAppendRandom() {
Random random = new Random();
return cookiePath+random.nextInt(7);
}

public SinaSearchCrawlerCommand(String word, String cookiePath, String outputpath) {
if(word.contains("&")) {
word = word.replace("&", " ");
}
this.word = word;
this.cookiePath = cookiePath;
this.outputpath = outputpath;
}

@Override
public String toString() {
return "SinaSearchCrawlerCommand [word=" + word + ", outputpath="
+ outputpath + "]";
}

@Override
public Object call() throws Exception {
// TODO Auto-generated method stub
return null;
}

}