JAVA jsoup网络抓取图片

http://jsoup.org/

Jsoup

介绍

Jsoup

is
a Java library for working with real-world HTML. It provides a very convenient API for extracting and manipulating data, using the best of DOM, CSS, and jquery-like methods.

Jsoup

其实就是发起一个http请求,然后拿到dom.然后可以按照jquery那样的格式去拿对应的元素.

由于目前的大部分网站都是动态数据,所以很多数据都是固定到某一个element下面.

抓取的工具类

public class CrawUtil {
private static String USER_AGENT="Mozilla/5.0 (compatible; MSIE 9.0; Windows NT 6.1; WOW64; Trident/5.0";
private static String timeOut="60000";
/**
* 根据url抓取网页,返回document对象
* @param uri 需要抓取的url
* @return 返回document 对象
* @throws IOException
*/
public static Document getDoc(String uri) throws IOException {
int time  = Integer.parseInt(timeOut);
Document doc = Jsoup.connect(uri).userAgent(USER_AGENT)
.ignoreContentType(true).timeout(time).get();
return doc;
}

}

从壁纸库抓取"美女图片"
程序猿太无聊

import java.io.IOException;
import java.util.ArrayList;
import java.util.List;
import java.util.Map;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;

import org.apache.commons.lang3.StringUtils;
import org.jsoup.nodes.Document;
import org.jsoup.nodes.Element;
import org.jsoup.select.Elements;

import net.sf.json.JSONArray;
import net.sf.json.JSONObject;

import com.luqiao.core.util.craw.CrawUtil;
import com.luqiao.core.util.httpclient.HttpClientUtil;

/**
* 从搜狗网站抓取图片
* @author anfu.yang
* @date 2015年6月2日 上午9:49:43
*/
public class CrawImages {
private static String url = "http://bizhi.sogou.com/cate/getCate/4/";
private static String detail_url = "http://bizhi.sogou.com/detail/info/";
private static String wp_id = "1171179";
private static String downloadPrefix = "H:\\Downloads\\";
/**
* 从分类信息获取图片ID 搜狗壁纸库
* @throws IOException
*/
@SuppressWarnings("unchecked")
public static List<String> getCategory() throws IOException{
String getCate = HttpClientUtil.doGet(url+wp_id, null);
JSONObject obj = JSONObject.fromObject(getCate);
wp_id = obj.getString("min_wp_id");
JSONArray array = obj.getJSONArray("wallpapers");
List<String> wpids = new ArrayList<String>();
for(int i = 0 ; i < array.size() ; i ++){
Map<String,Object> wallpaper = JSONObject.fromObject(array.get(i));
wpids.add((String)wallpaper.get("wp_id"));
}
return wpids;
}

/**
* 获取图片的url
* @param wpid
* @return
* @throws IOException
*/
public static String getImgUrl(String wpid) throws IOException{
Document detail = CrawUtil.getDoc(detail_url+wpid);
String img_url = "";
Elements elements = detail.getElementsByClass("unews_wp_big");
if(elements.size() > 0){
Element img = elements.get(0).getElementsByTag("img").get(0);
img_url = img.attr("src");

}
return img_url;
}
/**
* 下载图片
* @param imgUrl
*/
public static void downloadImg(String imgUrl){
String suffix = StringUtils.substring(imgUrl, StringUtils.lastIndexOf(imgUrl, "/"));
HttpClientUtil.download(imgUrl, downloadPrefix+suffix);
}

public static void main(String[] args) throws IOException {
ExecutorService pool = Executors.newFixedThreadPool(20);
List<String> wpids = getCategory();
execute(wpids, pool);
}

/**
* 递归获取图片地址
* @param wpids
* @param pool
* @throws IOException
*/
public  static void execute(List<String>  wpids,ExecutorService pool ) throws IOException{
for(int i = 0 ; i < wpids.size() ; i ++){
Download t = new CrawImages().new Download();
t.setWpid(wpids.get(i));
pool.execute(t);
if((i + 1) == wpids.size()){
wpids.clear();
wpids = getCategory();
execute(wpids,pool);
}
}
pool.shutdown();
}

/**
* 线程下载
* @author anfu.yang
* @date 2015年6月3日 上午11:22:28
*/
class Download implements Runnable{
String wpid;
@Override
public void run() {
try {
String img_url = getImgUrl(wpid);
System.out.println("下载... :"+img_url);
downloadImg(img_url);
} catch (IOException e) {
e.printStackTrace();
}
}
public String getWpid() {
return wpid;
}
public void setWpid(String wpid) {
this.wpid = wpid;
}

}
}

简单说下实现:

1,首先到搜狗壁纸分类去找到美女分类的url

2,然后通过页面的js请求的时候查看获取返回的json

3,解析json获取图片的详细信息页面

4,解析详细页面元素,拿到图片的url

5,执行多线程下载.

感兴趣的同学可以试试弄一个搜狗图片的图片抓取. 我就不贴代码了