java多线程下载美女图片

java多线程下载美女图片

Main.java

主线程，控制两个线程池，待任务处理完成后关闭线程池

package photo;

import java.util.List;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
import java.util.concurrent.LinkedBlockingDeque;

public class Main {

static final String IMG_ROOT = "D:/meinv/";

public static void main(String[] args) {
main2(args);
}

public static void main1(String[] args) {
long begin = System.currentTimeMillis();
Spider s = new Spider();
String url = "http://me2-sex.lofter.com/tag/%E7%BE%8E%E5%A5%B3%E6%91%84%E5%BD%B1";
for (int page = 1; page <= 10; page++) {
String content = s.getContent(url, page);
List<String> urlList = s.parseContent(content);
for (String picUrl : urlList) {
int index = picUrl.lastIndexOf("=");
String file = IMG_ROOT + picUrl.substring(index + 2);
s.downPhoto(picUrl, file);
}
}
long end = System.currentTimeMillis();
System.out.println("total cost time:" + (end - begin) / 1000 + "s");
}

public static void main2(String[] args) {
LinkedBlockingDeque<String> photoUrlQue = new LinkedBlockingDeque<String>(300);
ExecutorService executor = Executors.newFixedThreadPool(3);
for (int page = 1; page <= 10; page++) {
executor.execute(new DownPage(photoUrlQue, page));
}
ExecutorService photoexecutor = Executors.newFixedThreadPool(5);
Status status = new Status();
for (int i = 0; i < 5; i++) {
photoexecutor.execute(new DownPhoto(photoUrlQue,status));
}
executor.shutdown();
while (!executor.isTerminated()) {
try {
Thread.sleep(500);
} catch (InterruptedException e) {
e.printStackTrace();
}
}
status.doStop();
photoexecutor.shutdown();

}
}

DownPhoto.java

下载图片的线程，从队列中获得url，下载并保存到本地磁盘

package photo;

import java.util.concurrent.LinkedBlockingDeque;

public class DownPhoto implements Runnable {

static final String         IMG_ROOT = "D:/meinv/";
LinkedBlockingDeque<String> photoUrlQue;
private Status status;
public DownPhoto(LinkedBlockingDeque<String> photoUrlQue,Status status){
this.photoUrlQue = photoUrlQue;
this.status = status;
}

@Override
public void run() {
while (true) {
try {
if(photoUrlQue.isEmpty() && status.isStoped()){
break;
}
String picUrl = photoUrlQue.take();
Spider s = new Spider();
int index = picUrl.lastIndexOf("=");
String file = IMG_ROOT + picUrl.substring(index + 2);
s.downPhoto(picUrl, file);
} catch (InterruptedException e) {
break;
}
}
}

}

DownPage.java

下载网页内容，并分析图片url，存放到队列中，以供另外一个线程使用

package photo;

import java.util.List;
import java.util.concurrent.LinkedBlockingDeque;

public class DownPage implements Runnable {

LinkedBlockingDeque<String> photoUrlQue;
int                         page;
String                      url = "http://me2-sex.lofter.com/tag/%E7%BE%8E%E5%A5%B3%E6%91%84%E5%BD%B1";

public DownPage(LinkedBlockingDeque<String> photoUrlQue, int page){
this.photoUrlQue = photoUrlQue;
this.page = page;
}

@Override
public void run() {
Spider s = new Spider();
String content = s.getContent(url, page);
List<String> urlList = s.parseContent(content);
for (String picUrl : urlList) {
photoUrlQue.push(picUrl);
}

}
}

Spider.java

http通讯工具类

需要引入apache的commons-httpclient、commons-codec、commons-logging包

package photo;

import java.io.BufferedInputStream;
import java.io.BufferedOutputStream;
import java.io.File;
import java.io.FileOutputStream;
import java.io.IOException;
import java.io.InputStream;
import java.io.OutputStream;
import java.util.ArrayList;
import java.util.List;
import java.util.regex.Matcher;
import java.util.regex.Pattern;

import org.apache.commons.httpclient.HttpClient;
import org.apache.commons.httpclient.HttpMethod;
import org.apache.commons.httpclient.methods.GetMethod;
import org.apache.commons.httpclient.methods.PostMethod;

public class Spider {

public String getContent(String url, int page) {
HttpClient client = new HttpClient();
url += "?page=" + page;
HttpMethod method = new PostMethod(url);
method.setFollowRedirects(false);
method.addRequestHeader("User-Agent",
"Mozilla/5.0 (Windows NT 5.1) AppleWebKit/537.31 (KHTML, like Gecko) Chrome/26.0.1410.43 Safari/537.31");
try {
int status = client.executeMethod(method);
if (status != 200) {
return null;
}
return method.getResponseBodyAsString();
} catch (Exception e) {
e.printStackTrace();
} finally {
method.releaseConnection();
}
return null;
}

public void downPhoto(String url, String fileName) {
HttpClient client = new HttpClient();
HttpMethod method = new GetMethod(url);
method.setFollowRedirects(false);
method.addRequestHeader("User-Agent",
"Mozilla/5.0 (Windows NT 5.1) AppleWebKit/537.31 (KHTML, like Gecko) Chrome/26.0.1410.43 Safari/537.31");
try {
int status = client.executeMethod(method);
if (status != 200) {
return;
}
InputStream is = method.getResponseBodyAsStream();
BufferedInputStream bis = new BufferedInputStream(is);
File f = new File(fileName);
OutputStream os = new FileOutputStream(f);
BufferedOutputStream bos = new BufferedOutputStream(os);
byte[] buffer = new byte[128];
int readLen = -1;
while ((readLen = bis.read(buffer)) != -1) {
bos.write(buffer, 0, readLen);
bos.flush();
}
bos.close();
bis.close();
} catch (Exception e) {
e.printStackTrace();
} finally {
method.releaseConnection();
}

}

public static void main(String[] args) {

}

public List<String> parseContent(String content) {
List<String> urlList = new ArrayList<String>();
// <img src="http://imglf0.ph.126.net/BKehcpizRueVO8BuzwT8Lg==/6630835768745140932.jpg" />
Pattern p = Pattern.compile("<img\\s*src=\"(\\S+)\"\\s*/>");
// String input
Matcher m = p.matcher(content);
while (m.find()) {
String url = m.group(1);
//System.out.println(url);
urlList.add(url);
}
return urlList;
}

}

Status

状态控制类

package photo;

public class Status {

private volatile boolean stop;

public void doStop() {
stop = true;
}

public boolean isStoped() {
return stop;
}
}