Java实现多线程下载文件

原文地址：https://blog.dubby.cn/detail.html?id=9090

1.基本原理

先使用

head

方法查询得到对应文件的

Content-Length

，然后拆分成多个部分，交由多个线程去处理，使用

"Range", "bytes=" + start + "-" + end

这个header来指定下载文件的哪个部分。

2.代码实现

为了方便展示，我是用了一个类来实现，其余都是内部类

/**
* Created by yangzheng03 on 2018/1/16.
* https://www.dubby.cn/ */
public class DownloadTool {
private static String prefix = String.valueOf(System.currentTimeMillis());
private static String path = "";
public static void main(String[] args) {
long startTimestamp = System.currentTimeMillis();
if (args == null || args.length < 1) {
System.out.println("please input the file url.");
return;
}
final String urlString = args[0];
if (args.length >= 2) {
path = args[1];
}
int number = 10;
if (args.length >= 3) {
number = Integer.parseInt(args[2]);
}
System.out.println("Download started, url is \"" + urlString + "\"");
ExecutorService threadPool = Executors.newFixedThreadPool(number);
try {
URL url = new URL(urlString);
HttpURLConnection httpURLConnection = (HttpURLConnection) url.openConnection();
httpURLConnection.setRequestMethod("HEAD");
String contentLengthStr = httpURLConnection.getHeaderField("Content-Length");
long contentLength = Long.parseLong(contentLengthStr);
if (contentLength > 1024 * 1024) {
System.out.println("Content-Length\t" + (contentLength / 1024 / 1024) + "MB");
} else if (contentLength > 1024) {
System.out.println("Content-Length\t" + (contentLength / 1024) + "KB");
} else {
System.out.println("Content-Length\t" + (contentLength) + "B");
}
long tempLength = contentLength / number;
long start = 0, end = -1;
Map<Integer, Future<DownloadTemp>> futureMap = new HashMap<>(number);
for (int i = 0; i < number; ++i) {
start = end + 1;
end = end + tempLength;
if (i == number - 1) {
end = contentLength;
}
System.out.println("start:\t" + start + "\tend:\t" + end);
DownloadThread thread = new DownloadThread(i, start, end, urlString);
futureMap.put(i, threadPool.submit(thread));
}
System.out.println();
String filename = urlString.substring(urlString.lastIndexOf("/") + 1);
if (!path.equals("")) {
filename = path + "/" + filename;
}
RandomAccessFile resultFile = new RandomAccessFile(filename, "rw");
for (int i = 0; i < number; ++i) {
Future<DownloadTemp> future = futureMap.get(i);
DownloadTemp temp = future.get();
RandomAccessFile tempFile = new RandomAccessFile(temp.filename, "r");
tempFile.getChannel().transferTo(0, tempFile.length(), resultFile.getChannel());
}
threadPool.shutdown();
} catch (Exception e) {
e.printStackTrace();
}
long completedTimestamp = System.currentTimeMillis();
System.out.println();
System.out.println("cost " + (completedTimestamp - startTimestamp) / 1000 + "s");
}
private static class DownloadThread implements Callable<DownloadTemp> {
private int index;
private String filename;
private long start, end;
private String urlString;
DownloadThread(int index, long start, long end, String url) {
this.urlString = url;
this.index = index;
this.start = start;
this.end = end;
if (path.equals("")) {
this.filename = prefix + "-temp-" + index;
} else {
this.filename = path + "/" + prefix + "-temp-" + index;
}
}
@Override
public DownloadTemp call() throws Exception {
URL url = new URL(urlString);
HttpURLConnection httpURLConnection = (HttpURLConnection) url.openConnection();
httpURLConnection.setRequestMethod("GET");
httpURLConnection.setRequestProperty("Range", "bytes=" + start + "-" + end);
InputStream inputStream = null;
OutputStream outputStream = null;
try {
inputStream = httpURLConnection.getInputStream();
File file = new File(filename);
outputStream = new FileOutputStream(file);
while (true) {
byte[] bytes = new byte[10240];
int length = inputStream.read(bytes);
if (length <= 0) {
break;
}
outputStream.write(bytes, 0, length);
}
outputStream.flush();
} finally {
if (inputStream != null) {
inputStream.close();
}
if (outputStream != null) {
outputStream.close();
}
}
DownloadTemp downloadTemp = new DownloadTemp();
downloadTemp.index = index;
downloadTemp.filename = filename;
System.out.println("thread\t" + index + "\tcompleted.");
return downloadTemp;
}
}
private static class DownloadTemp {
private int index;
private String filename;
}
}

3.使用Jar包

下载工具 download-1.0.jar

下载下来后，改名为

download-1.0.jar

，执行：

java -jar download-1.0.jar https://blog.dubby.cn/upload/2018-01-13/68b780a2-1a6c-49dc-914e-0469cb969471.zip  /Users/test/Desktop/download 15

其中有三个参数:

下载的URL，必填

下载到目标目录，选填，如果不指定，下载到当前目前下

开启的线程数，选填，默认是10个线程

测试结果:

➜  target java -jar download-1.0.jar https://blog.dubby.cn/upload/2018-01-13/68b780a2-1a6c-49dc-914e-0469cb969471.zip  /Users/test/Desktop/download 15
Download started, url is "https://blog.dubby.cn/upload/2018-01-13/68b780a2-1a6c-49dc-914e-0469cb969471.zip"
Content-Length  1MB
start:  0       end:    83780
start:  83781   end:    167561
start:  167562  end:    251342
start:  251343  end:    335123
start:  335124  end:    418904
start:  418905  end:    502685
start:  502686  end:    586466
start:  586467  end:    670247
start:  670248  end:    754028
start:  754029  end:    837809
start:  837810  end:    921590
start:  921591  end:    1005371
start:  1005372 end:    1089152
start:  1089153 end:    1172933
start:  1172934 end:    1256726

thread  0       completed.
thread  10      completed.
thread  1       completed.
thread  9       completed.
thread  14      completed.
thread  11      completed.
thread  7       completed.
thread  6       completed.
thread  2       completed.
thread  4       completed.
thread  3       completed.
thread  8       completed.
thread  12      completed.
thread  13      completed.
thread  5       completed.

cost 9s