java:快速文件分割及合并

文件分割与合并是一个常见需求，比如：上传大文件时，可以先分割成小块，传到服务器后，再进行合并。很多高大上的分布式文件系统（比如：google的GFS、taobao的TFS）里，也是按block为单位，对文件进行分割或合并。

看下基本思路：



如果有一个大文件，指定分割大小后（比如：按1M切割）

step 1：

先根据原始文件大小、分割大小，算出最终分割的小文件数N

step 2:

在磁盘上创建这N个小文件

step 3:

开多个线程（线程数=分割文件数），每个线程里，利用RandomAccessFile的seek功能，将读取指针定位到原文件里每一段的段首位置，然后向后读取指定大小（即：分割块大小），最终写入对应的分割文件，因为多线程并行处理，各写各的小文件，速度相对还是比较快的。

合并时，把上面的思路逆向处理即可。

核心代码：

分割处理：

1 /**
2      * 拆分文件
3      * @param fileName 待拆分的完整文件名
4      * @param byteSize 按多少字节大小拆分
5      * @return 拆分后的文件名列表
6      * @throws IOException
7      */
8     public List<String> splitBySize(String fileName, int byteSize)
9             throws IOException {
10         List<String> parts = new ArrayList<String>();
11         File file = new File(fileName);
12         int count = (int) Math.ceil(file.length() / (double) byteSize);
13         int countLen = (count + "").length();
14         ThreadPoolExecutor threadPool = new ThreadPoolExecutor(count,
15                 count * 3, 1, TimeUnit.SECONDS,
16                 new ArrayBlockingQueue<Runnable>(count * 2));
17
18         for (int i = 0; i < count; i++) {
19             String partFileName = file.getName() + "."
20                     + leftPad((i + 1) + "", countLen, '0') + ".part";
21             threadPool.execute(new SplitRunnable(byteSize, i * byteSize,
22                     partFileName, file));
23             parts.add(partFileName);
24         }
25         return parts;
26     }

1 private class SplitRunnable implements Runnable {
2         int byteSize;
3         String partFileName;
4         File originFile;
5         int startPos;
6
7         public SplitRunnable(int byteSize, int startPos, String partFileName,
8                 File originFile) {
9             this.startPos = startPos;
10             this.byteSize = byteSize;
11             this.partFileName = partFileName;
12             this.originFile = originFile;
13         }
14
15         public void run() {
16             RandomAccessFile rFile;
17             OutputStream os;
18             try {
19                 rFile = new RandomAccessFile(originFile, "r");
20                 byte[] b = new byte[byteSize];
21                 rFile.seek(startPos);// 移动指针到每“段”开头
22                 int s = rFile.read(b);
23                 os = new FileOutputStream(partFileName);
24                 os.write(b, 0, s);
25                 os.flush();
26                 os.close();
27             } catch (IOException e) {
28                 e.printStackTrace();
29             }
30         }
31     }

合并处理：

1 /**
2      * 合并文件
3      *
4      * @param dirPath 拆分文件所在目录名
5      * @param partFileSuffix 拆分文件后缀名
6      * @param partFileSize 拆分文件的字节数大小
7      * @param mergeFileName 合并后的文件名
8      * @throws IOException
9      */
10     public void mergePartFiles(String dirPath, String partFileSuffix,
11             int partFileSize, String mergeFileName) throws IOException {
12         ArrayList<File> partFiles = FileUtil.getDirFiles(dirPath,
13                 partFileSuffix);
14         Collections.sort(partFiles, new FileComparator());
15
16         RandomAccessFile randomAccessFile = new RandomAccessFile(mergeFileName,
17                 "rw");
18         randomAccessFile.setLength(partFileSize * (partFiles.size() - 1)
19                 + partFiles.get(partFiles.size() - 1).length());
20         randomAccessFile.close();
21
22         ThreadPoolExecutor threadPool = new ThreadPoolExecutor(
23                 partFiles.size(), partFiles.size() * 3, 1, TimeUnit.SECONDS,
24                 new ArrayBlockingQueue<Runnable>(partFiles.size() * 2));
25
26         for (int i = 0; i < partFiles.size(); i++) {
27             threadPool.execute(new MergeRunnable(i * partFileSize,
28                     mergeFileName, partFiles.get(i)));
29         }
30
31     }

1 private class MergeRunnable implements Runnable {
2         long startPos;
3         String mergeFileName;
4         File partFile;
5
6         public MergeRunnable(long startPos, String mergeFileName, File partFile) {
7             this.startPos = startPos;
8             this.mergeFileName = mergeFileName;
9             this.partFile = partFile;
10         }
11
12         public void run() {
13             RandomAccessFile rFile;
14             try {
15                 rFile = new RandomAccessFile(mergeFileName, "rw");
16                 rFile.seek(startPos);
17                 FileInputStream fs = new FileInputStream(partFile);
18                 byte[] b = new byte[fs.available()];
19                 fs.read(b);
20                 fs.close();
21                 rFile.write(b);
22                 rFile.close();
23             } catch (IOException e) {
24                 e.printStackTrace();
25             }
26         }
27     }

为了方便文件操作，把关于文件读写的功能，全封装到FileUtil类：

1 package com.cnblogs.yjmyzz;
2
3 import java.io.*;
4 import java.util.*;
5 import java.util.concurrent.*;
6
7 /**
8  * 文件处理辅助类
9  *
10  * @author yjmyzz@126.com
11  * @version 0.2
12  * @since 2014-11-17
13  *
14  */
15 public class FileUtil {
16
17     /**
18      * 当前目录路径
19      */
20     public static String currentWorkDir = System.getProperty("user.dir") + "\\";
21
22     /**
23      * 左填充
24      *
25      * @param str
26      * @param length
27      * @param ch
28      * @return
29      */
30     public static String leftPad(String str, int length, char ch) {
31         if (str.length() >= length) {
32             return str;
33         }
34         char[] chs = new char[length];
35         Arrays.fill(chs, ch);
36         char[] src = str.toCharArray();
37         System.arraycopy(src, 0, chs, length - src.length, src.length);
38         return new String(chs);
39
40     }
41
42     /**
43      * 删除文件
44      *
45      * @param fileName
46      *            待删除的完整文件名
47      * @return
48      */
49     public static boolean delete(String fileName) {
50         boolean result = false;
51         File f = new File(fileName);
52         if (f.exists()) {
53             result = f.delete();
54
55         } else {
56             result = true;
57         }
58         return result;
59     }
60
61     /***
62      * 递归获取指定目录下的所有的文件（不包括文件夹）
63      *
64      * @param obj
65      * @return
66      */
67     public static ArrayList<File> getAllFiles(String dirPath) {
68         File dir = new File(dirPath);
69
70         ArrayList<File> files = new ArrayList<File>();
71
72         if (dir.isDirectory()) {
73             File[] fileArr = dir.listFiles();
74             for (int i = 0; i < fileArr.length; i++) {
75                 File f = fileArr[i];
76                 if (f.isFile()) {
77                     files.add(f);
78                 } else {
79                     files.addAll(getAllFiles(f.getPath()));
80                 }
81             }
82         }
83         return files;
84     }
85
86     /**
87      * 获取指定目录下的所有文件(不包括子文件夹)
88      *
89      * @param dirPath
90      * @return
91      */
92     public static ArrayList<File> getDirFiles(String dirPath) {
93         File path = new File(dirPath);
94         File[] fileArr = path.listFiles();
95         ArrayList<File> files = new ArrayList<File>();
96
97         for (File f : fileArr) {
98             if (f.isFile()) {
99                 files.add(f);
100             }
101         }
102         return files;
103     }
104
105     /**
106      * 获取指定目录下特定文件后缀名的文件列表(不包括子文件夹)
107      *
108      * @param dirPath
109      *            目录路径
110      * @param suffix
111      *            文件后缀
112      * @return
113      */
114     public static ArrayList<File> getDirFiles(String dirPath,
115             final String suffix) {
116         File path = new File(dirPath);
117         File[] fileArr = path.listFiles(new FilenameFilter() {
118             public boolean accept(File dir, String name) {
119                 String lowerName = name.toLowerCase();
120                 String lowerSuffix = suffix.toLowerCase();
121                 if (lowerName.endsWith(lowerSuffix)) {
122                     return true;
123                 }
124                 return false;
125             }
126
127         });
128         ArrayList<File> files = new ArrayList<File>();
129
130         for (File f : fileArr) {
131             if (f.isFile()) {
132                 files.add(f);
133             }
134         }
135         return files;
136     }
137
138     /**
139      * 读取文件内容
140      *
141      * @param fileName
142      *            待读取的完整文件名
143      * @return 文件内容
144      * @throws IOException
145      */
146     public static String read(String fileName) throws IOException {
147         File f = new File(fileName);
148         FileInputStream fs = new FileInputStream(f);
149         String result = null;
150         byte[] b = new byte[fs.available()];
151         fs.read(b);
152         fs.close();
153         result = new String(b);
154         return result;
155     }
156
157     /**
158      * 写文件
159      *
160      * @param fileName
161      *            目标文件名
162      * @param fileContent
163      *            写入的内容
164      * @return
165      * @throws IOException
166      */
167     public static boolean write(String fileName, String fileContent)
168             throws IOException {
169         boolean result = false;
170         File f = new File(fileName);
171         FileOutputStream fs = new FileOutputStream(f);
172         byte[] b = fileContent.getBytes();
173         fs.write(b);
174         fs.flush();
175         fs.close();
176         result = true;
177         return result;
178     }
179
180     /**
181      * 追加内容到指定文件
182      *
183      * @param fileName
184      * @param fileContent
185      * @return
186      * @throws IOException
187      */
188     public static boolean append(String fileName, String fileContent)
189             throws IOException {
190         boolean result = false;
191         File f = new File(fileName);
192         if (f.exists()) {
193             RandomAccessFile rFile = new RandomAccessFile(f, "rw");
194             byte[] b = fileContent.getBytes();
195             long originLen = f.length();
196             rFile.setLength(originLen + b.length);
197             rFile.seek(originLen);
198             rFile.write(b);
199             rFile.close();
200         }
201         result = true;
202         return result;
203     }
204
205     /**
206      * 拆分文件
207      *
208      * @param fileName
209      *            待拆分的完整文件名
210      * @param byteSize
211      *            按多少字节大小拆分
212      * @return 拆分后的文件名列表
213      * @throws IOException
214      */
215     public List<String> splitBySize(String fileName, int byteSize)
216             throws IOException {
217         List<String> parts = new ArrayList<String>();
218         File file = new File(fileName);
219         int count = (int) Math.ceil(file.length() / (double) byteSize);
220         int countLen = (count + "").length();
221         ThreadPoolExecutor threadPool = new ThreadPoolExecutor(count,
222                 count * 3, 1, TimeUnit.SECONDS,
223                 new ArrayBlockingQueue<Runnable>(count * 2));
224
225         for (int i = 0; i < count; i++) {
226             String partFileName = file.getName() + "."
227                     + leftPad((i + 1) + "", countLen, '0') + ".part";
228             threadPool.execute(new SplitRunnable(byteSize, i * byteSize,
229                     partFileName, file));
230             parts.add(partFileName);
231         }
232         return parts;
233     }
234
235     /**
236      * 合并文件
237      *
238      * @param dirPath
239      *            拆分文件所在目录名
240      * @param partFileSuffix
241      *            拆分文件后缀名
242      * @param partFileSize
243      *            拆分文件的字节数大小
244      * @param mergeFileName
245      *            合并后的文件名
246      * @throws IOException
247      */
248     public void mergePartFiles(String dirPath, String partFileSuffix,
249             int partFileSize, String mergeFileName) throws IOException {
250         ArrayList<File> partFiles = FileUtil.getDirFiles(dirPath,
251                 partFileSuffix);
252         Collections.sort(partFiles, new FileComparator());
253
254         RandomAccessFile randomAccessFile = new RandomAccessFile(mergeFileName,
255                 "rw");
256         randomAccessFile.setLength(partFileSize * (partFiles.size() - 1)
257                 + partFiles.get(partFiles.size() - 1).length());
258         randomAccessFile.close();
259
260         ThreadPoolExecutor threadPool = new ThreadPoolExecutor(
261                 partFiles.size(), partFiles.size() * 3, 1, TimeUnit.SECONDS,
262                 new ArrayBlockingQueue<Runnable>(partFiles.size() * 2));
263
264         for (int i = 0; i < partFiles.size(); i++) {
265             threadPool.execute(new MergeRunnable(i * partFileSize,
266                     mergeFileName, partFiles.get(i)));
267         }
268
269     }
270
271     /**
272      * 根据文件名，比较文件
273      *
274      * @author yjmyzz@126.com
275      *
276      */
277     private class FileComparator implements Comparator<File> {
278         public int compare(File o1, File o2) {
279             return o1.getName().compareToIgnoreCase(o2.getName());
280         }
281     }
282
283     /**
284      * 分割处理Runnable
285      *
286      * @author yjmyzz@126.com
287      *
288      */
289     private class SplitRunnable implements Runnable {
290         int byteSize;
291         String partFileName;
292         File originFile;
293         int startPos;
294
295         public SplitRunnable(int byteSize, int startPos, String partFileName,
296                 File originFile) {
297             this.startPos = startPos;
298             this.byteSize = byteSize;
299             this.partFileName = partFileName;
300             this.originFile = originFile;
301         }
302
303         public void run() {
304             RandomAccessFile rFile;
305             OutputStream os;
306             try {
307                 rFile = new RandomAccessFile(originFile, "r");
308                 byte[] b = new byte[byteSize];
309                 rFile.seek(startPos);// 移动指针到每“段”开头
310                 int s = rFile.read(b);
311                 os = new FileOutputStream(partFileName);
312                 os.write(b, 0, s);
313                 os.flush();
314                 os.close();
315             } catch (IOException e) {
316                 e.printStackTrace();
317             }
318         }
319     }
320
321     /**
322      * 合并处理Runnable
323      *
324      * @author yjmyzz@126.com
325      *
326      */
327     private class MergeRunnable implements Runnable {
328         long startPos;
329         String mergeFileName;
330         File partFile;
331
332         public MergeRunnable(long startPos, String mergeFileName, File partFile) {
333             this.startPos = startPos;
334             this.mergeFileName = mergeFileName;
335             this.partFile = partFile;
336         }
337
338         public void run() {
339             RandomAccessFile rFile;
340             try {
341                 rFile = new RandomAccessFile(mergeFileName, "rw");
342                 rFile.seek(startPos);
343                 FileInputStream fs = new FileInputStream(partFile);
344                 byte[] b = new byte[fs.available()];
345                 fs.read(b);
346                 fs.close();
347                 rFile.write(b);
348                 rFile.close();
349             } catch (IOException e) {
350                 e.printStackTrace();
351             }
352         }
353     }
354
355 }

单元测试：

1 package com.cnblogs.yjmyzz;
2
3 import java.io.IOException;
4
5 import org.junit.Test;
6
7 public class FileTest {
8
9     @Test
10     public void writeFile() throws IOException, InterruptedException {
11
12         System.out.println(FileUtil.currentWorkDir);
13
14         StringBuilder sb = new StringBuilder();
15
16         long originFileSize = 1024 * 1024 * 100;// 100M
17         int blockFileSize = 1024 * 1024 * 15;// 15M
18
19         // 生成一个大文件
20         for (int i = 0; i < originFileSize; i++) {
21             sb.append("A");
22         }
23
24         String fileName = FileUtil.currentWorkDir + "origin.myfile";
25         System.out.println(fileName);
26         System.out.println(FileUtil.write(fileName, sb.toString()));
27
28         // 追加内容
29         sb.setLength(0);
30         sb.append("0123456789");
31         FileUtil.append(fileName, sb.toString());
32
33         FileUtil fileUtil = new FileUtil();
34
35         // 将origin.myfile拆分
36         fileUtil.splitBySize(fileName, blockFileSize);
37
38         Thread.sleep(10000);// 稍等10秒，等前面的小文件全都写完
39
40         // 合并成新文件
41         fileUtil.mergePartFiles(FileUtil.currentWorkDir, ".part",
42                 blockFileSize, FileUtil.currentWorkDir + "new.myfile");
43
44     }
45 }