HDFS JavaAPI 学习

写给自己:晚上的时候,在看着这个HDFS的JavaAPI.于是也顺手写一写.不过,这并不是想说的话.小侄女还不到一岁.因为肚子不舒服,在一直哭.然后,心思也跟着碎了.一个人长大真不容易.希望我的小侄女找些长大,少经历一些痛苦.嘿嘿,也许在十多年后,我就能开始教她敲代码了.带她走入程序的世界.

希望在这之前,学习一下Maven的使用.maven安装配置就不说了.安装完成后比较重要的一个事情就是修改远程仓库的地址.修改成阿里云的.这样,下载jar包的时候就快了.这些网上都能搜索到.配置完成后,在eclipse配置一下(修改成自己的配置).然后可以开始了.

package managemen;

import java.io.File;
import java.io.FileNotFoundException;
import java.io.FileOutputStream;
import java.io.IOException;
import java.net.URI;
import java.net.URISyntaxException;

import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.BlockLocation;
import org.apache.hadoop.fs.FSDataInputStream;
import org.apache.hadoop.fs.FileStatus;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.LocatedFileStatus;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.fs.RemoteIterator;
import org.junit.Before;
import org.junit.Test;

public class Demo01 {

Configuration conf
4000
iguration=null;
FileSystem fileSystem=null;
@Before
public void init() throws IOException, InterruptedException, URISyntaxException{
configuration=new Configuration();
fileSystem=FileSystem.get(new URI("hdfs://hadoop:9000"),configuration,"root");

}

@Test
public void testMkdir() throws IllegalArgumentException, IOException{
System.out.println(fileSystem.mkdirs(new Path("/Date")));
fileSystem.close();
}

@Test
public void testCopyFromLocal() throws Exception{
Path path=new Path("F:/h.txt");
Path dfs=new Path("/Date");
fileSystem.copyFromLocalFile(path, dfs);
fileSystem.close();
}

@Test
public void testCopytoLocal () throws Exception{
Path src=new Path("/data/test.txt");
Path dst = new Path("F:/");
fileSystem.copyToLocalFile(src, dst);
fileSystem.close();
}

@Test
public void testRemove() throws Exception{
fileSystem.delete(new Path("/data"),false);
fileSystem.close();
}

@Test
public void testReName() throws IllegalArgumentException, IOException{
fileSystem.rename(new Path("/Data"),new Path("/Datas"));
fileSystem.close();
}

@Test
public void testList() throws FileNotFoundException, IllegalArgumentException, IOException{
RemoteIterator<LocatedFileStatus> listFiles = fileSystem.listFiles(new Path("/Data"), true);
while (listFiles.hasNext()) {
LocatedFileStatus fileStatus = listFiles.next();
System.out.println(fileStatus.getPath().getName());
System.out.println(fileStatus.getBlockSize());
System.out.println(fileStatus.getPermission());
System.out.println(fileStatus.getReplication());
System.out.println(fileStatus.getLen());
BlockLocation[] blockLocations = fileStatus.getBlockLocations();
for (BlockLocation blockLocation : blockLocations) {
System.out.println(blockLocation.getLength()+blockLocation.getOffset());
String[] hosts = blockLocation.getHosts();
for (String string : hosts) {
System.out.println(string);
}

}

}

}

@Test
public void testListStatus() throws Exception {
FileStatus[] listStatus = fileSystem.listStatus(new Path("/"));
String flag = " ";
for (FileStatus status : listStatus) {
if (status.isDirectory()) {
flag = "目录";
} else {
flag = "文件";
}
System.out.println(flag + "\t" + status.getPath().getName());
}
fileSystem.close();
}

@Test
public void testDownloadToLocal() throws IllegalArgumentException, IOException{
//上传一个文件。
//先获取HDFS 的一个输出流
//再获取本地文件的输出流
FSDataInputStream open = fileSystem.open(new Path("/data"));
FileOutputStream fileOutputStream=new FileOutputStream(new File("D:/date.txt"));
org.apache.commons.io.IOUtils.copy(open, fileOutputStream);
}

@Test
public void testCat() throws IllegalArgumentException, IOException{
FSDataInputStream in = fileSystem.open(new Path("/Mytest/jdk-7u71-linux-x64"));
FileStatus[] listStatus = fileSystem.listStatus(new Path("/Mytest/jdk-7u71-linux-x64"));
BlockLocation[] fileBlockLocations = fileSystem.getFileBlockLocations(
listStatus[0], 0L, listStatus[0].getLen());
long length=fileBlockLocations[0].getLength();
long offset=fileBlockLocations[0].getOffset();
System.out.println(length);
System.out.println(offset);
org.apache.hadoop.io.IOUtils.copyBytes(in, System.out, (int)length);
byte[] b=new byte[4096];
FileOutputStream os = new FileOutputStream(new File("d:/"));
while(in.read(offset,b,0,4096)!=-1){
os.write(b);
offset+=4096;
if (offset > length)
return;
}
os.flush();
os.close();
in.close();
}

}