Hadoop—HDFS的Java客户端编程
2016-02-05 12:17
489 查看
可以使用HDFS Shell命令来完成对HDFS的操作,当然也可以使用Java编程语言来对HDFS实现操作。以下是一些Unit测试代码
package cn.nuc.hadoop.hdfs;
import java.io.FileInputStream;
import java.io.FileNotFoundException;
import java.io.IOException;
import java.net.URI;
import java.net.URISyntaxException;
import org.apache.commons.compress.utils.IOUtils;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FSDataOutputStream;
import org.apache.hadoop.fs.FileStatus;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.LocatedFileStatus;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.fs.RemoteIterator;
import org.junit.Before;
import org.junit.Test;
public class HdfsUtil {
FileSystem fs = null;
@Before
public void init() throws IOException, InterruptedException,
URISyntaxException {
Configuration conf = new Configuration();// 读取配置文件
// 设置fs.defaultFS参数,如果没有设置,会出现java.lang.IllegalArgumentException:
// Wrong FS:hdfs://master:9000/user,expected: file:///
// 当然了,也可以将hadoop集群中的core-site.xml配置文件拷贝到该工程项目下,这样,在读取配置文件时就能够识别hdfs文件系统
conf.set("fs.defaultFS", "hdfs://master:9000/");
fs = FileSystem.get(new URI("hdfs://master:9000/"), conf, "hadoop");// 获取hdfs实例
}
// upload a loacl file to HDFS
@Test
public void upload() throws IOException {
Path dest = new Path("hdfs://master:9000/user/artifacts.xml");
FSDataOutputStream fsOut = fs.create(dest);
FileInputStream localIn = new FileInputStream(
"D:\\eclipse\\artifacts.xml");
IOUtils.copy(localIn, fsOut);
}
@Test
public void upload2() throws IllegalArgumentException, IOException {
fs.copyFromLocalFile(new Path("D:\\eclipse\\artifacts.xml"), new Path(
"hdfs://master:9000/user/artifacts2.xml"));
}
// download a file to local file
@Test
public void download() throws IllegalArgumentException, IOException {
fs.copyToLocalFile(new Path("hdfs://master:9000/user/artifacts2.xml"),
new Path("D:\\a.xml"));
}
// list HDFS file
@Test
public void listFiles() throws FileNotFoundException, IllegalArgumentException, IOException {
// listFiles列出的是文件信息,而且提供递归遍历
RemoteIterator<LocatedFileStatus> files = fs.listFiles(new Path("/"),
true);
while (files.hasNext()) {
LocatedFileStatus file = files.next();
Path filePath = file.getPath();
String fileName = filePath.getName();
System.out.println(fileName);
}
System.out.println("---------------------------------");
// listStatus 可以列出文件和文件夹的信息,但是不提供自带的递归遍历
FileStatus[] listStatus = fs.listStatus(new Path("/"));
for (FileStatus status : listStatus) {
String name = status.getPath().getName();
System.out.println(name
+ (status.isDirectory() ? " is dir" : " is file"));
}
}
// create HDFS folder
@Test
public void mkdir() throws IllegalArgumentException, IOException {
fs.mkdirs(new Path("hdfs://master:9000/aaa/bbb/ccc"));
}
// remove HDFS folder or file
@Test
public void remove() throws IllegalArgumentException, IOException {
fs.delete(new Path("hdfs://master:9000/aaa"), true);
}
}
package cn.nuc.hadoop.hdfs;
import java.io.FileInputStream;
import java.io.FileNotFoundException;
import java.io.IOException;
import java.net.URI;
import java.net.URISyntaxException;
import org.apache.commons.compress.utils.IOUtils;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FSDataOutputStream;
import org.apache.hadoop.fs.FileStatus;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.LocatedFileStatus;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.fs.RemoteIterator;
import org.junit.Before;
import org.junit.Test;
public class HdfsUtil {
FileSystem fs = null;
@Before
public void init() throws IOException, InterruptedException,
URISyntaxException {
Configuration conf = new Configuration();// 读取配置文件
// 设置fs.defaultFS参数,如果没有设置,会出现java.lang.IllegalArgumentException:
// Wrong FS:hdfs://master:9000/user,expected: file:///
// 当然了,也可以将hadoop集群中的core-site.xml配置文件拷贝到该工程项目下,这样,在读取配置文件时就能够识别hdfs文件系统
conf.set("fs.defaultFS", "hdfs://master:9000/");
fs = FileSystem.get(new URI("hdfs://master:9000/"), conf, "hadoop");// 获取hdfs实例
}
// upload a loacl file to HDFS
@Test
public void upload() throws IOException {
Path dest = new Path("hdfs://master:9000/user/artifacts.xml");
FSDataOutputStream fsOut = fs.create(dest);
FileInputStream localIn = new FileInputStream(
"D:\\eclipse\\artifacts.xml");
IOUtils.copy(localIn, fsOut);
}
@Test
public void upload2() throws IllegalArgumentException, IOException {
fs.copyFromLocalFile(new Path("D:\\eclipse\\artifacts.xml"), new Path(
"hdfs://master:9000/user/artifacts2.xml"));
}
// download a file to local file
@Test
public void download() throws IllegalArgumentException, IOException {
fs.copyToLocalFile(new Path("hdfs://master:9000/user/artifacts2.xml"),
new Path("D:\\a.xml"));
}
// list HDFS file
@Test
public void listFiles() throws FileNotFoundException, IllegalArgumentException, IOException {
// listFiles列出的是文件信息,而且提供递归遍历
RemoteIterator<LocatedFileStatus> files = fs.listFiles(new Path("/"),
true);
while (files.hasNext()) {
LocatedFileStatus file = files.next();
Path filePath = file.getPath();
String fileName = filePath.getName();
System.out.println(fileName);
}
System.out.println("---------------------------------");
// listStatus 可以列出文件和文件夹的信息,但是不提供自带的递归遍历
FileStatus[] listStatus = fs.listStatus(new Path("/"));
for (FileStatus status : listStatus) {
String name = status.getPath().getName();
System.out.println(name
+ (status.isDirectory() ? " is dir" : " is file"));
}
}
// create HDFS folder
@Test
public void mkdir() throws IllegalArgumentException, IOException {
fs.mkdirs(new Path("hdfs://master:9000/aaa/bbb/ccc"));
}
// remove HDFS folder or file
@Test
public void remove() throws IllegalArgumentException, IOException {
fs.delete(new Path("hdfs://master:9000/aaa"), true);
}
}
相关文章推荐
- java对世界各个时区(TimeZone)的通用转换处理方法(转载)
- java-注解annotation
- java-模拟tomcat服务器
- java-用HttpURLConnection发送Http请求.
- java-WEB中的监听器Lisener
- Android IPC进程间通讯机制
- android wifi 无线调试
- Android Native 绘图方法
- Android java 与 javascript互访(相互调用)的方法例子
- 介绍一款信息管理系统的开源框架---jeecg
- 聚类算法之kmeans算法java版本
- java实现 PageRank算法
- 详解HDFS Short Circuit Local Reads
- 运维入门
- 动态清空 nohup 输出文件
- install scrapy with pip and easy_install
- PropertyChangeListener简单理解
- c++11 + SDL2 + ffmpeg +OpenAL + java = Android播放器