您的位置:首页 > 编程语言 > Java开发

java 操作 hadoop hdfs

2017-05-22 00:00 302 查看
java 操作 hadoop hdfs

package com.traveller.bumble.hadoop.hdfs;

import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.*;
import org.apache.hadoop.io.IOUtils;
import org.junit.Test;

import java.io.ByteArrayInputStream;
import java.io.IOException;
import java.io.InputStream;
import java.io.OutputStream;
import java.net.URL;
import java.net.URLConnection;

/**
* Created by macbook on 2017/5/21.
*/
public class TestHDFS {

//    @Test
public void testRead() {
try {
URL.setURLStreamHandlerFactory(new FsUrlStreamHandlerFactory());
URL url = new URL("hdfs://master:8020/user/master/my.cnf");
URLConnection conn = url.openConnection();
InputStream inputStream = conn.getInputStream();
IOUtils.copyBytes(inputStream, System.out, 1024);
IOUtils.closeStream(inputStream);
} catch (Exception e) {
e.printStackTrace();
}
}

//    @Test
public void testWrite() {
try {
URL.setURLStreamHandlerFactory(new FsUrlStreamHandlerFactory());
URL url = new URL("hdfs://master:8020/user/slave1/my.cnf");
URLConnection conn = url.openConnection();
OutputStream outputStream = conn.getOutputStream();
ByteArrayInputStream bis = new ByteArrayInputStream("woxihuanni".getBytes());
IOUtils.copyBytes(bis, outputStream, 1024);
IOUtils.closeStream(bis);

} catch (Exception e) {
e.printStackTrace();
}

}

//    @Test
public void testFSRead() {

try {
Configuration conf = new Configuration();
conf.set("fs.defaultFS", "hdfs://master:8020");
FileSystem fileSystem = FileSystem.get(conf);
FSDataInputStream open = fileSystem.open(new Path("/user/master/my.cnf"));
IOUtils.copyBytes(open, System.out, 1024);
IOUtils.closeStream(open);
} catch (IOException e) {
e.printStackTrace();
}

}

//    @Test
public void testFSWrite() {
try {
Configuration conf = new Configuration();
conf.set("fs.defaultFS", "hdfs://master:8020");
FileSystem fileSystem = FileSystem.get(conf);
FSDataOutputStream fsDataOutputStream = fileSystem.create(new Path("/user/master/my.cnf"));
ByteArrayInputStream bis = new ByteArrayInputStream("woxihuanni henjiule".getBytes());
IOUtils.copyBytes(bis, fsDataOutputStream, 1024);
IOUtils.closeStream(bis);
} catch (IOException e) {
e.printStackTrace();
}

}

@Test
public void testRemove() {
try {
Configuration conf = new Configuration();
conf.set("fs.defaultFS", "hdfs://master:8020");
FileSystem fileSystem = FileSystem.get(conf);
boolean flag = fileSystem.deleteOnExit(new Path("/user/master/my.cnf"));
System.out.println(flag);
} catch (IOException e) {
e.printStackTrace();
}

}
}
内容来自用户分享和网络整理,不保证内容的准确性,如有侵权内容,可联系管理员处理 点击这里给我发消息
标签: