springMVC + hadoop + httpclient 文件上传请求直接写入hdfs

1、首先是一个基于httpclient的java 应用程序，代码在这篇文章的开头：点击打开链接

2、我们首先写一个基于springMVC框架的简单接收请求上传的文件保存本地文件系统的demo，程序代码在这：点击打开链接，不过因为之后与hadoop所以来的jar包同时使用，加上有一些三方jar包可能会冲突，这里我们简化这个demo里面maven所依赖的选项，更新的pom文件如下：

<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd"> <modelVersion>4.0.0</modelVersion>
<groupId>springmvx</groupId>
<artifactId>springmvx</artifactId>
<version>0.0.1-SNAPSHOT</version>
<packaging>war</packaging>
<build>
<sourceDirectory>src</sourceDirectory>
<plugins>
<plugin>
<artifactId>maven-compiler-plugin</artifactId>
<version>3.3</version>
<configuration>
<source>1.7</source>
<target>1.7</target>
</configuration>
</plugin>
<plugin>
<artifactId>maven-war-plugin</artifactId>
<version>2.6</version>
<configuration>
<warSourceDirectory>WebContent</warSourceDirectory>
<failOnMissingWebXml>false</failOnMissingWebXml>
</configuration>
</plugin>
</plugins>
</build>
<dependencies>
<dependency>
<groupId>jstl</groupId>
<artifactId>jstl</artifactId>
<version>1.2</version>
</dependency>
<dependency>
<groupId>org.springframework</groupId>
<artifactId>spring-webmvc</artifactId>
<version>3.1.1.RELEASE</version>
</dependency>
<dependency>
<groupId>org.springframework</groupId>
<artifactId>spring-web</artifactId>
<version>3.1.1.RELEASE</version>
</dependency>
<dependency>
<groupId>commons-fileupload</groupId>
<artifactId>commons-fileupload</artifactId>
<version>1.3.1</version>
</dependency>
<dependency>
<groupId>jdk.tools</groupId>
<artifactId>jdk.tools</artifactId>
<version>1.7</version>
<scope>system</scope>
<systemPath>${JAVA_HOME}/lib/tools.jar</systemPath>
</dependency>
<dependency>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-client</artifactId>
<version>2.6.0</version>
</dependency>
</dependencies>
</project>

3、我们将springMVC控制器里的代码稍加修改，改为直接上传文件到hdfs，而不是本地文件系统，代码如下：

package net.spring.controller;

import java.io.File;
import java.io.IOException;
import java.util.Iterator;
import javax.servlet.http.HttpServletRequest;
import javax.servlet.http.HttpServletResponse;

import org.apache.commons.fileupload.disk.DiskFileItem;
import org.springframework.stereotype.Controller;
import org.springframework.web.bind.annotation.RequestMapping;
import org.springframework.web.multipart.MultipartFile;
import org.springframework.web.multipart.MultipartHttpServletRequest;
import org.springframework.web.multipart.commons.CommonsMultipartFile;
import org.springframework.web.multipart.commons.CommonsMultipartResolver;
import org.springframework.web.servlet.ModelAndView;

import com.busymonkey.hadoop.*;

@Controller
public class HelloWorldController {

@RequestMapping("/hello")
public ModelAndView hello() {
String message = "Hello World, Spring 3.0!";
System.out.println(message);
return new ModelAndView("hello1", "message", message);
}
@RequestMapping("/upload")
public String upLoad(HttpServletRequest request, HttpServletResponse response)
throws IllegalStateException, IOException{
//解析器解析request的上下文
CommonsMultipartResolver multipartResolver = new CommonsMultipartResolver(request.getSession().getServletContext());
//先判断request中是否包涵multipart类型的数据，
if(multipartResolver.isMultipart(request)) {
//再将request中的数据转化成multipart类型的数据
MultipartHttpServletRequest multiRequest = (MultipartHttpServletRequest) request;
Iterator<String> iter = multiRequest.getFileNames();
while(iter.hasNext()) {
MultipartFile file = multiRequest.getFile((String)iter.next());
if(file != null) {
CommonsMultipartFile cf= (CommonsMultipartFile)file;
DiskFileItem fi = (DiskFileItem)cf.getFileItem();
File inputFile = fi.getStoreLocation();
HdfsFileSystem.createFile(inputFile, "hdfs://192.168.241.149:9000/upload/1.mp4");
}
}
}
return "/hello1";
}
}

4、同时我们再新建一个hdfs的工具类，在上面的类中有引用，代码如下：

package com.busymonkey.hadoop;

import java.io.BufferedInputStream;
import java.io.File;
import java.io.FileInputStream;
import java.io.IOException;
import java.io.InputStream;
import java.net.URI;

import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FSDataOutputStream;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.io.IOUtils;

public class HdfsFileSystem {
/**
* 按路径上传文件到hdfs
* @param conf
* @param local
* @param remote
* @throws IOException
*/
public static void copyFile(Configuration conf , String uri , String local, String remote) throws IOException {
FileSystem fs = FileSystem.get(URI.create(uri), conf);
fs.copyFromLocalFile(new Path(local), new Path(remote));
System.out.println("copy from: " + local + " to " + remote);
fs.close();
}

/**
* 按路径下载hdfs上的文件
* @param conf
* @param uri
* @param remote
* @param local
* @throws IOException
*/
public static void download(Configuration conf , String uri ,String remote, String local) throws IOException {
Path path = new Path(remote);
FileSystem fs = FileSystem.get(URI.create(uri), conf);
fs.copyToLocalFile(path, new Path(local));
System.out.println("download: from" + remote + " to " + local);
fs.close();
}

/**
* File对象上传到hdfs
* @param conf
* @param uri
* @param remote
* @param local
* @throws IOException
*/
public static void createFile(File localPath, String hdfsPath) throws IOException {
InputStream in = null;
try {
Configuration conf = new Configuration();
FileSystem fileSystem = FileSystem.get(URI.create(hdfsPath), conf);
FSDataOutputStream out = fileSystem.create(new Path(hdfsPath));
in = new BufferedInputStream(new FileInputStream(localPath));
IOUtils.copyBytes(in, out, 4096, false);
out.hsync();
out.close();
System.out.println("create file in hdfs:" + hdfsPath);
} finally {
IOUtils.closeStream(in);
}
}
}

其他代码跟本地上传文件的demo是一样的。结果如下：



5、jsp页面提交文件上传请求，不使用 java 应用 httpclient，将 index.jsp 修改如下：

<%@ page language="java" contentType="text/html; charset=ISO-8859-1"  pageEncoding="ISO-8859-1"%>
<!DOCTYPE html PUBLIC "-//W3C//DTD HTML 4.01 Transitional//EN" "http://www.w3.org/TR/html4/loose.dtd">
<html>
<head>
<title>Spring 3.0 MVC demo</title>
</head>
<body>
<a href="hello.html">Say Hello</a>
<h3>upload :</h3>
Please choose the file :<br />
<form action="upload.html" method="post" enctype="multipart/form-data">
<input type="file" name="file" size="50" />
<br />
<input type="submit" value="Upload" />
</form>
</body>
</html>

6、将hello1.jsp 重命名为 result.jsp，代码如下：

<%@ page language="java" contentType="text/html; charset=ISO-8859-1"
pageEncoding="ISO-8859-1"%>
<!DOCTYPE html PUBLIC "-//W3C//DTD HTML 4.01 Transitional//EN" "http://www.w3.org/TR/html4/loose.dtd">
<html>
<head>
<meta http-equiv="Content-Type" content="text/html; charset=ISO-8859-1">
<title>result</title>
</head>
<body>
<p>success!</p>
</body>
</html>