windows下eclipse非插件连接Hadoop集群

  相信有很多人正在学习Hadoop，因此在学习的过程中很多人希望通过windows下的eclipse或者myeclipse去连接远程的集群，那大家可能就会想到用Hadoop提供的插件了，插件的具体安装过程我就不多说了，因为网上有各种各样的教程。但是该方法有个特定的缺陷，就是在运行的时候必须点击运行run on Hadoop，在一般情况下是没有任何问题的，但是在某些特定的工程里，大家就不可能去点run on hadoop。

  现在我要讲的就是怎样可以不用Hadoop给的插件去连接Hadoop集群，就像我们运行普通的Java程序。

  在开讲之前给大家普及一下点知识，eclipse里的插件是干什么用的？说的简单点就是为了方便，

这他妈的不是废话吗，不是为了方便弄插件干啥。方便那他方便在哪了，他的作用就是我在写某个方面的工程的时候不用自己去搭建整个项目的框架，它把该导入的包等都给你弄。然而其实我们不用插件，自己手动也可以完成。行了，废话少说下面教给大家怎么不用插件，eclipse连接Hadoop集群。

  第一、我们需要创建一个普通的Java工程。

  第二、导入我们需要的包，也就是Hadoop包，由于包太多，我把必要的包的目录放到文章的最后。（注意：我的Hadoop是2.3.0版本的）

  第三、要想连接Hadoop集群Configuration说这参数是举足轻重不为过的，这个参数配置不好，是不能连接集群的，这个参数的具体详解请自行查阅，在此不多做解释。接下来我们配置Configuration参数

/**
*configuration 参数配置函数  用于hadoop集群的连接
* @param HOST //hadoop集群master IP
* @param JOBTRACKER_PORT
* @param NAMENODE_PORT
* @return
* /
public static Configuration  getConf(String HOST,int JOBTRACKER_PORT,int NAMENODE_PORT){
Configuration conf=new Configuration();
conf.set("fs.defaultFS", "hdfs://"+HOST+":"+NAMENODE_PORT);
conf.set("yarn.nodemanager.aux-services","mapreduce_shuffle");
conf.set("yarn.resourcemanager.hostname", HOST);
conf.set("mapreduce.framework.name", "yarn");
return conf;
}

  第四、连接集群，啥都不说上代码

public int checkConnection(String HOST,int JOBTRACKER_PORT,int NAMENODE_PORT) throws IOException{
Configuration conf=HadoopUtils.getConf(HOST,JOBTRACKER_PORT,NAMENODE_PORT);
if(conf==null){
return 0;
}
//验证hdfs
FileSystem fs=FileSystem.get(conf);
boolean fsOnline=fs.exists(new Path("/"));
if(!fsOnline){
return 1;
}
//验证jobclent
InetSocketAddress jobTracker=new InetSocketAddress(HOST,JOBTRACKER_PORT);
JobClient jobclient=new JobClient(jobTracker,conf);
ClusterStatus status=jobclient.getClusterStatus();
if(!"RUNNING".equals(status.getJobTrackerStatus().toString())){
return 2;
}
//		集群验证成功
HadoopUtils.setFs(fs);
HadoopUtils.setconf(conf);
//      通过判断Hadoop.getConf()是否为null来确定是否已经配置过集群
return 3;

}

hadoop必要包名：

hadoop-2.3.0-cdh5.1.2\share\hadoop\yarn\lib\zookeeper-3.4.5-cdh5.1.2.jar

hadoop-2.3.0-cdh5.1.2\share\hadoop\yarn\lib\xz-1.0.jar

hadoop-2.3.0-cdh5.1.2\share\hadoop\yarn\lib\stax-api-1.0-2.jar

hadoop-2.3.0-cdh5.1.2\share\hadoop\yarn\lib\servlet-api-2.5.jar

hadoop-2.3.0-cdh5.1.2\share\hadoop\yarn\lib\protobuf-java-2.5.0.jar

hadoop-2.3.0-cdh5.1.2\share\hadoop\yarn\lib\log4j-1.2.17.jar

hadoop-2.3.0-cdh5.1.2\share\hadoop\yarn\lib\jsr305-1.3.9.jar

hadoop-2.3.0-cdh5.1.2\share\hadoop\yarn\lib\jline-0.9.94.jar

hadoop-2.3.0-cdh5.1.2\share\hadoop\yarn\lib\jetty-util-6.1.26.jar

hadoop-2.3.0-cdh5.1.2\share\hadoop\yarn\lib\jetty-6.1.26.jar

hadoop-2.3.0-cdh5.1.2\share\hadoop\yarn\lib\jettison-1.1.jar

hadoop-2.3.0-cdh5.1.2\share\hadoop\yarn\lib\jersey-server-1.9.jar

hadoop-2.3.0-cdh5.1.2\share\hadoop\yarn\lib\jersey-json-1.9.jar

hadoop-2.3.0-cdh5.1.2\share\hadoop\yarn\lib\jersey-guice-1.9.jar

hadoop-2.3.0-cdh5.1.2\share\hadoop\yarn\lib\jersey-core-1.9.jar

hadoop-2.3.0-cdh5.1.2\share\hadoop\yarn\lib\jersey-client-1.9.jar

hadoop-2.3.0-cdh5.1.2\share\hadoop\yarn\lib\jaxb-impl-2.2.3-1.jar

hadoop-2.3.0-cdh5.1.2\share\hadoop\yarn\lib\jaxb-api-2.2.2.jar

hadoop-2.3.0-cdh5.1.2\share\hadoop\yarn\lib\javax.inject-1.jar

hadoop-2.3.0-cdh5.1.2\share\hadoop\yarn\lib\jackson-xc-1.8.8.jar

hadoop-2.3.0-cdh5.1.2\share\hadoop\yarn\lib\jackson-mapper-asl-1.8.8.jar

hadoop-2.3.0-cdh5.1.2\share\hadoop\yarn\lib\jackson-jaxrs-1.8.8.jar

hadoop-2.3.0-cdh5.1.2\share\hadoop\yarn\lib\jackson-core-asl-1.8.8.jar

hadoop-2.3.0-cdh5.1.2\share\hadoop\yarn\lib\guice-servlet-3.0.jar

hadoop-2.3.0-cdh5.1.2\share\hadoop\yarn\lib\guice-3.0.jar

hadoop-2.3.0-cdh5.1.2\share\hadoop\yarn\lib\guava-11.0.2.jar

hadoop-2.3.0-cdh5.1.2\share\hadoop\yarn\lib\commons-logging-1.1.3.jar

hadoop-2.3.0-cdh5.1.2\share\hadoop\yarn\lib\commons-lang-2.6.jar

hadoop-2.3.0-cdh5.1.2\share\hadoop\yarn\lib\commons-io-2.4.jar

hadoop-2.3.0-cdh5.1.2\share\hadoop\yarn\lib\commons-httpclient-3.1.jar

hadoop-2.3.0-cdh5.1.2\share\hadoop\yarn\lib\commons-compress-1.4.1.jar

hadoop-2.3.0-cdh5.1.2\share\hadoop\yarn\lib\commons-codec-1.4.jar

hadoop-2.3.0-cdh5.1.2\share\hadoop\yarn\lib\commons-cli-1.2.jar

hadoop-2.3.0-cdh5.1.2\share\hadoop\yarn\lib\asm-3.2.jar

hadoop-2.3.0-cdh5.1.2\share\hadoop\yarn\lib\aopalliance-1.0.jar

hadoop-2.3.0-cdh5.1.2\share\hadoop\yarn\lib\activation-1.1.jar

hadoop-2.3.0-cdh5.1.2\share\hadoop\yarn\hadoop-yarn-server-web-proxy-2.3.0-cdh5.1.2.jar

hadoop-2.3.0-cdh5.1.2\share\hadoop\yarn\hadoop-yarn-server-resourcemanager-2.3.0-cdh5.1.2.jar

hadoop-2.3.0-cdh5.1.2\share\hadoop\yarn\hadoop-yarn-server-nodemanager-2.3.0-cdh5.1.2.jar

hadoop-2.3.0-cdh5.1.2\share\hadoop\yarn\hadoop-yarn-server-common-2.3.0-cdh5.1.2.jar

hadoop-2.3.0-cdh5.1.2\share\hadoop\yarn\hadoop-yarn-common-2.3.0-cdh5.1.2.jar

hadoop-2.3.0-cdh5.1.2\share\hadoop\yarn\hadoop-yarn-client-2.3.0-cdh5.1.2.jar

hadoop-2.3.0-cdh5.1.2\share\hadoop\yarn\hadoop-yarn-applications-unmanaged-am-launcher-2.3.0-cdh5.1.2.jar

hadoop-2.3.0-cdh5.1.2\share\hadoop\yarn\hadoop-yarn-applications-distributedshell-2.3.0-cdh5.1.2.jar

hadoop-2.3.0-cdh5.1.2\share\hadoop\yarn\hadoop-yarn-api-2.3.0-cdh5.1.2.jar

hadoop-2.3.0-cdh5.1.2\share\hadoop\hdfs\lib\xmlenc-0.52.jar

hadoop-2.3.0-cdh5.1.2\share\hadoop\hdfs\lib\servlet-api-2.5.jar

hadoop-2.3.0-cdh5.1.2\share\hadoop\hdfs\lib\protobuf-java-2.5.0.jar

hadoop-2.3.0-cdh5.1.2\share\hadoop\hdfs\lib\netty-3.6.2.Final.jar

hadoop-2.3.0-cdh5.1.2\share\hadoop\hdfs\lib\log4j-1.2.17.jar

hadoop-2.3.0-cdh5.1.2\share\hadoop\hdfs\lib\jsr305-1.3.9.jar

hadoop-2.3.0-cdh5.1.2\share\hadoop\hdfs\lib\jsp-api-2.1.jar

hadoop-2.3.0-cdh5.1.2\share\hadoop\hdfs\lib\jetty-util-6.1.26.jar

hadoop-2.3.0-cdh5.1.2\share\hadoop\hdfs\lib\jetty-6.1.26.jar

hadoop-2.3.0-cdh5.1.2\share\hadoop\hdfs\lib\jersey-server-1.9.jar

hadoop-2.3.0-cdh5.1.2\share\hadoop\hdfs\lib\jersey-core-1.9.jar

hadoop-2.3.0-cdh5.1.2\share\hadoop\hdfs\lib\jasper-runtime-5.5.23.jar

hadoop-2.3.0-cdh5.1.2\share\hadoop\hdfs\lib\jackson-mapper-asl-1.8.8.jar

hadoop-2.3.0-cdh5.1.2\share\hadoop\hdfs\lib\jackson-core-asl-1.8.8.jar

hadoop-2.3.0-cdh5.1.2\share\hadoop\hdfs\lib\guava-11.0.2.jar

hadoop-
4000
2.3.0-cdh5.1.2\share\hadoop\hdfs\lib\commons-logging-1.1.3.jar

hadoop-2.3.0-cdh5.1.2\share\hadoop\hdfs\lib\commons-lang-2.6.jar

hadoop-2.3.0-cdh5.1.2\share\hadoop\hdfs\lib\commons-io-2.4.jar

hadoop-2.3.0-cdh5.1.2\share\hadoop\hdfs\lib\commons-el-1.0.jar

hadoop-2.3.0-cdh5.1.2\share\hadoop\hdfs\lib\commons-daemon-1.0.13.jar

hadoop-2.3.0-cdh5.1.2\share\hadoop\hdfs\lib\commons-codec-1.4.jar

hadoop-2.3.0-cdh5.1.2\share\hadoop\hdfs\lib\commons-cli-1.2.jar

hadoop-2.3.0-cdh5.1.2\share\hadoop\hdfs\lib\asm-3.2.jar

hadoop-2.3.0-cdh5.1.2\share\hadoop\hdfs\hadoop-hdfs-nfs-2.3.0-cdh5.1.2.jar

hadoop-2.3.0-cdh5.1.2\share\hadoop\hdfs\hadoop-hdfs-2.3.0-cdh5.1.2.jar

hadoop-2.3.0-cdh5.1.2\share\hadoop\mapreduce\lib\xz-1.0.jar

hadoop-2.3.0-cdh5.1.2\share\hadoop\mapreduce\lib\snappy-java-1.0.4.1.jar

hadoop-2.3.0-cdh5.1.2\share\hadoop\mapreduce\lib\protobuf-java-2.5.0.jar

hadoop-2.3.0-cdh5.1.2\share\hadoop\mapreduce\lib\paranamer-2.3.jar

hadoop-2.3.0-cdh5.1.2\share\hadoop\mapreduce\lib\netty-3.6.2.Final.jar

hadoop-2.3.0-cdh5.1.2\share\hadoop\mapreduce\lib\log4j-1.2.17.jar

hadoop-2.3.0-cdh5.1.2\share\hadoop\mapreduce\lib\junit-4.11.jar

hadoop-2.3.0-cdh5.1.2\share\hadoop\mapreduce\lib\jersey-server-1.9.jar

hadoop-2.3.0-cdh5.1.2\share\hadoop\mapreduce\lib\jersey-guice-1.9.jar

hadoop-2.3.0-cdh5.1.2\share\hadoop\mapreduce\lib\jersey-core-1.9.jar

hadoop-2.3.0-cdh5.1.2\share\hadoop\mapreduce\lib\javax.inject-1.jar

hadoop-2.3.0-cdh5.1.2\share\hadoop\mapreduce\lib\jackson-mapper-asl-1.8.8.jar

hadoop-2.3.0-cdh5.1.2\share\hadoop\mapreduce\lib\jackson-core-asl-1.8.8.jar

hadoop-2.3.0-cdh5.1.2\share\hadoop\mapreduce\lib\hamcrest-core-1.3.jar

hadoop-2.3.0-cdh5.1.2\share\hadoop\mapreduce\lib\hadoop-annotations-2.3.0-cdh5.1.2.jar

hadoop-2.3.0-cdh5.1.2\share\hadoop\mapreduce\lib\guice-servlet-3.0.jar

hadoop-2.3.0-cdh5.1.2\share\hadoop\mapreduce\lib\guice-3.0.jar

hadoop-2.3.0-cdh5.1.2\share\hadoop\mapreduce\lib\commons-io-2.4.jar

hadoop-2.3.0-cdh5.1.2\share\hadoop\mapreduce\lib\commons-compress-1.4.1.jar

hadoop-2.3.0-cdh5.1.2\share\hadoop\mapreduce\lib\avro-1.7.5-cdh5.1.2.jar

hadoop-2.3.0-cdh5.1.2\share\hadoop\mapreduce\lib\asm-3.2.jar

hadoop-2.3.0-cdh5.1.2\share\hadoop\mapreduce\lib\aopalliance-1.0.jar

hadoop-2.3.0-cdh5.1.2\share\hadoop\mapreduce\hadoop-mapreduce-client-shuffle-2.3.0-cdh5.1.2.jar

hadoop-2.3.0-cdh5.1.2\share\hadoop\mapreduce\hadoop-mapreduce-client-jobclient-2.3.0-cdh5.1.2.jar

hadoop-2.3.0-cdh5.1.2\share\hadoop\mapreduce\hadoop-mapreduce-client-hs-plugins-2.3.0-cdh5.1.2.jar

hadoop-2.3.0-cdh5.1.2\share\hadoop\mapreduce\hadoop-mapreduce-client-hs-2.3.0-cdh5.1.2.jar

hadoop-2.3.0-cdh5.1.2\share\hadoop\mapreduce\hadoop-mapreduce-client-core-2.3.0-cdh5.1.2.jar

hadoop-2.3.0-cdh5.1.2\share\hadoop\mapreduce\hadoop-mapreduce-client-common-2.3.0-cdh5.1.2.jar

hadoop-2.3.0-cdh5.1.2\share\hadoop\mapreduce\hadoop-mapreduce-client-app-2.3.0-cdh5.1.2.jar

hadoop-2.3.0-cdh5.1.2\share\hadoop\common\lib\zookeeper-3.4.5-cdh5.1.2.jar

hadoop-2.3.0-cdh5.1.2\share\hadoop\common\lib\xz-1.0.jar

hadoop-2.3.0-cdh5.1.2\share\hadoop\common\lib\xmlenc-0.52.jar

hadoop-2.3.0-cdh5.1.2\share\hadoop\common\lib\stax-api-1.0-2.jar

hadoop-2.3.0-cdh5.1.2\share\hadoop\common\lib\snappy-java-1.0.4.1.jar

hadoop-2.3.0-cdh5.1.2\share\hadoop\common\lib\slf4j-log4j12-1.7.5.jar

hadoop-2.3.0-cdh5.1.2\share\hadoop\common\lib\slf4j-api-1.7.5.jar

hadoop-2.3.0-cdh5.1.2\share\hadoop\common\lib\servlet-api-2.5.jar

hadoop-2.3.0-cdh5.1.2\share\hadoop\common\lib\protobuf-java-2.5.0.jar

hadoop-2.3.0-cdh5.1.2\share\hadoop\common\lib\paranamer-2.3.jar

hadoop-2.3.0-cdh5.1.2\share\hadoop\common\lib\netty-3.6.2.Final.jar

hadoop-2.3.0-cdh5.1.2\share\hadoop\common\lib\mockito-all-1.8.5.jar

hadoop-2.3.0-cdh5.1.2\share\hadoop\common\lib\log4j-1.2.17.jar

hadoop-2.3.0-cdh5.1.2\share\hadoop\common\lib\junit-4.11.jar

hadoop-2.3.0-cdh5.1.2\share\hadoop\common\lib\jsr305-1.3.9.jar

hadoop-2.3.0-cdh5.1.2\share\hadoop\common\lib\jsp-api-2.1.jar

hadoop-2.3.0-cdh5.1.2\share\hadoop\common\lib\jsch-0.1.42.jar

hadoop-2.3.0-cdh5.1.2\share\hadoop\common\lib\jetty-util-6.1.26.jar

hadoop-2.3.0-cdh5.1.2\share\hadoop\common\lib\jetty-6.1.26.jar

hadoop-2.3.0-cdh5.1.2\share\hadoop\common\lib\jettison-1.1.jar

hadoop-2.3.0-cdh5.1.2\share\hadoop\common\lib\jets3t-0.9.0.jar

hadoop-2.3.0-cdh5.1.2\share\hadoop\common\lib\jersey-server-1.9.jar

hadoop-2.3.0-cdh5.1.2\share\hadoop\common\lib\jersey-json-1.9.jar

hadoop-2.3.0-cdh5.1.2\share\hadoop\common\lib\jersey-core-1.9.jar

hadoop-2.3.0-cdh5.1.2\share\hadoop\common\lib\jaxb-impl-2.2.3-1.jar

hadoop-2.3.0-cdh5.1.2\share\hadoop\common\lib\jaxb-api-2.2.2.jar

hadoop-2.3.0-cdh5.1.2\share\hadoop\common\lib\java-xmlbuilder-0.4.jar

hadoop-2.3.0-cdh5.1.2\share\hadoop\common\lib\jasper-runtime-5.5.23.jar

hadoop-2.3.0-cdh5.1.2\share\hadoop\common\lib\jasper-compiler-5.5.23.jar

hadoop-2.3.0-cdh5.1.2\share\hadoop\common\lib\jackson-xc-1.8.8.jar

hadoop-2.3.0-cdh5.1.2\share\hadoop\common\lib\jackson-mapper-asl-1.8.8.jar

hadoop-2.3.0-cdh5.1.2\share\hadoop\common\lib\jackson-jaxrs-1.8.8.jar

hadoop-2.3.0-cdh5.1.2\share\hadoop\common\lib\jackson-core-asl-1.8.8.jar

hadoop-2.3.0-cdh5.1.2\share\hadoop\common\lib\httpcore-4.2.5.jar

hadoop-2.3.0-cdh5.1.2\share\hadoop\common\lib\httpclient-4.2.5.jar

hadoop-2.3.0-cdh5.1.2\share\hadoop\common\lib\hamcrest-core-1.3.jar

hadoop-2.3.0-cdh5.1.2\share\hadoop\common\lib\hadoop-auth-2.3.0-cdh5.1.2.jar

hadoop-2.3.0-cdh5.1.2\share\hadoop\common\lib\hadoop-annotations-2.3.0-cdh5.1.2.jar

hadoop-2.3.0-cdh5.1.2\share\hadoop\common\lib\guava-11.0.2.jar

hadoop-2.3.0-cdh5.1.2\share\hadoop\common\lib\commons-net-3.1.jar

hadoop-2.3.0-cdh5.1.2\share\hadoop\common\lib\commons-math3-3.1.1.jar

hadoop-2.3.0-cdh5.1.2\share\hadoop\common\lib\commons-logging-1.1.3.jar

hadoop-2.3.0-cdh5.1.2\share\hadoop\common\lib\commons-lang-2.6.jar

hadoop-2.3.0-cdh5.1.2\share\hadoop\common\lib\commons-io-2.4.jar

hadoop-2.3.0-cdh5.1.2\share\hadoop\common\lib\commons-httpclient-3.1.jar

hadoop-2.3.0-cdh5.1.2\share\hadoop\common\lib\commons-el-1.0.jar

hadoop-2.3.0-cdh5.1.2\share\hadoop\common\lib\commons-digester-1.8.jar

hadoop-2.3.0-cdh5.1.2\share\hadoop\common\lib\commons-configuration-1.6.jar

hadoop-2.3.0-cdh5.1.2\share\hadoop\common\lib\commons-compress-1.4.1.jar

hadoop-2.3.0-cdh5.1.2\share\hadoop\common\lib\commons-collections-3.2.1.jar

hadoop-2.3.0-cdh5.1.2\share\hadoop\common\lib\commons-codec-1.4.jar

hadoop-2.3.0-cdh5.1.2\share\hadoop\common\lib\commons-cli-1.2.jar

hadoop-2.3.0-cdh5.1.2\share\hadoop\common\lib\commons-beanutils-core-1.8.0.jar

hadoop-2.3.0-cdh5.1.2\share\hadoop\common\lib\commons-beanutils-1.7.0.jar

hadoop-2.3.0-cdh5.1.2\share\hadoop\common\lib\avro-1.7.5-cdh5.1.2.jar

hadoop-2.3.0-cdh5.1.2\share\hadoop\common\lib\asm-3.2.jar

hadoop-2.3.0-cdh5.1.2\share\hadoop\common\lib\api-util-1.0.0-M20.jar

hadoop-2.3.0-cdh5.1.2\share\hadoop\common\lib\api-asn1-api-1.0.0-M20.jar

hadoop-2.3.0-cdh5.1.2\share\hadoop\common\lib\apacheds-kerberos-codec-2.0.0-M15.jar

hadoop-2.3.0-cdh5.1.2\share\hadoop\common\lib\apacheds-i18n-2.0.0-M15.jar

hadoop-2.3.0-cdh5.1.2\share\hadoop\common\lib\activation-1.1.jar

hadoop-2.3.0-cdh5.1.2\share\hadoop\common\hadoop-nfs-2.3.0-cdh5.1.2.jar

hadoop-2.3.0-cdh5.1.2\share\hadoop\common\hadoop-common-2.3.0-cdh5.1.2.jar