运行wordcount程序报 Input path does not exist 错
2017-11-02 16:07
288 查看
在Windows上部署了hadoop,在执行官网给出的wordcount程序是遇到一下报错,
Exception in thread "main" org.apache.hadoop.mapreduce.lib.input.InvalidInputException: Input path does not exist: file:/input
at org.apache.hadoop.mapreduce.lib.input.FileInputFormat.singleThreadedListStatus(FileInputFormat.java:323)
at org.apache.hadoop.mapreduce.lib.input.FileInputFormat.listStatus(FileInputFormat.java:265)
at org.apache.hadoop.mapreduce.lib.input.FileInputFormat.getSplits(FileInputFormat.java:387)
at org.apache.hadoop.mapreduce.JobSubmitter.writeNewSplits(JobSubmitter.java:301)
at org.apache.hadoop.mapreduce.JobSubmitter.writeSplits(JobSubmitter.java:318)
at org.apache.hadoop.mapreduce.JobSubmitter.submitJobInternal(JobSubmitter.java:196)
at org.apache.hadoop.mapreduce.Job$10.run(Job.java:1290)
at org.apache.hadoop.mapreduce.Job$10.run(Job.java:1287)
at java.security.AccessController.doPrivileged(Native Method)
at javax.security.auth.Subject.doAs(Unknown Source)
at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1746)
at org.apache.hadoop.mapreduce.Job.submit(Job.java:1287)
at org.apache.hadoop.mapreduce.Job.waitForCompletion(Job.java:1308)
at WordCount.main(WordCount.java:61)
搜了下,网上主要说是程序在本地文件系统查找输入目录,没有到HDFS中查找目录,配置如下:
Configuration conf = new Configuration();
conf.addResource(new Path("D:/hadoop-2.7.4/etc/hadoop/core-site.xml"));
conf.addResource(new Path("D:/hadoop-2.7.4/etc/hadoop/hdfs-site.xml"));
一下是在官网wordcount程序上经过修改可以在eclipse中执行的程序。
import java.io.IOException;
import java.util.StringTokenizer;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.io.IntWritable;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapreduce.Job;
import org.apache.hadoop.mapreduce.Mapper;
import org.apache.hadoop.mapreduce.Reducer;
import org.apache.hadoop.mapreduce.lib.input.FileInputFormat;
import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;
public class WordCount {
public static class TokenizerMapper
extends Mapper<Object, Text, Text, IntWritable>{
private final static IntWritable one = new IntWritable(1);
private Text word = new Text();
public void map(Object key, Text value, Context context
) throws IOException, InterruptedException {
StringTokenizer itr = new StringTokenizer(value.toString());
while (itr.hasMoreTokens()) {
word.set(itr.nextToken());
context.write(word, one);
}
}
}
public static class IntSumReducer
extends Reducer<Text,IntWritable,Text,IntWritable> {
private IntWritable result = new IntWritable();
public void reduce(Text key, Iterable<IntWritable> values,
Context context
) throws IOException, InterruptedException {
int sum = 0;
for (IntWritable val : values) {
sum += val.get();
}
result.set(sum);
context.write(key, result);
}
}
public static void main(String[] args) throws Exception {
Configuration conf = new Configuration();
conf.addResource(new Path("D:/hadoop-2.7.4/etc/hadoop/core-site.xml"));
conf.addResource(new Path("D:/hadoop-2.7.4/etc/hadoop/hdfs-site.xml"));
Job job = Job.getInstance(conf, "word count");
job.setJarByClass(WordCount.class);
job.setMapperClass(TokenizerMapper.class);
job.setCombinerClass(IntSumReducer.class);
job.setReducerClass(IntSumReducer.class);
job.setOutputKeyClass(Text.class);
job.setOutputValueClass(IntWritable.class);
// FileInputFormat.addInputPath(job, new Path(args[0]));
FileInputFormat.addInputPath(job, new Path("/input"));
// FileOutputFormat.setOutputPath(job, new Path(args[1]));
FileOutputFormat.setOutputPath(job, new Path("/output"));
System.exit(job.waitForCompletion(true) ? 0 : 1);
}
}
执行结果如下:
Exception in thread "main" org.apache.hadoop.mapreduce.lib.input.InvalidInputException: Input path does not exist: file:/input
at org.apache.hadoop.mapreduce.lib.input.FileInputFormat.singleThreadedListStatus(FileInputFormat.java:323)
at org.apache.hadoop.mapreduce.lib.input.FileInputFormat.listStatus(FileInputFormat.java:265)
at org.apache.hadoop.mapreduce.lib.input.FileInputFormat.getSplits(FileInputFormat.java:387)
at org.apache.hadoop.mapreduce.JobSubmitter.writeNewSplits(JobSubmitter.java:301)
at org.apache.hadoop.mapreduce.JobSubmitter.writeSplits(JobSubmitter.java:318)
at org.apache.hadoop.mapreduce.JobSubmitter.submitJobInternal(JobSubmitter.java:196)
at org.apache.hadoop.mapreduce.Job$10.run(Job.java:1290)
at org.apache.hadoop.mapreduce.Job$10.run(Job.java:1287)
at java.security.AccessController.doPrivileged(Native Method)
at javax.security.auth.Subject.doAs(Unknown Source)
at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1746)
at org.apache.hadoop.mapreduce.Job.submit(Job.java:1287)
at org.apache.hadoop.mapreduce.Job.waitForCompletion(Job.java:1308)
at WordCount.main(WordCount.java:61)
搜了下,网上主要说是程序在本地文件系统查找输入目录,没有到HDFS中查找目录,配置如下:
Configuration conf = new Configuration();
conf.addResource(new Path("D:/hadoop-2.7.4/etc/hadoop/core-site.xml"));
conf.addResource(new Path("D:/hadoop-2.7.4/etc/hadoop/hdfs-site.xml"));
一下是在官网wordcount程序上经过修改可以在eclipse中执行的程序。
import java.io.IOException;
import java.util.StringTokenizer;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.io.IntWritable;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapreduce.Job;
import org.apache.hadoop.mapreduce.Mapper;
import org.apache.hadoop.mapreduce.Reducer;
import org.apache.hadoop.mapreduce.lib.input.FileInputFormat;
import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat;
public class WordCount {
public static class TokenizerMapper
extends Mapper<Object, Text, Text, IntWritable>{
private final static IntWritable one = new IntWritable(1);
private Text word = new Text();
public void map(Object key, Text value, Context context
) throws IOException, InterruptedException {
StringTokenizer itr = new StringTokenizer(value.toString());
while (itr.hasMoreTokens()) {
word.set(itr.nextToken());
context.write(word, one);
}
}
}
public static class IntSumReducer
extends Reducer<Text,IntWritable,Text,IntWritable> {
private IntWritable result = new IntWritable();
public void reduce(Text key, Iterable<IntWritable> values,
Context context
) throws IOException, InterruptedException {
int sum = 0;
for (IntWritable val : values) {
sum += val.get();
}
result.set(sum);
context.write(key, result);
}
}
public static void main(String[] args) throws Exception {
Configuration conf = new Configuration();
conf.addResource(new Path("D:/hadoop-2.7.4/etc/hadoop/core-site.xml"));
conf.addResource(new Path("D:/hadoop-2.7.4/etc/hadoop/hdfs-site.xml"));
Job job = Job.getInstance(conf, "word count");
job.setJarByClass(WordCount.class);
job.setMapperClass(TokenizerMapper.class);
job.setCombinerClass(IntSumReducer.class);
job.setReducerClass(IntSumReducer.class);
job.setOutputKeyClass(Text.class);
job.setOutputValueClass(IntWritable.class);
// FileInputFormat.addInputPath(job, new Path(args[0]));
FileInputFormat.addInputPath(job, new Path("/input"));
// FileOutputFormat.setOutputPath(job, new Path(args[1]));
FileOutputFormat.setOutputPath(job, new Path("/output"));
System.exit(job.waitForCompletion(true) ? 0 : 1);
}
}
执行结果如下:
相关文章推荐
- hadoop机群 运行wordcount出现 Input path does not exist: hdfs://ns1/user/root/a.txt
- Hadoop错误2_Hadoop伪分布式下,运行WordCount时,Input path does not exist错误原因
- 错误:org.apache.hadoop.mapreduce.lib.input.InvalidInputException: Input path does not exist: hdfs://localhost/
- org.apache.hadoop.mapred.InvalidInputException: Input path does not exist:
- 解决Spark读取Hive分区表出现Input path does not exist的问题
- sc.textFile("file:///home/spark/data.txt") Input path does not exist解决方法——submit 加参数 --master local 即可解决
- eclipse编译hadoop错误:Input path does not exist: hdfs://192.168.184.21:9001/user/xxxx/input
- 解决问题:Input path does not exist: hdfs://localhost:9000/input
- org.apache.hadoop.mapreduce.lib.input.InvalidInputException: Input path does not exist:
- Hadoop问题:Input path does not exist: hdfs://Master:9000/user/hadoop/input
- org.apache.hadoop.mapreduce.lib.input.InvalidInputException: Input path does not exist:
- MapReduce程序运行错误:The auxService: mapreduce_shuffle does not exist
- org.apache.hadoop.mapreduce.lib.input.InvalidInputException: Input path does not exist:
- "main" org.apache.hadoop.mapred.InvalidInputException: Input path does not exist: file
- mapreduce error:input path does not exist 的解决方法
- org.apache.hadoop.mapreduce.lib.input.InvalidInputException:input path does not exist:hdfs://10.0.2.
- org.apache.hadoop.mapred.InvalidInputException: Input path does not exist问题
- Input path does not exist: file:/.......
- 在伪分布式环境下通过宿主机上的eclipse运行WordCount小程序出现的问题小结
- Hadoop 1.x 使用eclipse集成环境运行WordCount程序