读取nutch爬取内容方法

读取nutch内容有如下两种方法：

1 通过Nutch api SegmentReader读取。

         public Content readSegment(String segPath,String url){
 

              

            Text key= new Text(url);  

            Path path= new Path(segPath);  

            Content content = null;  

      

            ArrayList<Writable> parsedLst = null;  

            Map<String,List<Writable>> results=new HashMap<String, List<Writable>>();  

            SegmentReader reader= new SegmentReader(configuration,true,true,true,true,true,true);  

            try {  

                reader.get(path, key, new StringWriter(), results);  

                parsedLst=(ArrayList<Writable>) results.get("co");  

                Iterator<Writable> parseIter=parsedLst.iterator();  

                while(parseIter.hasNext()){  

                    content=(Content) parseIter.next();  

                }  

            } catch (Exception e) {  

                e.printStackTrace();  

            }  

      

            return content;  

        }  

 

  2 通过SequenceFile 读取

        public static void main(String[] args) throws IOException { 

            args=new String[]{"D:\\nutchv\\nutch12\\apache-nutch-1.2\\data\\csdn2\\segments\\20140904104348"};   
            Configuration conf = NutchConfiguration.create();       

            Options opts = new Options();       

            GenericOptionsParser parser = new GenericOptionsParser(conf, opts, args);       

            String[] remainingArgs = parser.getRemainingArgs();     

            FileSystem fs = FileSystem.get(conf);

            String segment = remainingArgs[0];

            Path file = new Path(segment, Content.DIR_NAME + "/part-00000/data");

            SequenceFile.Reader reader = new SequenceFile.Reader(fs, file, conf);

            Text key = new Text();

            Content content = new Content();

            // Loop through sequence files
            while (reader.next(key, content)) {

                try {

                    System.out.write(content.getContent(), 0,

                            content.getContent().length);

                } catch (Exception e) {

                }

            }
 

        }