使用springboot + lucence + ik +angularjs+jsoup 搜索自己博客

一。 爬取博客信息

      使用jsoup可以下载并且使用jquery语法来解析xml  这里先通过http://blog.csdn.net/liaomin416100569 右键查看源代码可以看到分类的源代码为：

      声明：该内容为个人学习使用 不做其他用途 如果侵权 请联系以下邮箱 416100569@qq.com

<div id="panel_Category" class="panel">
<ul class="panel_head"><span>文章分类</span></ul>
<ul class="panel_body">
<li>
<a href="/liaomin416100569/article/category/744627" onclick="_gaq.push(['_trackEvent','function', 'onclick', 'blog_articles_wenzhangfenlei']); ">activeMQ</a><span>(3)</span>
</li>
<li>
<a href="/liaomin416100569/article/category/643417" onclick="_gaq.push(['_trackEvent','function', 'onclick', 'blog_articles_wenzhangfenlei']); ">ant</a><span>(3)</span>
</li>
<li>

</ul>
</div>

获取所有的分类 通过jquery选择器  #panel_Category a 即可获取所有的超链接 通过 text()获取分类名称 通过 attr("href")获取分类url

同理 获取分类下的所有文章的标题 描述 内容 最后更新时间等  最后通过lucene生成索引  代码如下

》分类实体类

package cn.et.spilder;

public class Category {
private String name;
private String count;
private String url;
public String getName() {
return name;
}
public void setName(String name) {
this.name = name;
}
public String getCount() {
return count;
}
public void setCount(String count) {
this.count = count;
}
public String getUrl() {
return url;
}
public void setUrl(String url) {
this.url = url;
}

}

》文章实体

package cn.et.spilder;

public class Arcticle {
private String title;
private String description;
private String url;
private String createTime;
private String filePath;
public String getTitle() {
return title;
}
public void setTitle(String title) {
this.title = title;
}
public String getDescription() {
return description;
}
public void setDescription(String description) {
this.description = description;
}
public String getUrl() {
return url;
}
public void setUrl(String url) {
this.url = url;
}
public String getCreateTime() {
return createTime;
}
public void setCreateTime(String createTime) {
this.createTime = createTime;
}
public String getFilePath() {
return filePath;
}
public void setFilePath(String filePath) {
this.filePath = filePath;
}

}

》lucene工具类

package cn.et.index;

import java.io.File;
import java.io.IOException;
import java.io.StringReader;
import java.util.ArrayList;
import java.util.List;

import org.apache.commons.io.FileUtils;
import org.apache.lucene.analysis.Analyzer;
import org.apache.lucene.analysis.TokenStream;
import org.apache.lucene.document.Document;
import org.apache.lucene.document.TextField;
import org.apache.lucene.document.Field.Store;
import org.apache.lucene.index.DirectoryReader;
import org.apache.lucene.index.IndexWriter;
import org.apache.lucene.index.IndexWriterConfig;
import org.apache.lucene.queryparser.classic.MultiFieldQueryParser;
import org.apache.lucene.search.IndexSearcher;
import org.apache.lucene.search.Query;
import org.apache.lucene.search.ScoreDoc;
import org.apache.lucene.search.TopDocs;
import org.apache.lucene.search.highlight.Highlighter;
import org.apache.lucene.search.highlight.QueryScorer;
import org.apache.lucene.search.highlight.SimpleHTMLFormatter;
import org.apache.lucene.store.FSDirectory;
import org.apache.lucene.store.RAMDirectory;
import org.apache.lucene.util.Version;
import org.jsoup.Jsoup;
import org.wltea.analyzer.lucene.IKAnalyzer;

import cn.et.spilder.Arcticle;
import cn.et.spilder.CsdnSpilder;
/**
* lucene操作类
*时间：2017-6-30 上午09:14:48
*作者： LM
*联系方式：973465719@qq.com
*
*/
public class LuceneUtils {
//定义IK分词器
static Analyzer analyzer = new IKAnalyzer();
//批量处理内存存储的最大docuemnt个数
final static int dealCount=50;
//html格式化器 在关键字前后加上红色字体标签
static SimpleHTMLFormatter htmlFormatter=new SimpleHTMLFormatter("<font color=\"red\">","</font>");
/**
* 关闭writer对象
* @param iw
* @throws IOException
*/
public static void close(IndexWriter iw) throws IOException{
iw.close();
}
public static IndexWriter getRamWriter(RAMDirectory ramDirectory) throws IOException{
IndexWriterConfig iwc=new IndexWriterConfig(Version.LUCENE_47,analyzer);
IndexWriter iw=new IndexWriter(ramDirectory, iwc);
return iw;
}
public static IndexWriter getWriter(String dir) throws IOException{
//		FSDirectory fsDirectory= FSDirectory.open(Paths.get(dir));
FSDirectory fsDirectory= FSDirectory.open(new File(dir));
IndexWriterConfig iwc=new IndexWriterConfig(Version.LUCENE_47,analyzer);
IndexWriter iw=new IndexWriter(fsDirectory, iwc);
return iw;
}
public static List<Arcticle> search(String indexDir,String keys) throws Exception{
FSDirectory fsDirectory= FSDirectory.open(new File(indexDir));
IndexSearcher searcher=new IndexSearcher(DirectoryReader.open(fsDirectory));
MultiFieldQueryParser qp=new MultiFieldQueryParser(Version.LUCENE_47, new String[]{"title","content"}, analyzer);
Query query=qp.parse(keys);
//初始化高亮器
Highlighter high=new Highlighter(htmlFormatter, new QueryScorer(query));
TopDocs td=searcher.search(query, 10);
ScoreDoc[] sd=td.scoreDocs;
List<Arcticle> arcList=new ArrayList<Arcticle>();
for(ScoreDoc ss:sd){
Document doc=searcher.doc(ss.doc);
Arcticle arc=new Arcticle();
arc.setTitle(doc.getField("title").stringValue());
TokenStream tokenStream = analyzer.tokenStream("desc", new StringReader(doc.get("desc")));
String str = high.getBestFragment(tokenStream,  doc.get("desc"));
arc.setDescription(str);
arc.setUrl(CsdnSpilder.rootDir+doc.getField("url").stringValue());
arc.setCreateTime(doc.getField("createTime").stringValue());
arcList.add(arc);
}
return arcList;
}
/**
* 优化方式 先将索引写入内存 到一定量后写入磁盘
* @param dir
* @param arc
* @throws IOException
*/
public static void indexs(String dir,List<Arcticle> arc) throws IOException{
RAMDirectory ramDirectory=new RAMDirectory();
IndexWriter ramWriter=getRamWriter(ramDirectory);
new File(dir).mkdirs();
//索引文件存在于目录中 IndexWriter只是个写入对象
IndexWriter iw=getWriter(dir);
for(int i=1;i<=arc.size();i++){
Arcticle srcTmp=arc.get(i-1);
if(i%dealCount==0 || i==arc.size()){
//必须关闭writer才能将目录索引数据写入到其他的writer中
ramWriter.commit();
ramWriter.close();
iw.addIndexes(ramDirectory);
if(i<arc.size())
ramWriter=getRamWriter(ramDirectory);
}else{
index(ramWriter,srcTmp);
}
}
iw.commit();
iw.close();
}
/**
* 直接写入磁盘
* @param writer
* @param arc
* @return
* @throws IOException
*/
public static Document index(IndexWriter writer,Arcticle arc) throws IOException{
Document doc=new Document();
doc.add(new TextField("title", arc.getTitle(), Store.YES));
doc.add(new TextField("url", arc.getUrl(), Store.YES));
doc.add(new TextField("createTime", arc.getCreateTime(), Store.YES));
doc.add(new TextField("desc", arc.getDescription(), Store.YES));
doc.add(new TextField("content", Jsoup.parse(FileUtils.readFileToString(new File(arc.getFilePath()))).text(), Store.YES));
writer.addDocument(doc);
return doc;
}
}

》爬取类

package cn.et.spilder;

import java.io.File;
import java.io.FileInputStream;
import java.io.FileOutputStream;
import java.io.IOException;
import java.util.ArrayList;
import java.util.List;
import java.util.Properties;

import org.apache.commons.io.FileUtils;
import org.apache.log4j.Logger;
import org.jsoup.Jsoup;
import org.jsoup.nodes.Document;
import org.jsoup.nodes.Element;
import org.jsoup.select.Elements;
import org.junit.Test;

import cn.et.index.LuceneUtils;
/**
* Csdn的蜘蛛 需要自行分析博客的html语法的特点
*时间：2017-6-29 上午10:42:47
*作者： LM
*联系方式：973465719@qq.com
*
*/
public class CsdnSpilder {
//存放下载文件的目录
public final static  String SAVED_DIR="F:/mycsdn/";
//CSDN的blog的官网
public final static  String rootDir="http://blog.csdn.net/";
//爬取的用户名
public final static  String userName="liaomin416100569";
//路徑分隔符
public final static String separatorChar="/";
//索引目录
public final static String indexDir="F:/myindex/";
//日志处理
public final static Logger logger=Logger.getLogger(CsdnSpilder.class);

/**
开始爬虫的入口
* @param args
* @throws IOException
*/
public static void main(String[] args) throws IOException {
logger.debug("开始处理:"+rootDir+userName);
Document doc = getDoc(rootDir+userName);
//获取所有的分类
List<Category> category=getCategory(doc);
logger.debug("获取到类型个数为：:"+category.size());
for(Category tmCate:category){
String curl=tmCate.getUrl();
logger.debug("开始处理文章分类："+tmCate.getName()+",路径："+curl+",文章数："+tmCate.getCount());
//创建目录
new File(SAVED_DIR+curl).mkdirs();
List<Arcticle> arcList=getArcticle(curl);
logger.debug("开始索引");
LuceneUtils.indexs(indexDir, arcList);
logger.debug("处理完成");
}

}
/**
* 无法获取重试获取Document对象
* @return
*/
public static Document getDoc(String path){
Document doc;
while(true){
try {
doc = Jsoup.connect(path).get();
break;
} catch (Exception e) {
e.printStackTrace();
}
}
return doc;
}

@Test
public void testArcticle() throws IOException{
String curl="/liaomin416100569/article/category/650802";
//创建目录
new File(SAVED_DIR+curl).mkdirs();
List<Arcticle> arcList=getArcticle(curl);
LuceneUtils.indexs(indexDir, arcList);
System.out.println(arcList.size());
}

/**
* 获取所有分类
* @return
* @throws IOException
*/
public static List<Category> getCategory(Document doc) throws IOException{
Elements newsHeadlines = doc.select("#panel_Category a");
String[] categoryArr=newsHeadlines.text().replaceAll(" +", " ").split(" ");
List<Category> cates=new ArrayList<Category>();
for(int i=0;i<newsHeadlines.size();i++){
Element el=newsHeadlines.get(i);
Category c=new Category();
c.setName(el.text().trim());
c.setCount(el.nextElementSibling().text());
c.setUrl(el.attr("href"));
cates.add(c);
}
return cates;
}
/**
* 通过分类获取文章 文章过多不建议调用此方法
* @param cl
* @return
* @throws IOException
*/
public static List<Arcticle> getArcticle(String typeUrl) throws IOException{
String category=rootDir+typeUrl;
Document doc = getDoc(category);
//判断是否存在分页
Elements newsHeadlines = doc.select("#papelist");
int totalPage=0;
List<Arcticle> arcs=new ArrayList<Arcticle>();
//存在分页
if(newsHeadlines.size()>0){
totalPage=Integer.parseInt(newsHeadlines.get(0).child(0).text().split("共")[1].split("页")[0]);
for(int i=1;i<=totalPage;i++){
arcs.addAll(getArcticle(typeUrl,i));
}
}else{
arcs.addAll(getArcticle(typeUrl,1));
}
return arcs;
}
/**
* 获取分配中某一页的文章
* @param typeUrl
* @param page
* @return
* @throws IOException
*/
public static List<Arcticle> getArcticle(String typeUrl,int page) throws IOException{
//獲取當前頁的url地址
String pageUrl=rootDir+typeUrl+separatorChar+page;
//获取需要保存的目录
String destPath=SAVED_DIR+typeUrl+separatorChar+page;
//提前创建该目录
new File(destPath).mkdirs();
//如果如法连接自动重连
Document doc = getDoc(pageUrl);
//获取到所有文章的标题标签
Elements docs = doc.select(".link_title");
List<Arcticle> arcs=new ArrayList<Arcticle>();
for(int i=0;i<docs.size();i++){
Element el=docs.get(i);
Element ael=el.child(0);
//获取标题上超链接的地址
String url=ael.attr("href");
String title=ael.text();
Arcticle a=new Arcticle();
a.setUrl(url);
a.setTitle(title);
//获取描述的标签
Element descElement=el.parent().parent().nextElementSibling();
String description=descElement.text();
a.setDescription(description);
//获取最后更新时间的标签
Element timeElement=descElement.nextElementSibling();
String createTime=timeElement.select(".link_postdate").text();
a.setCreateTime(createTime);
logger.debug(title+url+description+createTime);
arcs.add(a);
//保存到文件中
String filePath=saveFile(destPath,rootDir+url,a);
a.setFilePath(filePath);
}
return arcs;
}
/**
* 保存html内容到文件中
* @param destDir 需要保存的目標目錄
* @param htmlUrl 抓取的htmlurl
* @throws IOException
*/
public static String saveFile(String destDir,String htmlUrl,Arcticle arc) throws IOException{
Document doc = getDoc(htmlUrl);
String fileName=htmlUrl.substring(htmlUrl.lastIndexOf("/")+1);
String file=destDir+separatorChar+fileName+".html";
File hfile=new File(file);
boolean ifUpdate=true;
//文件存在需要判断文件是否存在更新
if(hfile.exists()){
Properties p=new Properties();
p.load(new FileInputStream(destDir+separatorChar+fileName+".ini"));
String createTime=p.getProperty("createTime");
//之前的文章创建时间 小于 网上爬取文章 时间 所以文章被修改过 需要更新
if(createTime.compareTo(arc.getCreateTime())<0){
hfile.delete();
ifUpdate=true;
}else{
ifUpdate=false;
}
}
if(ifUpdate){
//写入文件 并将信息写入资源文件
Properties p=new Properties();
p.setProperty("title", arc.getTitle());
p.setProperty("url", arc.getUrl());
p.setProperty("description", arc.getDescription());
p.setProperty("createTime", arc.getCreateTime());
p.store(new FileOutputStream(destDir+separatorChar+fileName+".ini"),htmlUrl);
FileUtils.writeStringToFile(hfile, doc.toString(),"UTF-8");
}
return file;
}
}

》maven依赖配置（IK分词器 2012停止更新 只支持到lucene4.7.2所以使用该版本）

<dependency>
<groupId>org.jsoup</groupId>
<artifactId>jsoup</artifactId>
<version>1.7.2</version>
</dependency>
<dependency>
<groupId>org.apache.commons</groupId>
<artifactId>commons-io</artifactId>
<version>1.3.2</version>
</dependency>
<dependency>
<groupId>org.slf4j</groupId>
<artifactId>slf4j-log4j12</artifactId>
<version>1.4.3</version>
</dependency>

<dependency>
<groupId>com.janeluo</groupId>
<artifactId>ikanalyzer</artifactId>
<version>2012_u6</version>
</dependency>
<dependency>
<groupId>org.apache.lucene</groupId>
<artifactId>lucene-highlighter</artifactId>
<version>4.7.2</version>
</dependency>

》log4j配置 在 src/main/java添加 log4j.properties 

log4j.rootLogger=debug,stdout
log4j.appender.stdout = org.apache.log4j.ConsoleAppender
log4j.appender.stdout.Target = System.out
log4j.appender.stdout.layout = org.apache.log4j.PatternLayout
log4j.appender.stdout.layout.ConversionPattern =%d{ABSOLUTE} %5p %c \:%L - %m%n

执行 CsdnSpilder 自动在f:/myindex目录创建索引  f:/mycsdn备份自己的博客html文件 
效果如下





二。 搜索索引文件 

使用springboot快速发布web应用 使用angularjs 实现ajax请求和迭代输出

》添加maven依赖

<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/maven-v4_0_0.xsd"> <modelVersion>4.0.0</modelVersion>
<groupId>cn.et</groupId>
<artifactId>spilder</artifactId>
<version>0.0.1-SNAPSHOT</version>
<parent>
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-starter-parent</artifactId>
<version>1.5.4.RELEASE</version>
</parent>
<dependencies>
<dependency>
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-starter-web</artifactId>
</dependency>
<!--对jsp的支持-->
<dependency>
<groupId>org.apache.tomcat.embed</groupId>
<artifactId>tomcat-embed-jasper</artifactId>
</dependency>
<dependency>
<groupId>javax.servlet</groupId>
<artifactId>javax.servlet-api</artifactId>
<scope>provided</scope>
</dependency>
<dependency>
<groupId>javax.servlet</groupId>
<artifactId>jstl</artifactId>
</dependency>
<dependency>
<groupId>com.janeluo</groupId>
<artifactId>ikanalyzer</artifactId>
<version>2012_u6</version>
</dependency>
<dependency>
<groupId>org.apache.lucene</groupId>
<artifactId>lucene-highlighter</artifactId>
<version>4.7.2</version>
</dependency>
</dependencies>
</project>

》启动springboot类

package cn.et;

import org.springframework.boot.SpringApplication;
import org.springframework.boot.autoconfigure.SpringBootApplication;

@SpringBootApplication
public class WebStart {

public static void main(String[] args) {
SpringApplication.run(WebStart.class, args);
}
}

》添加springmvc控制层类

package cn.et.controller;

import java.util.List;

import javax.servlet.http.HttpServletResponse;

import org.springframework.web.bind.annotation.RequestMapping;
import org.springframework.web.bind.annotation.RestController;

import cn.et.index.LuceneUtils;
import cn.et.spilder.Arcticle;

@RestController
public class SearchController {
//索引目录
public final static String indexDir="F:/myindex/";
@RequestMapping("search")
public List<Arcticle> search(String key,HttpServletResponse rsp) throws Exception{
rsp.setHeader("Content-Type", "application/json;charset=UTF-8");
return LuceneUtils.search(indexDir, key);
}
}

》src/main添加 webapp目录 新建jsp文件和添加angularjs库

<!DOCTYPE HTML PUBLIC "-//W3C//DTD HTML 4.01 Transitional//EN">
<html>
<head>
<title>s.html</title>

<meta http-equiv="keywords" content="keyword1,keyword2,keyword3">
<meta http-equiv="description" content="this is my page">
<meta http-equiv="content-type" content="text/html; charset=UTF-8">

<!--<link rel="stylesheet" type="text/css" href="./styles.css">-->
<script type="text/javascript" src="angular.min.js"></script>
</head>

<body>
<div ng-app="searchApp" ng-controller="searchController">
搜索key：<input type="text" name="key" ng-model="key"> <input type="button" value='查询' ng-click="search()">
<div ng-repeat="x in searchData">
<a href="{{x.url}}"><font size="5"><titl ng-bind-html="x.title | trustHtml"></titl><br/></font></a>
   <desp ng-bind-html="x.description | trustHtml"></desp><hr>
</div>
</div>
</body>
<script type="text/javascript">
var searchApp=angular.module("searchApp",[]);
searchApp.controller("searchController",function($scope,$http,$sce){
$scope.search=function(){
$http({
method:"GET",
url:"search?key="+$scope.key
}).then(function(res){
var d=res.data;
$scope.searchData=d;
},function(res){
alert("查询失败"+res.status+"-"+res.statusText);
});
}
})
//因为angular会自动将html内容转义 需要添加一个过滤器 表示信任
searchApp.filter("trustHtml",function($sce){
return function(val){
return $sce.trustAsHtml(val);
}
});
</script>
</html>

启动运行Webstart 自动内嵌tomcat并运行 默认tomcat 端口 8080 上下文路径 /

访问 http://localhost:8080/s.html   尝试搜索关键 点击标题 自动链接到csdn博客

技术学习路线(学习周期 一周)

  jsoup  学习jquery用法即可 （http://www.runoob.com/jquery/jquery-tutorial.html）

  angularjs学习 （http://www.runoob.com/angularjs/angularjs-tutorial.html）

  springboot 学习 （http://docs.spring.io/spring-boot/docs/1.5.4.RELEASE/reference/htmlsingle/）

  lucence 官网关于lucence例子 （http://lucene.apache.org/core/6_6_0/index.html）