您的位置:首页 > 其它

全文检索ElasticSearch的运用 用Logstash实现搜索业务

2020-06-04 07:28 651 查看

1  业务流程

1、课程管理服务将数据写到MySQL数据库
2、使用Logstash将MySQL数据库中的数据写到ES的索引库。
3、用户在前端搜索课程信息,请求到搜索服务。
4、搜索服务请求ES搜索课程信息

2 创建ElasticSearch索引库

Post http://localhost:9200/xc_course/doc/_mapping

[code]{
"properties" : {

"description" : {
"analyzer" : "ik_max_word",
"search_analyzer": "ik_smart",
"type" : "text"
},
"grade" : {
"type" : "keyword"
},
"id" : {
"type" : "keyword"
},
"mt" : {
"type" : "keyword"
},
"name" : {
"analyzer" : "ik_max_word",
"search_analyzer": "ik_smart",
"type" : "text"
},
"users" : {
"index" : false,
"type" : "text"
},
"charge" : {
"type" : "keyword"
},
"valid" : {
"type" : "keyword"
},
"pic" : {
"index" : false,
"type" : "keyword"
},
"qq" : {
"index" : false,
"type" : "keyword"
},
"price" : {
"type" : "float"
},
"price_old" : {
"type" : "float"
},
"st" : {
"type" : "keyword"
},
"status" : {
"type" : "keyword"
},
"studymodel" : {
"type" : "keyword"
},
"teachmode" : {
"type" : "keyword"
},
"teachplan" : {
"analyzer" : "ik_max_word",
"search_analyzer": "ik_smart",
"type" : "text"
},

"expires" : {
"type" : "date",
"format": "yyyy-MM-dd HH:mm:ss"
},
"pub_time" : {
"type" : "date",
"format": "yyyy-MM-dd HH:mm:ss"
},
"start_time" : {
"type" : "date",
"format": "yyyy-MM-dd HH:mm:ss"
},
"end_time" : {
"type" : "date",
"format": "yyyy-MM-dd HH:mm:ss"
}
}
}

3 Logstash创建索引

Logstash5.x以上版本本身自带有logstash-input-jdbc,6.x版本本身不带logstash-input-jdbc,需要手动安装

[code].\logstash-plugin.bat install logstash-input-jdbc

在logstash-6.2.1/config/xc_course_template.json配置的模板文件

[code]{
"mappings" : {
"doc" : {
"properties" : {
"charge" : {
"type" : "keyword"
},
"description" : {
"analyzer" : "ik_max_word",
"search_analyzer" : "ik_smart",
"type" : "text"
},
"end_time" : {
"format" : "yyyy-MM-dd HH:mm:ss",
"type" : "date"
},
"expires" : {
"format" : "yyyy-MM-dd HH:mm:ss",
"type" : "date"
},
"grade" : {
"type" : "keyword"
},
"id" : {
"type" : "keyword"
},
"mt" : {
"type" : "keyword"
},
"name" : {
"analyzer" : "ik_max_word",
"search_analyzer" : "ik_smart",
"type" : "text"
},
"pic" : {
"index" : false,
"type" : "keyword"
},
"price" : {
"type" : "float"
},
"price_old" : {
"type" : "float"
},
"pub_time" : {
"format" : "yyyy-MM-dd HH:mm:ss",
"type" : "date"
},
"qq" : {
"index" : false,
"type" : "keyword"
},
"st" : {
"type" : "keyword"
},
"start_time" : {
"format" : "yyyy-MM-dd HH:mm:ss",
"type" : "date"
},
"status" : {
"type" : "keyword"
},
"studymodel" : {
"type" : "keyword"
},
"teachmode" : {
"type" : "keyword"
},
"teachplan" : {
"analyzer" : "ik_max_word",
"search_analyzer" : "ik_smart",
"type" : "text"
},
"users" : {
"index" : false,
"type" : "text"
},
"valid" : {
"type" : "keyword"
}
}
}
},
"template" : "xc_course"
}

配置Mysql.conf

[code]input {
stdin {
}
jdbc {
jdbc_connection_string => "jdbc:mysql://127.0.0.1:12345/xc_course?useUnicode=true&characterEncoding=utf-8&useSSL=true&serverTimezone=UTC"
# the user we wish to excute our statement as
jdbc_user => "root"
jdbc_password => "123456"
# the path to our downloaded jdbc driver
jdbc_driver_library => "C:/Users/沐雨橙风/.m2/repository/mysql/mysql-connector-java/5.1.6/mysql-connector-java-5.1.6.jar"  # mysql-connector包的位置
# the name of the driver class for mysql
jdbc_driver_class => "com.mysql.jdbc.Driver"
jdbc_paging_enabled => "true"
jdbc_page_size => "50000"
#要执行的sql文件
#statement_filepath => "/conf/course.sql"
statement => "select * from course_pub where timestamp > date_add(:sql_last_value,INTERVAL 8 HOUR)"
#定时配置
schedule => "* * * * *"
record_last_run => true
last_run_metadata_path => "D:/lucene/logstash-6.2.1/config/logstash_metadata"
}
}

output {
elasticsearch {
#ES的ip地址和端口
hosts => "localhost:9200"
#hosts => ["localhost:9200","localhost:9202","localhost:9203"]
#ES索引库名称
index => "xc_course"
document_id => "%{id}"
document_type => "doc"
template =>"D:/lucene/logstash-6.2.1/config/xc_course_template.json" #模板文件存放位置
template_name =>"xc_course"
template_overwrite =>"true"
}
stdout {
#日志输出
codec => json_lines
}
}

执行下面命令,实现索引与mysql的同步。

[code].\logstash.bat -f ..\config\mysql.conf

4 创建索引服务工程

application.xml

[code]xuecheng:
elasticsearch:
hostlist: ${eshostlist:127.0.0.1:9200} #多个结点中间用逗号分隔
course:
index: xc_course
type: doc
source_field: id,name,grade,mt,st,charge,valid,pic,qq,price,price_old,status,studymodel,teachmode,expires,pub_time,start_time,end_time

配置类

[code]@Configuration
public class ElasticsearchConfig {

@Value("${xuecheng.elasticsearch.hostlist}")
private String hostlist;

@Bean
public RestHighLevelClient restHighLevelClient(){
//解析hostlist配置信息
String[] split = hostlist.split(",");
//创建HttpHost数组,其中存放es主机和端口的配置信息
HttpHost[] httpHostArray = new HttpHost[split.length];
for(int i=0;i<split.length;i++){
String item = split[i];
httpHostArray[i] = new HttpHost(item.split(":")[0], Integer.parseInt(item.split(":")[1]), "http");
}
//创建RestHighLevelClient客户端
return new RestHighLevelClient(RestClient.builder(httpHostArray));
}

//项目主要使用RestHighLevelClient,对于低级的客户端暂时不用
@Bean
public RestClient restClient(){
//解析hostlist配置信息
String[] split = hostlist.split(",");
//创建HttpHost数组,其中存放es主机和端口的配置信息
HttpHost[] httpHostArray = new HttpHost[split.length];
for(int i=0;i<split.length;i++){
String item = split[i];
httpHostArray[i] = new HttpHost(item.split(":")[0], Integer.parseInt(item.split(":")[1]), "http");
}
return RestClient.builder(httpHostArray).build();
}

}

业务类

[code]    @Value("${xuecheng.course.index}")
private String index;
@Value("${xuecheng.course.type}")
private String type;
@Value("${xuecheng.course.source_field}")
private String source_field;

@Autowired
RestHighLevelClient restHighLevelClient;

public QueryResponseResult<CoursePub> list(int page, int size, CourseSearchParam courseSearchParam) {
if (courseSearchParam==null){
courseSearchParam=new CourseSearchParam();
}
//设置索引
SearchRequest searchRequest=new SearchRequest(index);
//设置类型
searchRequest.types(type);

SearchSourceBuilder searchSourceBuilder=new SearchSourceBuilder();
//字段过滤
String[] source_field_array = source_field.split(",");
searchSourceBuilder.fetchSource(source_field_array,new String[]{});
BoolQueryBuilder boolQueryBuilder= QueryBuilders.boolQuery();

if (StringUtils.isNotEmpty(courseSearchParam.getKeyword())){
//匹配的关键字 设置匹配占比 提升另一个字段的boost值
MultiMatchQueryBuilder multiMatchQueryBuilder = QueryBuilders.multiMatchQuery(courseSearchParam.getKeyword(), "name", "description", "teachplan")
.minimumShouldMatch("70%")
.field("name", 10);
boolQueryBuilder.must(multiMatchQueryBuilder);
}
//过滤
if (StringUtils.isNotEmpty(courseSearchParam.getMt())){
boolQueryBuilder.filter(QueryBuilders.termQuery("mt",courseSearchParam.getMt()));
}
if (StringUtils.isNotEmpty(courseSearchParam.getSt())){
boolQueryBuilder.filter(QueryBuilders.termQuery("st",courseSearchParam.getSt()));
}
if (StringUtils.isNotEmpty(courseSearchParam.getGrade())){
boolQueryBuilder.filter(QueryBuilders.termQuery("grade",courseSearchParam.getGrade()));
}
//布尔查询
searchSourceBuilder.query(boolQueryBuilder);
//      分页
if (page<=0){
page=1;
}
if (size<=0){
size=12;
}
int from=(page-1)*size;
searchSourceBuilder.from(from);
searchSourceBuilder.size(size);
// 高亮设置
HighlightBuilder highlightBuilder=new HighlightBuilder();
highlightBuilder.preTags("<font class='eslight'>");
highlightBuilder.postTags("</font>");
//高亮设置的字段
highlightBuilder.fields().add(new HighlightBuilder.Field("name"));
searchSourceBuilder.highlighter(highlightBuilder);
//请求搜索
searchRequest.source(searchSourceBuilder);

QueryResult<CoursePub> queryResult=new QueryResult();
//数据列表
List<CoursePub> list=new ArrayList<>();
try {
SearchResponse searchResponse = restHighLevelClient.search(searchRequest);
//结果集处理
SearchHits hits = searchResponse.getHits();
//记录总数
long totalHits = hits.totalHits;
queryResult.setTotal(totalHits);
SearchHit[] searchHits = hits.getHits();
for (SearchHit hit:searchHits){
CoursePub coursePub=new CoursePub();
//取出source
Map<String, Object> sourceAsMap = hit.getSourceAsMap();

String id = (String)sourceAsMap.get("id");
coursePub.setId(id);
//取出名称
String name = (String)sourceAsMap.get("name");
//取出高亮字段内容
Map<String, HighlightField> highlightFields = hit.getHighlightFields();
if (highlightFields!=null){
HighlightField highlightFieldName=highlightFields.get("name");
if (highlightFieldName!=null){
Text[] fragments = highlightFieldName.fragments();
StringBuilder stringBuilder=new StringBuilder();
for (Text text:fragments){
stringBuilder.append(text);
}
name=stringBuilder.toString();
}

}
coursePub.setName(name);
//图片
String pic = (String) sourceAsMap.get("pic");
coursePub.setPic(pic);
//价格
Double price = null;
try {
if(sourceAsMap.get("price")!=null ){
price = (Double) sourceAsMap.get("price");
}
} catch (Exception e) {
e.printStackTrace();
}
coursePub.setPrice(price);
Double price_old = null;
try {
if(sourceAsMap.get("price_old")!=null ){
price_old = (Double) sourceAsMap.get("price_old");
}
} catch (Exception e) {
e.printStackTrace();
}
coursePub.setPrice_old(price_old);
list.add(coursePub);
}

} catch (IOException e) {
e.printStackTrace();
}

queryResult.setList(list);
QueryResponseResult<CoursePub> queryResponseResult = new QueryResponseResult<CoursePub>(CommonCode.SUCCESS,queryResult);

return queryResponseResult;
}

 

内容来自用户分享和网络整理,不保证内容的准确性,如有侵权内容,可联系管理员处理 点击这里给我发消息
标签: