yoho-search / yohomars-search-service · Commits

Go to a project

Authored by wangnan
Commit d5fac6fd1d3c878df733bb9476c511e5f7b29764 1 parent 96060f1d

整理代码

Inline Side-by-side
Showing 10 changed files with 113 additions and 218 deletions
... ... @@ -36,78 +36,48 @@ public interface IElasticsearchClient {
/**
* 获取客户端名称
*
* @return
*/
public String getName();
/**
* 执行集群操作请求
*
* @param callback
* @return
*/
public <T extends ActionResponse> T executeGet(final ClusterCallback<T> callback);
/**
* 执行数据操作请求
*
* @param callback
* @return
*/
public <T extends ActionResponse> T executeGet(final ClientCallback<T> callback);
/**
* 执行索引操作请求
*
* @param callback
* @return
*/
public <T extends ActionResponse> T executeGet(final NodeCallback<T> callback);
/**
* 判断索引是否存在
* <p>
* 索引名称
*
* @return
*/
public boolean indexExists(final String realIndexName);
/**
* 获取真实索引到索引别名的映射
*
* @return
*/
public Map<String, List<String>> getRealNameToAliasesMap();
/**
* 获取集群健康状态
* <p>
* 索引名称
*
* @return
*/
public ClusterHealthStatus getClusterHealthStatus(String aliasIndexName);
/**
* 判断索引是否健康
* <p>
* 索引名称
*
* @return
*/
public boolean indexHealth(final String realIndexName);
/**
* 创建索引
*
* @param realIndexName 索引名称
* @param type 索引类型
* @param settings 索引配置
* @param mappingContent 索引内容
*/
public CreateIndexResponse createIndex(final String realIndexName, final String type, final Map<String, String> settings, final String mappingContent);
... ... @@ -118,88 +88,49 @@ public interface IElasticsearchClient {
/**
* 在批量插入数据之前，进行索引优化配置 返回优化配置之前的配置
*
* @return
*/
public Map<String, String> optimumSettingsForBulkIndexing(String realIndexName);
/**
* 索引配置
*
* @param setting
*/
public UpdateSettingsResponse setIndexSettings(String realIndexName, Map<String, String> setting);
/**
* 删除索引
* <p>
* 索引名称
*/
public DeleteIndexResponse deleteIndex(final String realIndexName);
/**
* 更新索引缓存
* <p>
* 索引名称
*/
public RefreshResponse refreshIndex(final String realIndexName);
/**
* 关闭索引
* <p>
* 索引名称
*/
public CloseIndexResponse closeIndex(final String realIndexName);
/**
* 刷新索引文件
* <p>
* 索引名称
*/
public FlushResponse flushIndex(final String realIndexName);
/**
* 为索引创建别名
*
* @param alias
*/
public IndicesAliasesResponse addAlias(String alias, String... indices);
/**
* 为索引删除别名
*
* @param realIndexName
*/
public IndicesAliasesResponse removeAlias(String realIndexName, String... aliases);
/**
* 添加索引数据
* <p>
* 索引名称
*
* @param id 索引ID
* @param data 索引数据
*/
public IndexResponse addIndexData(final String realIndexName, final String type, final String id, final Object data);
/**
* 添加索引数据
*
* 索引名称
* @param
* 索引数据列表
*/
//@SuppressWarnings("rawtypes")
//public BulkResponse addIndexData(final String realIndexName, final String type, final List<? extends Map> datas);
/**
* 添加索引数据
*
* 索引名称
* @param datas
* 索引数据列表
*/
// public BulkResponse addIndexDataBean(final String realIndexName, final String type, final List<?> datas);
/**
* 删除索引数据
... ... @@ -207,14 +138,6 @@ public interface IElasticsearchClient {
public DeleteResponse deleteIndexData(final String realIndexName, final String type, String id);
/**
* 根据query删除索引数据
*
* @param indexName
* @param query
*/
//public DeleteByQueryResponse deleteIndexDataByQuery(final String realIndexName, final String type, String query);
/**
* 更新索引数据
*/
public UpdateResponse updateIndexData(final String realIndexName, final String type, final String id, final Object data);
... ... @@ -226,73 +149,52 @@ public interface IElasticsearchClient {
/**
* 根据多个id获取数据
*
* @param idList
* @param fields
* @return
*/
public MultiGetResponse multiGet(final String realIndexName, final String type, final Set<String> idList, final List<String> fields);
/**
* 根据多个id获取数据
*
* @param idList
* @param fields
* @return
*/
public MultiGetResponse multiGet(final String realIndexName, final String type, final String[] idList, final List<String> fields);
/**
* 根据id获取数据
*
* @param id
* @return
*/
public GetResponse get(final String realIndexName, final String type, final String id);
/**
* 检索接口
*
* @param searchParam
* @return
*/
public SearchResult search(final String aliasIndexName, final String type, final SearchParam searchParam);
/**
* 检索接口
*
* @param searchParam
* @return
*/
public SearchResult search(final List<String> aliasIndexNameList, final String type, final SearchParam searchParam);
/**
* 检索接口
*
* @return
*/
public List<SearchResult> multiSearch(final String aliasIndexName, final String type, final List<SearchParam> searchParams);
/**
* 批量检索接口
*
* @return
*/
public List<SearchResult> multiSearch(final List<String> aliasIndexNameList, final String type, final List<SearchParam> searchParams);
/**
* 获取分词结果
*
* @param text
* @param analyzer
* @return
*/
public AnalyzeResponse getAnalyzeResponse(final String aliasIndexName, final String text, final String analyzer);
/**
*批量操作es
*/
public BulkResponse bulk(List<ESBluk> esBluks);
/**
*联想推荐
*/
public SuggestResponse suggeest(final SearchParam searchParam, final String realIndexName);
}
... ...
... ... @@ -448,49 +448,7 @@ public class ElasticsearchClientImpl implements IElasticsearchClient {
return search(indexNameList, type, searchParam);
}
private SearchResult getSearchResultFromSearchResponse(SearchParam searchParam, SearchResponse response) {
// 构造返回结果
SearchResult searchResult = new SearchResult();
SearchHits hits = response.getHits();
List<Map<String, Object>> resultList = new ArrayList<Map<String, Object>>();
for (SearchHit hit : hits.getHits()) {
Map<String, Object> map;
if (hit.getSource() != null) {
map = hit.getSource();
} else {
map = new HashMap<String, Object>();
}
if (hit.getFields() != null && hit.getFields().size() > 0) {
// 返回指定字段
for (SearchHitField shf : hit.fields().values()) {
map.put(shf.name(), shf.value());
}
}
if (searchParam.isHighlight()) {
// 添加高亮字段信息
Map<String, Object> highlightMap = new HashMap<String, Object>();
// highlightMap.putAll(map);
for (HighlightField hf : hit.highlightFields().values()) {
highlightMap.put(hf.name(), hf.fragments()[0].toString());
}
map.put("_highlight", highlightMap);
}
resultList.add(map);
}
searchResult.setResultList(resultList);
searchResult.setTotal(hits.getTotalHits());
searchResult.setPage(searchParam.getPage());
if (searchResult.getTotal() % searchParam.getSize() == 0) {
searchResult.setTotalPage(searchResult.getTotal() / searchParam.getSize());
} else {
searchResult.setTotalPage(searchResult.getTotal() / searchParam.getSize() + 1);
}
if (response.getAggregations() != null) {
Map<String, Aggregation> aggregations = response.getAggregations().asMap();
searchResult.setAggMaps(aggregations);
}
return searchResult;
}
@Override
public SearchResult search(final List<String> aliasIndexNameList, final String type, final SearchParam searchParam) {
... ... @@ -562,4 +520,54 @@ public class ElasticsearchClientImpl implements IElasticsearchClient {
logger.debug("[model=EsIndexClient][method=search][aliasIndexNameList={}][cost={}ms]", aliasIndexNameList, System.currentTimeMillis() - begin);
return searchResults;
}
/**
* 拼装返回数据
* @param searchParam
* @param response
* @return
*/
private SearchResult getSearchResultFromSearchResponse(SearchParam searchParam, SearchResponse response) {
// 构造返回结果
SearchResult searchResult = new SearchResult();
SearchHits hits = response.getHits();
List<Map<String, Object>> resultList = new ArrayList<Map<String, Object>>();
for (SearchHit hit : hits.getHits()) {
Map<String, Object> map;
if (hit.getSource() != null) {
map = hit.getSource();
} else {
map = new HashMap<String, Object>();
}
if (hit.getFields() != null && hit.getFields().size() > 0) {
// 返回指定字段
for (SearchHitField shf : hit.fields().values()) {
map.put(shf.name(), shf.value());
}
}
if (searchParam.isHighlight()) {
// 添加高亮字段信息
Map<String, Object> highlightMap = new HashMap<String, Object>();
// highlightMap.putAll(map);
for (HighlightField hf : hit.highlightFields().values()) {
highlightMap.put(hf.name(), hf.fragments()[0].toString());
}
map.put("_highlight", highlightMap);
}
resultList.add(map);
}
searchResult.setResultList(resultList);
searchResult.setTotal(hits.getTotalHits());
searchResult.setPage(searchParam.getPage());
if (searchResult.getTotal() % searchParam.getSize() == 0) {
searchResult.setTotalPage(searchResult.getTotal() / searchParam.getSize());
} else {
searchResult.setTotalPage(searchResult.getTotal() / searchParam.getSize() + 1);
}
if (response.getAggregations() != null) {
Map<String, Aggregation> aggregations = response.getAggregations().asMap();
searchResult.setAggMaps(aggregations);
}
return searchResult;
}
}
\ No newline at end of file
... ...
... ... @@ -6,10 +6,7 @@ import com.yoho.search.dal.model.BizArea;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Component;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.*;
/**
* Created by wangnan on 2016/8/25.
... ... @@ -29,8 +26,7 @@ public class BizareaIndexBuilder extends IIndexBuilder {
List<BizArea> list = bizAreaMapper.getPageLists(offset, limit);
List<Map<String, Object>> dataList = new ArrayList<Map<String, Object>>();
if (list != null && list.size() > 0) {
int size = list.size();
for (int i = 0; i < size; i++) {
for (int i = 0; i < list.size(); i++) {
dataList.add(beanToMap(list.get(i)));
}
}
... ... @@ -42,8 +38,13 @@ public class BizareaIndexBuilder extends IIndexBuilder {
return ((Map) object).get("id").toString();
}
public Map<String, Object> beanToMap(BizArea bizArea) {
Map map = new HashMap();
/**
* 拼装对象数据为map
* @param bizArea
* @return
*/
private Map<String, Object> beanToMap(BizArea bizArea) {
Map map = new LinkedHashMap<>();
map.put("id", bizArea.getId());
map.put("name", bizArea.getName());
map.put("english_name", bizArea.getEnglishName());
... ...
... ... @@ -6,10 +6,7 @@ import com.yoho.search.dal.model.Comment;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Component;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.*;
/**
* Created by wangnan on 2016/8/25.
... ... @@ -29,8 +26,7 @@ public class CommentIndexBuilder extends IIndexBuilder {
List<Comment> list = commentMapper.getPageLists(offset,limit);
List<Map<String, Object>> dataList = new ArrayList<Map<String, Object>>();
if (list != null && list.size() > 0) {
int size = list.size();
for (int i = 0; i < size; i++) {
for (int i = 0; i < list.size(); i++) {
dataList.add(beanToMap(list.get(i)));
}
}
... ... @@ -42,8 +38,13 @@ public class CommentIndexBuilder extends IIndexBuilder {
return ((Map)object).get("id").toString();
}
/**
* 拼装对象数据为map
* @param comment
* @return
*/
public Map<String, Object> beanToMap(Comment comment) {
Map map = new HashMap();
Map map = new LinkedHashMap<>();
map.put("id", comment.getId());
map.put("uid", comment.getUid());
map.put("description", comment.getDescription());
... ...
package com.yohomars.search.index.builder.impls;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Component;
import com.yoho.search.dal.LineMapper;
import com.yoho.search.dal.model.Line;
import com.yohomars.search.index.builder.IIndexBuilder;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Component;
import java.util.*;
/**
* Created by wangnan on 2016/8/25.
... ... @@ -31,8 +27,7 @@ public class LineIndexBuilder extends IIndexBuilder {
List<Line> list = lineMapper.getPageLists(offset,limit);
List<Map<String, Object>> dataList = new ArrayList<Map<String, Object>>();
if (list != null && list.size() > 0) {
int size = list.size();
for (int i = 0; i < size; i++) {
for (int i = 0; i < list.size(); i++) {
dataList.add(beanToMap(list.get(i)));
}
}
... ... @@ -44,8 +39,13 @@ public class LineIndexBuilder extends IIndexBuilder {
return ((Map)object).get("id").toString();
}
/**
* 拼装对象数据为map
* @param line
* @return
*/
public Map<String, Object> beanToMap(Line line) {
Map map = new HashMap();
Map map = new LinkedHashMap<>();
map.put("id", line.getId());
map.put("title", line.getTitle());
map.put("title_tc", line.getTitleTc());
... ...
... ... @@ -7,10 +7,7 @@ import com.yohomars.search.index.builder.IIndexBuilder;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Component;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.*;
/**
* Created by wangnan on 2016/8/25.
... ... @@ -31,8 +28,7 @@ public class StoreIndexBuilder extends IIndexBuilder {
List<Store> list = storeMapper.getPageLists(offset, limit);
List<Map<String, Object>> dataList = new ArrayList<Map<String, Object>>();
if (list != null && list.size() > 0) {
int size = list.size();
for (int i = 0; i < size; i++) {
for (int i = 0; i < list.size(); i++) {
dataList.add(beanToMap(list.get(i)));
}
}
... ... @@ -44,8 +40,13 @@ public class StoreIndexBuilder extends IIndexBuilder {
return ((Map) object).get("id").toString();
}
/**
* 拼装对象数据为map
* @param store
* @return
*/
public Map<String, Object> beanToMap(Store store) {
Map map = new HashMap();
Map map = new LinkedHashMap<>();
map.put("id", store.getId());
map.put("store_name", store.getStoreName());
map.put("store_english_name", store.getStoreEnglishName());
... ... @@ -66,6 +67,7 @@ public class StoreIndexBuilder extends IIndexBuilder {
map.put("sort", store.getSort());
map.put("suggest", store.getStoreEnglishName() + " " + store.getStoreName());
//拼装地理位置信息
List<Double> locationList = new ArrayList<>();
locationList.add(Double.valueOf(store.getLongitude()));
locationList.add(Double.valueOf(store.getLatitude()));
... ...
package com.yohomars.search.index.builder.impls;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Component;
import com.yoho.search.dal.TopicMapper;
import com.yoho.search.dal.model.Topic;
import com.yohomars.search.index.builder.IIndexBuilder;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Component;
import java.util.*;
/**
* Created by wangnan on 2016/8/25.
... ... @@ -31,8 +27,7 @@ public class TopicIndexBuilder extends IIndexBuilder {
List<Topic> list = topicMapper.getPageLists(offset,limit);
List<Map<String, Object>> dataList = new ArrayList<Map<String, Object>>();
if (list != null && list.size() > 0) {
int size = list.size();
for (int i = 0; i < size; i++) {
for (int i = 0; i < list.size(); i++) {
dataList.add(beanToMap(list.get(i)));
}
}
... ... @@ -44,8 +39,13 @@ public class TopicIndexBuilder extends IIndexBuilder {
return ((Map)object).get("id").toString();
}
/**
* 拼装对象数据为map
* @param topic
* @return
*/
public Map<String, Object> beanToMap(Topic topic) {
Map map = new HashMap();
Map map = new LinkedHashMap<>();
map.put("id", topic.getId());
map.put("title", topic.getTitle());
map.put("title_tc", topic.getTitleTc());
... ...
... ... @@ -9,71 +9,52 @@ public interface IYohoIndex {
/**
* 获取索引名
*
* @return
*/
public String getYohoIndexName();
/**
* 设置索引名
*
* @param name
*/
public void setYohoIndexName(String name);
/**
* 获取索引配置属性
*
* @return
*/
public Map<String, String> getProperties();
/**
* 设置索引配置属性
*
* @param properties
*/
public void setProperties(Map<String, String> properties);
/**
* 获取索引建造器
*
* @return
*/
public IIndexBuilder getIndexBuilder();
/**
* 设置索引建造器的类名
*
* @return
*/
public void setIndexBuilderClassName(String indexBuilderClassName);
/**
* 获取索引建造器的类名
* @return
*/
public String getIndexBuilderClassName();
/**
* 设置索引mapping文件
*
* @return
*/
public void setMappingContent(String mappingContent);
/**
* 获取索引mapping内容
*
* @return
*/
public String getMappingContent();
/**
* 返回indexClient列表，按优先级排列
*
* @return
*/
public IElasticsearchClient getIndexClient();
... ...
... ... @@ -51,7 +51,7 @@ public class YohoIndexDataLoader implements ApplicationEventPublisherAware, Appl
final int totalCount = indexBuilder.getTotalCount();
final int limit = ISearchConstans.SEARCH_INDEX_BATCH_LIMIT;
final int totalPageSize = (int) ((totalCount - 1) / limit + 1); // 总页数
INDEX_REBUILD_LOG.info("[yohoIndexName=[{}]],[loadAllData],[totalPageSize={}]", yohoIndexName, totalPageSize);
INDEX_REBUILD_LOG.info("[yohomarsIndexName=[{}]],[loadAllData],[totalPageSize={}]", yohoIndexName, totalPageSize);
// 3、分配任务、并包装异步执行结果
List<Future<Boolean>> futureResults = new ArrayList<Future<Boolean>>();
for (int pageNo = 1; pageNo <= totalPageSize; pageNo++) {
... ... @@ -91,10 +91,10 @@ public class YohoIndexDataLoader implements ApplicationEventPublisherAware, Appl
boolean result = false;
while (tryCount <= 5 && !result) {
long begin = System.currentTimeMillis();
INDEX_REBUILD_LOG.info("[yohoIndexName=[{}]],[pageNo={}],[tryCount={}],[begin={}]", yohoIndexName, pageNo, tryCount, begin);
INDEX_REBUILD_LOG.info("[yohomarsIndexName=[{}]],[pageNo={}],[tryCount={}],[begin={}]", yohoIndexName, pageNo, tryCount, begin);
result = this.doLoadData(yohoIndexName, tempIndexRealName, indexBuilder, client, pageNo, limit);
long cost = System.currentTimeMillis() - begin;
INDEX_REBUILD_LOG.info("[yohoIndexName=[{}]],[pageNo={}],[tryCount={}],[begin={}],[cost={}],[{result={}]", yohoIndexName, pageNo, tryCount, begin, cost, result);
INDEX_REBUILD_LOG.info("[yohomarsIndexName=[{}]],[pageNo={}],[tryCount={}],[begin={}],[cost={}],[{result={}]", yohoIndexName, pageNo, tryCount, begin, cost, result);
tryCount++;
}
return result;
... ... @@ -106,7 +106,7 @@ public class YohoIndexDataLoader implements ApplicationEventPublisherAware, Appl
int start = (pageNo - 1) * limit;
List<?> dataList = indexBuilder.getPageLists(start, limit);
int resultSize = dataList != null ? dataList.size() : 0;
INDEX_REBUILD_LOG.info("[yohoIndexName=[{}]],[pageNo={}][{resultSize={}]", yohoIndexName, pageNo,resultSize);
INDEX_REBUILD_LOG.info("[yohomarsIndexName=[{}]],[pageNo={}][{resultSize={}]", yohoIndexName, pageNo,resultSize);
if (resultSize > 0) {
performanceMonitor.addVisitCount();
long begin = System.currentTimeMillis();
... ...
... ... @@ -22,7 +22,7 @@ public class YohoIndexHelper {
public static final String INDEX_ID_PREFIX = "mars_";
/**
* 为【有货索引名】建立一个临时索引名称
* 建立一个临时索引名称
*
* @param yohoIndexName
* @return
... ... @@ -36,7 +36,7 @@ public class YohoIndexHelper {
}
/**
* 为【有货索引名】建立真实索引名
* 建立真实索引名
*
* @param aliasIndexName
* @return
... ... @@ -46,7 +46,7 @@ public class YohoIndexHelper {
}
/**
* 为【有货临时索引名】建立真实索引名
* 建立真实索引名
*
* @param aliasIndexName
* @return
... ...