yoho-search / yohomars-search-service · Commits

Go to a project

Authored by all4you
Commit 9ddf3a17ca7374ba2f60ff92280faa3d8f3b8b57 1 parent 060d9fe1

update

Inline Side-by-side
Showing 10 changed files with 92 additions and 34 deletions
... ... @@ -466,7 +466,7 @@ public class ElasticsearchClientImpl implements IElasticsearchClient {
if (searchParam.getSearchType() != null) {
request.searchType(searchParam.getSearchType());
}
logger.info("[client={}] [query=\n{}]", name, sourceBuilder.toString());
logger.debug("[client={}] [query=\n{}]", name, sourceBuilder.toString());
return client.search(request);
}
});
... ...
... ... @@ -38,6 +38,7 @@ public class SocialUserIndexBuilder extends IIndexBuilder {
private Long lastUpdateTime;
private Long getLastUpdateTime(){
// return 1504050513724L;
return redisValueHelper.get(Index.social_user.getKey(),Long.class);
}
... ...
... ... @@ -56,7 +56,7 @@ public class YohoIndexDataLoader implements ApplicationContextAware {
final int limit = ISearchConstans.SEARCH_INDEX_BATCH_LIMIT;
// 总页数
final int totalPageSize = (int) ((totalCount - 1) / limit + 1);
INDEX_REBUILD_LOG.info("[yohomarsIndexName=[{}]],[loadAllData],[totalPageSize={}]", yohoIndexName, totalPageSize);
INDEX_REBUILD_LOG.info("loadAllData [yohoIndexName=[{}]],[loadAllData],[totalPageSize={}]", yohoIndexName, totalPageSize);
// 3、分配任务、并包装异步执行结果
List<Future<Boolean>> futureResults = new ArrayList<Future<Boolean>>();
for (int pageNo = 1; pageNo <= totalPageSize; pageNo++) {
... ... @@ -115,7 +115,7 @@ public class YohoIndexDataLoader implements ApplicationContextAware {
final int limit = ISearchConstans.SEARCH_INDEX_BATCH_LIMIT;
// 总页数
final int totalPageSize = (int) ((totalCount - 1) / limit + 1);
INDEX_REBUILD_LOG.info("[yohomarsIndexName=[{}]],[loadAppendData],[totalCount={}],[totalPageSize={}]", yohoIndexName, totalCount, totalPageSize);
INDEX_REBUILD_LOG.info("loadAppendData [yohoIndexName=[{}]],[loadAppendData],[totalCount={}],[totalPageSize={}]", yohoIndexName, totalCount, totalPageSize);
// 3、分配任务、并包装异步执行结果
List<Future<Boolean>> futureResults = new ArrayList<Future<Boolean>>();
for (int pageNo = 1; pageNo <= totalPageSize; pageNo++) {
... ... @@ -158,7 +158,7 @@ public class YohoIndexDataLoader implements ApplicationContextAware {
try {
List<?> dataList = indexBuilder.getUpdatedLists();
int resultSize = dataList != null ? dataList.size() : 0;
INDEX_REBUILD_LOG.info("[yohoIndexName=[{}]],[{resultSize={}]", yohoIndexName, resultSize);
INDEX_REBUILD_LOG.info("loadUpdatedData [yohoIndexName=[{}]],[{resultSize={}]", yohoIndexName, resultSize);
if (resultSize > 0) {
performanceMonitor.addVisitCount();
long begin = System.currentTimeMillis();
... ...
... ... @@ -449,7 +449,6 @@ public class YohoIndexServiceImpl implements IYohoIndexService {
if (yohoIndex == null) {
return;
}
Index index = Index.getIndex(yohoIndexName);
if(!this.indexExists(yohoIndexName)){
INDEX_REBUILD_LOG.info("YohoIndex does not exists when buildUpdated with indexName=[{}] will call rebuild instead", yohoIndexName);
this.rebuild(yohoIndexName);
... ...
... ... @@ -97,12 +97,16 @@ public class ISearchConstans {
*/
public static final String SOCIAL_USER_S_UID = "s_uid";
public static final String SOCIAL_USER_S_NICK_NAME = "s_nick_name";
public static final String SOCIAL_USER_S_NICK_NAME_IK = "s_nick_name_ik";
public static final String SOCIAL_USER_S_NICK_NAME_NO_ANALYZER = "s_nick_name_no_analyzer";
public static final String SOCIAL_USER_S_HEAD_PIC = "s_head_pic";
public static final String SOCIAL_USER_S_TALENT = "s_talent";
public static final String SOCIAL_USER_S_FOLLOWERS_NUMBER = "s_followers_number";
public static final String SOCIAL_USER_S_FOLLOWING = "s_following";
public static final String SOCIAL_USER_T_UID = "t_uid";
public static final String SOCIAL_USER_T_NICK_NAME = "t_nick_name";
public static final String SOCIAL_USER_T_NICK_NAME_IK = "t_nick_name_ik";
public static final String SOCIAL_USER_T_NICK_NAME_NO_ANALYZER = "t_nick_name_no_analyzer";
public static final String SOCIAL_USER_T_HEAD_PIC = "t_head_pic";
public static final String SOCIAL_USER_T_FOLLOWER_NUMBERS = "t_follower_numbers";
public static final String SOCIAL_USER_T_FOLLOW_BACK = "t_follow_back";
... ...
... ... @@ -59,7 +59,6 @@ public class IndexRebuildJob {
/**
* 重建索引
* @param indexName
*/
private void rebuildIndex(String indexName) {
try {
... ...
... ... @@ -63,7 +63,7 @@ public class SocialUserService extends BaseService {
*/
private SearchResult matchEqually(SocialUserRequest request,SearchParam searchParam){
// 构建filter ==> s_nick_name==query_name
searchParam.setFilter(QueryBuilders.termQuery(ISearchConstans.SOCIAL_USER_S_NICK_NAME, request.getQueryName()));
searchParam.setFilter(QueryBuilders.termQuery(ISearchConstans.SOCIAL_USER_S_NICK_NAME+"."+ISearchConstans.SOCIAL_USER_S_NICK_NAME_NO_ANALYZER, request.getQueryName()));
// 构建query ==> matchAll
searchParam.setQuery(QueryBuilders.matchAllQuery());
// 指定excludes字段
... ... @@ -83,7 +83,7 @@ public class SocialUserService extends BaseService {
// 构建filter ==> s_uid==current_uid
searchParam.setFilter(QueryBuilders.termQuery(ISearchConstans.SOCIAL_USER_S_UID, request.getUid()));
// 构建query ==> s_nick_name match query_name TODO matchQuery是模糊匹配还是精确匹配
searchParam.setQuery(QueryBuilders.matchQuery(ISearchConstans.SOCIAL_USER_S_NICK_NAME, request.getQueryName()));
searchParam.setQuery(QueryBuilders.matchQuery(ISearchConstans.SOCIAL_USER_S_NICK_NAME+"."+ISearchConstans.SOCIAL_USER_S_NICK_NAME_IK, request.getQueryName()));
// 指定excludes字段
searchParam.setSourceExcludes(SOCIAL_USER_S_EXCLUDES);
return searchCommonService.doSearch(SOCIAL_USER_INDEX_NAME, searchParam);
... ... @@ -102,8 +102,8 @@ public class SocialUserService extends BaseService {
// 构建filter ==> s_uid==current_uid
searchParam.setFilter(QueryBuilders.termQuery(ISearchConstans.SOCIAL_USER_S_UID, request.getUid()));
// 构造query ==> t_nick_name match query_name
QueryBuilder termQuery = QueryBuilders.termQuery(ISearchConstans.SOCIAL_USER_S_FOLLOWING+"."+ISearchConstans.SOCIAL_USER_T_NICK_NAME, request.getQueryName());
QueryBuilder nestedQuery = QueryBuilders.nestedQuery(ISearchConstans.SOCIAL_USER_S_FOLLOWING,termQuery)
QueryBuilder matchQuery = QueryBuilders.matchQuery(ISearchConstans.SOCIAL_USER_S_FOLLOWING+"."+ISearchConstans.SOCIAL_USER_T_NICK_NAME+"."+ISearchConstans.SOCIAL_USER_T_NICK_NAME_IK, request.getQueryName());
QueryBuilder nestedQuery = QueryBuilders.nestedQuery(ISearchConstans.SOCIAL_USER_S_FOLLOWING,matchQuery)
.innerHit(new QueryInnerHitBuilder());
searchParam.setQuery(nestedQuery);
searchParam.setInnerHits(true);
... ... @@ -137,7 +137,7 @@ public class SocialUserService extends BaseService {
boolFilter.mustNot(QueryBuilders.termQuery(ISearchConstans.SOCIAL_USER_S_UID, request.getUid()));
searchParam.setFilter(boolFilter);
// 构建query ==> s_nick_name match query_name
searchParam.setQuery(QueryBuilders.matchQuery(ISearchConstans.SOCIAL_USER_S_NICK_NAME, request.getQueryName()));
searchParam.setQuery(QueryBuilders.matchQuery(ISearchConstans.SOCIAL_USER_S_NICK_NAME+"."+ISearchConstans.SOCIAL_USER_S_NICK_NAME_IK, request.getQueryName()));
// 构建order by ==> s_talent desc,s_followers_number desc
LinkedHashMap<String, String> sortFields = new LinkedHashMap<>();
sortFields.put(ISearchConstans.SOCIAL_USER_S_TALENT, SortOrder.DESC.toString());
... ... @@ -173,7 +173,7 @@ public class SocialUserService extends BaseService {
private List<SocialUserResponse.User> transfer(int rule, List<Map<String, Object>> resultList){
List<SocialUserResponse.User> userList = new ArrayList<>();
LOGGER.info("rule={},resultList={}",rule,resultList);
LOGGER.debug("rule={},resultList={}",rule,resultList);
switch (rule){
case RULE_MATCH_EQUALLY:
case RULE_MATCH_MYSELF:
... ... @@ -290,7 +290,7 @@ public class SocialUserService extends BaseService {
response.setRule(rule);
response.setPage(page);
long end = System.currentTimeMillis();
LOGGER.info("[model=SocialUserService][func=search][request={}][end={}][cost={}ms]", request, end, end - begin);
LOGGER.info("[model=SocialUserService][func=search][request={}][rule={}][page={}][end={}][cost={}ms]", request,rule,page,end,end - begin);
return response;
}
... ...
... ... @@ -15,11 +15,20 @@
},
"s_nick_name": {
"type": "string",
"analyzer": "ik_max_word_syno"
"fields":{
"s_nick_name_ik":{
"type": "string",
"analyzer": "ik_complex"
},
"s_nick_name_no_analyzer":{
"type": "string",
"index": "not_analyzed"
}
}
},
"s_head_pic": {
"type": "string",
"analyzer": "ik_max_word_syno"
"index": "not_analyzed"
},
"s_talent": {
"type": "integer"
... ... @@ -28,7 +37,35 @@
"type": "integer"
},
"s_following": {
"type": "nested"
"type": "nested",
"properties": {
"t_uid": {
"type": "integer"
},
"t_nick_name": {
"type": "string",
"fields":{
"t_nick_name_ik":{
"type": "string",
"analyzer": "ik_complex"
},
"t_nick_name_no_analyzer":{
"type": "string",
"index": "not_analyzed"
}
}
},
"t_head_pic": {
"type": "string",
"index": "not_analyzed"
},
"t_followers_number": {
"type": "integer"
},
"t_follow_back": {
"type": "integer"
}
}
}
}
}
... ...
... ... @@ -12,9 +12,10 @@
<!-- ConsoleAppender 控制台输出日志 -->
<appender name="STDOUT" class="ch.qos.logback.core.ConsoleAppender">
<!-- 对日志进行格式化 -->
<encoder>
<encoder class="ch.qos.logback.classic.encoder.PatternLayoutEncoder">
<pattern>%d{yyyy-MM-dd HH:mm:ss,SSS} [%thread] %-5level %logger:%line -%msg%n</pattern>
</encoder>
<charset>UTF-8</charset>
</encoder>
</appender>
<!-- DEBUG、INFO、WARN、ERROR级别日志 appender -->
... ... @@ -30,8 +31,9 @@
<!-- keep 30 days' worth of history -->
<maxHistory>7</maxHistory>
</rollingPolicy>
<encoder>
<encoder class="ch.qos.logback.classic.encoder.PatternLayoutEncoder">
<pattern>%d{yyyy-MM-dd HH:mm:ss.SSS} [%thread] %-5level %logger:%line - %msg%n</pattern>
<charset>UTF-8</charset>
</encoder>
</appender>
... ... @@ -52,8 +54,9 @@
<!-- keep 30 days' worth of history -->
<maxHistory>7</maxHistory>
</rollingPolicy>
<encoder>
<encoder class="ch.qos.logback.classic.encoder.PatternLayoutEncoder">
<pattern>%d{yyyy-MM-dd HH:mm:ss.SSS} [%thread] %-5level %logger:%line - %msg%n</pattern>
<charset>UTF-8</charset>
</encoder>
</appender>
... ... @@ -71,8 +74,9 @@
<!-- keep 30 days' worth of history -->
<maxHistory>7</maxHistory>
</rollingPolicy>
<encoder>
<encoder class="ch.qos.logback.classic.encoder.PatternLayoutEncoder">
<pattern>%d{yyyy-MM-dd HH:mm:ss.SSS} [%thread] %-5level %logger:%line - %msg%n</pattern>
<charset>UTF-8</charset>
</encoder>
</appender>
... ... @@ -89,8 +93,9 @@
<!-- keep 30 days' worth of history -->
<maxHistory>7</maxHistory>
</rollingPolicy>
<encoder>
<encoder class="ch.qos.logback.classic.encoder.PatternLayoutEncoder">
<pattern>%d{yyyy-MM-dd HH:mm:ss.SSS} [%thread] %-5level %logger:%line - %msg%n</pattern>
<charset>UTF-8</charset>
</encoder>
</appender>
... ... @@ -107,8 +112,9 @@
<!-- keep 30 days' worth of history -->
<maxHistory>7</maxHistory>
</rollingPolicy>
<encoder>
<encoder class="ch.qos.logback.classic.encoder.PatternLayoutEncoder">
<pattern>%d{yyyy-MM-dd HH:mm:ss.SSS} [%thread] %-5level %logger:%line - %msg%n</pattern>
<charset>UTF-8</charset>
</encoder>
</appender>
... ... @@ -125,8 +131,9 @@
<!-- keep 30 days' worth of history -->
<maxHistory>7</maxHistory>
</rollingPolicy>
<encoder>
<encoder class="ch.qos.logback.classic.encoder.PatternLayoutEncoder">
<pattern>%d{yyyy-MM-dd HH:mm:ss.SSS} [%thread] %-5level %logger:%line - %msg%n</pattern>
<charset>UTF-8</charset>
</encoder>
</appender>
... ... @@ -143,8 +150,9 @@
<!-- keep 30 days' worth of history -->
<maxHistory>7</maxHistory>
</rollingPolicy>
<encoder>
<encoder class="ch.qos.logback.classic.encoder.PatternLayoutEncoder">
<pattern>%d{yyyy-MM-dd HH:mm:ss.SSS} [%thread] %-5level %logger:%line - %msg%n</pattern>
<charset>UTF-8</charset>
</encoder>
</appender>
... ... @@ -161,8 +169,9 @@
<!-- keep 30 days' worth of history -->
<maxHistory>7</maxHistory>
</rollingPolicy>
<encoder>
<encoder class="ch.qos.logback.classic.encoder.PatternLayoutEncoder">
<pattern>%d{yyyy-MM-dd HH:mm:ss.SSS} [%thread] %-5level %logger:%line - %msg%n</pattern>
<charset>UTF-8</charset>
</encoder>
</appender>
... ...
... ... @@ -22,8 +22,10 @@
<!-- keep 30 days' worth of history -->
<maxHistory>${yoho.logs.maxHistory}</maxHistory>
</rollingPolicy>
<encoder>
<encoder class="ch.qos.logback.classic.encoder.PatternLayoutEncoder">
<pattern>%d{yyyy-MM-dd HH:mm:ss.SSS} [%thread] %-5level %logger:%line - %msg%n</pattern>
<charset>UTF-8</charset>
<charset>UTF-8</charset>
</encoder>
</appender>
... ... @@ -44,8 +46,9 @@
<!-- keep 30 days' worth of history -->
<maxHistory>${yoho.logs.maxHistory}</maxHistory>
</rollingPolicy>
<encoder>
<encoder class="ch.qos.logback.classic.encoder.PatternLayoutEncoder">
<pattern>%d{yyyy-MM-dd HH:mm:ss.SSS} [%thread] %-5level %logger:%line - %msg%n</pattern>
<charset>UTF-8</charset>
</encoder>
</appender>
... ... @@ -63,8 +66,9 @@
<!-- keep 30 days' worth of history -->
<maxHistory>${yoho.logs.maxHistory}</maxHistory>
</rollingPolicy>
<encoder>
<encoder class="ch.qos.logback.classic.encoder.PatternLayoutEncoder">
<pattern>%d{yyyy-MM-dd HH:mm:ss.SSS} [%thread] %-5level %logger:%line - %msg%n</pattern>
<charset>UTF-8</charset>
</encoder>
</appender>
... ... @@ -81,8 +85,9 @@
<!-- keep 30 days' worth of history -->
<maxHistory>${yoho.logs.maxHistory}</maxHistory>
</rollingPolicy>
<encoder>
<encoder class="ch.qos.logback.classic.encoder.PatternLayoutEncoder">
<pattern>%d{yyyy-MM-dd HH:mm:ss.SSS} [%thread] %-5level %logger:%line - %msg%n</pattern>
<charset>UTF-8</charset>
</encoder>
</appender>
... ... @@ -99,8 +104,9 @@
<!-- keep 30 days' worth of history -->
<maxHistory>${yoho.logs.maxHistory}</maxHistory>
</rollingPolicy>
<encoder>
<encoder class="ch.qos.logback.classic.encoder.PatternLayoutEncoder">
<pattern>%d{yyyy-MM-dd HH:mm:ss.SSS} [%thread] %-5level %logger:%line - %msg%n</pattern>
<charset>UTF-8</charset>
</encoder>
</appender>
... ... @@ -117,8 +123,9 @@
<!-- keep 30 days' worth of history -->
<maxHistory>${yoho.logs.maxHistory}</maxHistory>
</rollingPolicy>
<encoder>
<encoder class="ch.qos.logback.classic.encoder.PatternLayoutEncoder">
<pattern>%d{yyyy-MM-dd HH:mm:ss.SSS} [%thread] %-5level %logger:%line - %msg%n</pattern>
<charset>UTF-8</charset>
</encoder>
</appender>
... ... @@ -135,8 +142,9 @@
<!-- keep 30 days' worth of history -->
<maxHistory>${yoho.logs.maxHistory}</maxHistory>
</rollingPolicy>
<encoder>
<encoder class="ch.qos.logback.classic.encoder.PatternLayoutEncoder">
<pattern>%d{yyyy-MM-dd HH:mm:ss.SSS} [%thread] %-5level %logger:%line - %msg%n</pattern>
<charset>UTF-8</charset>
</encoder>
</appender>
... ... @@ -153,8 +161,9 @@
<!-- keep 30 days' worth of history -->
<maxHistory>${yoho.logs.maxHistory}</maxHistory>
</rollingPolicy>
<encoder>
<encoder class="ch.qos.logback.classic.encoder.PatternLayoutEncoder">
<pattern>%d{yyyy-MM-dd HH:mm:ss.SSS} [%thread] %-5level %logger:%line - %msg%n</pattern>
<charset>UTF-8</charset>
</encoder>
</appender>
... ...