yoho-search / yoho-search-consumer · Commits

Go to a project

Authored by Gino Zhang
Commit c8b9a150e9c78987376eb032b2f188e007d6c2b8 1 parent 8d48953f

suggest索引增加一个standardKeyword用于排重

Inline Side-by-side
Showing 5 changed files with 17 additions and 65 deletions
... ... @@ -53,7 +53,7 @@ public class SuggestIndexBuilder extends IIndexBuilder {
@Override
public String getId(Object object) {
return MD5Util.string2MD5(((SuggestIndexBO) object).getKeyword().trim().toLowerCase());
return MD5Util.string2MD5(((SuggestIndexBO) object).getStandardKeyword().trim().toLowerCase());
}
}
... ...
package com.yoho.search.consumer.index.fullbuild;
import com.yoho.search.base.utils.MD5Util;
import com.yoho.search.consumer.common.DynamicConfigService;
import com.yoho.search.consumer.index.common.IIndexBuilder;
import com.yoho.search.consumer.service.base.SuggestWordDefService;
import com.yoho.search.consumer.service.bo.SuggestIndexBO;
import com.yoho.search.consumer.suggests.common.KeywordType;
import com.yoho.search.consumer.suggests.common.SuggestionConstants;
import com.yoho.search.dal.model.SuggestWordDef;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Component;
import java.util.ArrayList;
import java.util.List;
import java.util.Set;
import java.util.stream.Collectors;
import java.util.stream.Stream;
@Component
public class SuggestIndexBuilderByConversation extends IIndexBuilder {
@Autowired
private SuggestWordDefService suggestWordDefService;
@Autowired
private DynamicConfigService dynamicConfigService;
@Override
public int getTotalCount() throws Exception {
return suggestWordDefService.selectTotalCount();
}
@Override
public List<?> getPageLists(int offset, int limit) throws Exception {
Set<Integer> enabledKeywordTypes = Stream.of(KeywordType.values()).filter(keywordType -> dynamicConfigService.suggestKeywordTypeOpen(keywordType))
.map(KeywordType::getType).collect(Collectors.toSet());
// 获取分页列表
List<SuggestWordDef> list = suggestWordDefService.selectPageList(offset, limit);
// 构建结果
List<SuggestIndexBO> results = new ArrayList<SuggestIndexBO>();
for (SuggestWordDef suggestWordDef : list) {
if (!suggestWordDef.getStatus().equals(SuggestionConstants.VALID_STATUS) || !enabledKeywordTypes.contains(suggestWordDef.getType())) {
continue;
}
// 根据Type取权重
int weight = KeywordType.getWeightValueByType(suggestWordDef.getType());
results.add(new SuggestIndexBO(suggestWordDef.getKeyword(), suggestWordDef.getType(), weight, suggestWordDef.getCount(), suggestWordDef.getCountForApp(), suggestWordDef.getCountForBlk()));
}
return results;
}
@Override
public String getId(Object object) {
return MD5Util.string2MD5(((SuggestIndexBO) object).getKeyword().trim().toLowerCase());
}
}
... ... @@ -3,6 +3,7 @@ package com.yoho.search.consumer.restapi;
import com.alibaba.fastjson.JSON;
import com.yoho.error.event.SearchEvent;
import com.yoho.error.event.SearchLogsEvent;
import com.yoho.search.base.utils.CharUtils;
import com.yoho.search.base.utils.EventReportEnum;
import com.yoho.search.base.utils.ISearchConstants;
import com.yoho.search.base.utils.MD5Util;
... ... @@ -318,6 +319,7 @@ public class IndexController implements ApplicationEventPublisherAware {
Map<String, Object> dataMap = new HashMap();
dataMap.put("keyword", keyword);
dataMap.put("standardKeyword", CharUtils.standardized(keyword));
dataMap.put("type", suggestWordDef.getType());
dataMap.put("weight", suggestWordDef.getWeight());
dataMap.put("count", suggestWordDef.getCount());
... ...
... ... @@ -13,11 +13,6 @@
"type": "string",
"index": "not_analyzed"
},
"keyword_lowercase": {
"type": "string",
"store": false,
"analyzer": "lowercase_keyword"
},
"keyword_ik": {
"type": "string",
"store": false,
... ... @@ -36,6 +31,11 @@
},
"type": "multi_field"
},
"standardKeyword": {
"type": "string",
"store": false,
"analyzer": "keyword"
},
"type": {
"type": "long",
"doc_values": true,
... ...
package com.yoho.search.consumer.service.bo;
import com.yoho.search.base.utils.CharUtils;
import java.io.Serializable;
public class SuggestIndexBO implements Serializable {
private static final long serialVersionUID = 7154651415633074270L;
private String keyword;
private String standardKeyword;
private int type;
private int weight;
private int count;
... ... @@ -15,6 +18,7 @@ public class SuggestIndexBO implements Serializable {
public SuggestIndexBO(String keyword, int type, int weight, Integer count, Integer countForApp, Integer countForBlk) {
super();
this.keyword = keyword;
this.standardKeyword = CharUtils.standardized(keyword);
this.type = type;
this.weight = weight;
this.count = count != null ? count.intValue() : 0;
... ... @@ -28,6 +32,11 @@ public class SuggestIndexBO implements Serializable {
public void setKeyword(String keyword) {
this.keyword = keyword;
this.standardKeyword = CharUtils.standardized(keyword);
}
public String getStandardKeyword() {
return standardKeyword != null ? standardKeyword : keyword;
}
public int getType() {
... ...