yoho-search / yoho-search-service · Commits

Go to a project

Authored by wangnan
Commit 37cf1329674af1564e499402146b00492b1307d9 1 parent af078044

【图片搜索】 两次查询

Inline Side-by-side
Showing 1 changed file with 37 additions and 18 deletions
... ... @@ -2,6 +2,7 @@ package com.yoho.search.service.servicenew.impl;
import com.alibaba.fastjson.JSONObject;
import com.yoho.error.event.SearchEvent;
import com.yoho.search.base.utils.ConvertUtils;
import com.yoho.search.base.utils.EventReportEnum;
import com.yoho.search.base.utils.ISearchConstants;
import com.yoho.search.base.utils.ProductIndexEsField;
... ... @@ -54,6 +55,8 @@ public class SknImageVectorsServiceImpl implements ISknImageVectorsService, Appl
private static final String SCRIPT_NAME = "yoho_hamming_score";
private static final int VECTORS_32_QUERY_BACK_NUM = 100;
private ApplicationEventPublisher publisher;
@Autowired
... ... @@ -85,13 +88,6 @@ public class SknImageVectorsServiceImpl implements ISknImageVectorsService, Appl
// 1.构造查询参数
String vectors_32 = paramMap.get(VECTORS_32_KEY);
String vectors_128 = paramMap.get(VECTORS_128_KEY);
//两个向量入参都不为空，默认用32位查询
String vectorsFieldName=VECTORS_32_KEY;
String vectorsValue=vectors_32;
if(StringUtils.isBlank(vectors_32)&&StringUtils.isNotBlank(vectors_128)){
vectorsFieldName=VECTORS_128_KEY;
vectorsValue=vectors_128;
}
SearchParam searchParam = new SearchParam();
int pageSize = StringUtils.isBlank(paramMap.get("viewNum")) ? 10 : Integer.parseInt(paramMap.get("viewNum"));
... ... @@ -102,25 +98,41 @@ public class SknImageVectorsServiceImpl implements ISknImageVectorsService, Appl
if (pageSize > 50) {
pageSize = 50;
}
searchParam.setPage(page);
searchParam.setOffset((page - 1) * pageSize);
searchParam.setSize(pageSize);
searchParam.setQuery(buildFunctionScoreQueryBuilder(vectorsFieldName,vectorsValue));
searchParam.setSize(VECTORS_32_QUERY_BACK_NUM);
searchParam.setQuery(buildFunctionScoreQueryBuilder(VECTORS_32_KEY, vectors_32, null));
//2.根据特征，查询SknImageVectors索引
//2.1第一次查询，用32维向量粗查询，召回一些skn
final String indexName = ISearchConstants.INDEX_NAME_IMAGE_VECTORS;
SearchResult searchResult = searchCommonService.doSearch(indexName, searchParam);
SearchApiResult searchApiResult = new SearchApiResult();
if (CollectionUtils.isNotEmpty(searchResult.getResultList())) {
List<Map<String, Object>> sknList = getSknImageMap(searchResult.getResultList());
List<String> querySknList = new ArrayList<>();
String sknFilterString = "";
for (Map<String, Object> map : sknList) {
querySknList.add(map.get("productSkn").toString());
sknFilterString += map.get("productSkn").toString() + ",";
}
//2.2第二次查询,用128维向量粗查询这些skn
searchParam.setPage(page);
searchParam.setOffset((page - 1) * pageSize);
searchParam.setSize(pageSize);
//todo 先用32维的，后面要改成128的
searchParam.setQuery(buildFunctionScoreQueryBuilder(VECTORS_32_KEY, vectors_32, sknFilterString));
searchResult = searchCommonService.doSearch(indexName, searchParam);
if (CollectionUtils.isNotEmpty(searchResult.getResultList())) {
sknList = getSknImageMap(searchResult.getResultList());
List<String> querySknList = new ArrayList<>();
for (Map<String, Object> map : sknList) {
querySknList.add(map.get("productSkn").toString());
}
//3.根据返回的skn列表，查询ProductIndex
searchApiResult = searchProductList(paramMap, querySknList, page, pageSize);
logger.info("[func=searchSknByPhoto][cost={}]", System.currentTimeMillis() - begin);
}
//3.根据返回的skn列表，查询ProductIndex
searchApiResult = searchProductList(paramMap, querySknList, page, pageSize);
}
logger.info("[func=searchSknByPhoto][cost={}]", System.currentTimeMillis() - begin);
return searchApiResult;
} catch (Exception e) {
publisher.publishEvent(new SearchEvent(EventReportEnum.SEARCHCONTROLLER_SEARCHSKNBYPHOTO.getEventName(), EventReportEnum.SEARCHCONTROLLER_SEARCHSKNBYPHOTO.getFunctionName(),
... ... @@ -169,12 +181,10 @@ public class SknImageVectorsServiceImpl implements ISknImageVectorsService, Appl
return new SearchApiResult().setData(photoListData);
}
private FunctionScoreQueryBuilder buildFunctionScoreQueryBuilder(String vectorsFieldName,String vectorsValue) {
private FunctionScoreQueryBuilder buildFunctionScoreQueryBuilder(String vectorsFieldName, String vectorsValue, String sknFilterString) {
MatchAllQueryBuilder matchAllQueryBuilder = QueryBuilders.matchAllQuery().boost(1.0f);
//query
FunctionScoreQueryBuilder functionScoreQueryBuilder = new FunctionScoreQueryBuilder(matchAllQueryBuilder);
//filter
QueryBuilder filter = QueryBuilders.matchAllQuery().boost(1.0f);
//script_score
String[] feaArray = vectorsValue.split(",");
Long[] fea = new Long[feaArray.length];
... ... @@ -190,7 +200,16 @@ public class SknImageVectorsServiceImpl implements ISknImageVectorsService, Appl
Script script = new Script(inlineScript, ScriptService.ScriptType.INLINE, "native", params);
ScriptScoreFunctionBuilder scriptBuilder = ScoreFunctionBuilders.scriptFunction(script);
//function_score
functionScoreQueryBuilder.add(filter, scriptBuilder);
//filter,sknFilterString为空是第一次粗查询，不为空是第二次精查询
if(StringUtils.isBlank(sknFilterString)){
QueryBuilder filter = QueryBuilders.matchAllQuery().boost(1.0f);
functionScoreQueryBuilder.add(filter, scriptBuilder);
}else{
int[] sknArray = ConvertUtils.stringToIntArray(sknFilterString, ",");
BoolQueryBuilder boolFilter = QueryBuilders.boolQuery();
boolFilter.must(QueryBuilders.termsQuery("productSkn", sknArray));
functionScoreQueryBuilder.add(boolFilter, scriptBuilder);
}
functionScoreQueryBuilder.scoreMode("sum");
functionScoreQueryBuilder.boostMode(CombineFunction.REPLACE);
functionScoreQueryBuilder.maxBoost(3.4028235E38F);
... ...