yoho-search / yoho-search-service · Commits

Go to a project

Authored by wangnan
Commit 93a45e4ae180337e71f5ce172b0a78b5e31259b7 2 parents b65542a9 cb3a196c

Merge branch 'wn_photoSearch_doubleQuery' into wn_photoSearch 

# Conflicts:
#	service/src/main/java/com/yoho/search/service/servicenew/impl/SknImageVectorsServiceImpl.java
Inline Side-by-side
Showing 2 changed files with 65 additions and 47 deletions
@@ -34,10 +34,10 @@ public class SknImageVectorsController { @@ -34,10 +34,10 @@ public class SknImageVectorsController {
34 @ResponseBody 34 @ResponseBody
35 public SearchApiResult searchSknByPhoto(@RequestBody JSONObject jsonObject) { 35 public SearchApiResult searchSknByPhoto(@RequestBody JSONObject jsonObject) {
36 try { 36 try {
37 - String vectors_32 = jsonObject.get("vectors")==null?null:jsonObject.get("vectors").toString(); 37 + String vectors_32 = jsonObject.get(VECTORS_32_KEY)==null?null:jsonObject.get(VECTORS_32_KEY).toString();
38 String vectors_128 = jsonObject.get(VECTORS_128_KEY)==null?null:jsonObject.get(VECTORS_128_KEY).toString(); 38 String vectors_128 = jsonObject.get(VECTORS_128_KEY)==null?null:jsonObject.get(VECTORS_128_KEY).toString();
39 - if (StringUtils.isBlank(vectors_32)&&StringUtils.isBlank(vectors_128)) {  
40 - return new SearchApiResult().setCode(400).setMessage("vectors_32与vectors_128不能都为空"); 39 + if (StringUtils.isBlank(vectors_32)||StringUtils.isBlank(vectors_128)) {
  40 + return new SearchApiResult().setCode(400).setMessage("vectors_32或vectors_128不能为空");
41 } 41 }
42 Map<String, String> paramMap = new HashMap<>(); 42 Map<String, String> paramMap = new HashMap<>();
43 paramMap.put(VECTORS_32_KEY, vectors_32); 43 paramMap.put(VECTORS_32_KEY, vectors_32);
1 package com.yoho.search.service.servicenew.impl; 1 package com.yoho.search.service.servicenew.impl;
2 2
3 -import java.util.ArrayList;  
4 -import java.util.HashMap;  
5 -import java.util.LinkedHashMap;  
6 -import java.util.List;  
7 -import java.util.Map;  
8 -  
9 -import javax.annotation.PostConstruct;  
10 - 3 +import com.alibaba.fastjson.JSONObject;
  4 +import com.yoho.error.event.SearchEvent;
  5 +import com.yoho.search.base.utils.ConvertUtils;
  6 +import com.yoho.search.base.utils.EventReportEnum;
  7 +import com.yoho.search.base.utils.ISearchConstants;
  8 +import com.yoho.search.base.utils.ProductIndexEsField;
  9 +import com.yoho.search.core.es.model.SearchParam;
  10 +import com.yoho.search.core.es.model.SearchResult;
  11 +import com.yoho.search.core.es.utils.IgnoreSomeException;
  12 +import com.yoho.search.service.cache.SearchCacheFactory;
  13 +import com.yoho.search.service.cache.model.SearchCache;
  14 +import com.yoho.search.service.service.SearchCacheService;
  15 +import com.yoho.search.service.service.SearchCommonService;
  16 +import com.yoho.search.service.service.base.ProductIndexBaseService;
  17 +import com.yoho.search.service.servicenew.ISknImageVectorsService;
  18 +import com.yoho.search.service.utils.SearchApiResultUtils;
  19 +import com.yoho.search.service.vo.SearchApiResult;
11 import org.apache.commons.collections.CollectionUtils; 20 import org.apache.commons.collections.CollectionUtils;
12 -import org.apache.commons.lang.StringUtils;  
13 import org.elasticsearch.common.lucene.search.function.CombineFunction; 21 import org.elasticsearch.common.lucene.search.function.CombineFunction;
14 import org.elasticsearch.index.query.BoolQueryBuilder; 22 import org.elasticsearch.index.query.BoolQueryBuilder;
15 import org.elasticsearch.index.query.MatchAllQueryBuilder; 23 import org.elasticsearch.index.query.MatchAllQueryBuilder;
@@ -18,7 +26,6 @@ import org.elasticsearch.index.query.QueryBuilders; @@ -18,7 +26,6 @@ import org.elasticsearch.index.query.QueryBuilders;
18 import org.elasticsearch.index.query.functionscore.FunctionScoreQueryBuilder; 26 import org.elasticsearch.index.query.functionscore.FunctionScoreQueryBuilder;
19 import org.elasticsearch.index.query.functionscore.ScoreFunctionBuilders; 27 import org.elasticsearch.index.query.functionscore.ScoreFunctionBuilders;
20 import org.elasticsearch.index.query.functionscore.script.ScriptScoreFunctionBuilder; 28 import org.elasticsearch.index.query.functionscore.script.ScriptScoreFunctionBuilder;
21 -import org.elasticsearch.script.Script;  
22 import org.elasticsearch.script.ScriptService; 29 import org.elasticsearch.script.ScriptService;
23 import org.elasticsearch.search.sort.SortBuilder; 30 import org.elasticsearch.search.sort.SortBuilder;
24 import org.elasticsearch.search.sort.SortBuilders; 31 import org.elasticsearch.search.sort.SortBuilders;
@@ -30,22 +37,15 @@ import org.springframework.context.ApplicationEventPublisher; @@ -30,22 +37,15 @@ import org.springframework.context.ApplicationEventPublisher;
30 import org.springframework.context.ApplicationEventPublisherAware; 37 import org.springframework.context.ApplicationEventPublisherAware;
31 import org.springframework.stereotype.Service; 38 import org.springframework.stereotype.Service;
32 39
33 -import com.alibaba.fastjson.JSONObject;  
34 -import com.yoho.error.event.SearchEvent;  
35 -import com.yoho.search.base.utils.EventReportEnum;  
36 -import com.yoho.search.base.utils.ISearchConstants;  
37 -import com.yoho.search.base.utils.ProductIndexEsField;  
38 -import com.yoho.search.core.es.model.SearchParam;  
39 -import com.yoho.search.core.es.model.SearchResult;  
40 -import com.yoho.search.core.es.utils.IgnoreSomeException;  
41 -import com.yoho.search.service.cache.SearchCacheFactory;  
42 -import com.yoho.search.service.cache.model.SearchCache;  
43 -import com.yoho.search.service.service.SearchCacheService;  
44 -import com.yoho.search.service.service.SearchCommonService;  
45 -import com.yoho.search.service.service.base.ProductIndexBaseService;  
46 -import com.yoho.search.service.servicenew.ISknImageVectorsService;  
47 -import com.yoho.search.service.utils.SearchApiResultUtils;  
48 -import com.yoho.search.service.vo.SearchApiResult; 40 +import javax.annotation.PostConstruct;
  41 +import java.util.ArrayList;
  42 +import java.util.HashMap;
  43 +import java.util.LinkedHashMap;
  44 +import java.util.List;
  45 +import java.util.Map;
  46 +
  47 +import org.apache.commons.lang.StringUtils;
  48 +import org.elasticsearch.script.Script;
49 49
50 /** 50 /**
51 * Created by wangnan on 2017/7/28. 51 * Created by wangnan on 2017/7/28.
@@ -60,6 +60,8 @@ public class SknImageVectorsServiceImpl implements ISknImageVectorsService, Appl @@ -60,6 +60,8 @@ public class SknImageVectorsServiceImpl implements ISknImageVectorsService, Appl
60 60
61 private static final String SCRIPT_NAME = "yoho_hamming_score"; 61 private static final String SCRIPT_NAME = "yoho_hamming_score";
62 62
  63 + private static final int VECTORS_32_QUERY_BACK_NUM = 100;
  64 +
63 private ApplicationEventPublisher publisher; 65 private ApplicationEventPublisher publisher;
64 66
65 @Autowired 67 @Autowired
@@ -91,13 +93,6 @@ public class SknImageVectorsServiceImpl implements ISknImageVectorsService, Appl @@ -91,13 +93,6 @@ public class SknImageVectorsServiceImpl implements ISknImageVectorsService, Appl
91 // 1.构造查询参数 93 // 1.构造查询参数
92 String vectors_32 = paramMap.get(VECTORS_32_KEY); 94 String vectors_32 = paramMap.get(VECTORS_32_KEY);
93 String vectors_128 = paramMap.get(VECTORS_128_KEY); 95 String vectors_128 = paramMap.get(VECTORS_128_KEY);
94 - //两个向量入参都不为空，默认用32位查询  
95 - String vectorsFieldName=VECTORS_32_KEY;  
96 - String vectorsValue=vectors_32;  
97 - if(StringUtils.isBlank(vectors_32)&&StringUtils.isNotBlank(vectors_128)){  
98 - vectorsFieldName=VECTORS_128_KEY;  
99 - vectorsValue=vectors_128;  
100 - }  
101 96
102 SearchParam searchParam = new SearchParam(); 97 SearchParam searchParam = new SearchParam();
103 int pageSize = StringUtils.isBlank(paramMap.get("viewNum")) ? 10 : Integer.parseInt(paramMap.get("viewNum")); 98 int pageSize = StringUtils.isBlank(paramMap.get("viewNum")) ? 10 : Integer.parseInt(paramMap.get("viewNum"));
@@ -108,24 +103,40 @@ public class SknImageVectorsServiceImpl implements ISknImageVectorsService, Appl @@ -108,24 +103,40 @@ public class SknImageVectorsServiceImpl implements ISknImageVectorsService, Appl
108 if (pageSize > 50) { 103 if (pageSize > 50) {
109 pageSize = 50; 104 pageSize = 50;
110 } 105 }
  106 + searchParam.setPage(page);
111 searchParam.setOffset((page - 1) * pageSize); 107 searchParam.setOffset((page - 1) * pageSize);
112 - searchParam.setSize(pageSize);  
113 - searchParam.setQuery(buildFunctionScoreQueryBuilder(vectorsFieldName,vectorsValue)); 108 + searchParam.setSize(VECTORS_32_QUERY_BACK_NUM);
  109 + searchParam.setQuery(buildFunctionScoreQueryBuilder(VECTORS_32_KEY, vectors_32, null));
114 110
115 //2.根据特征，查询SknImageVectors索引 111 //2.根据特征，查询SknImageVectors索引
  112 + //2.1第一次查询，用32维向量粗查询，召回一些skn
116 final String indexName = ISearchConstants.INDEX_NAME_IMAGE_VECTORS; 113 final String indexName = ISearchConstants.INDEX_NAME_IMAGE_VECTORS;
117 SearchResult searchResult = searchCommonService.doSearch(indexName, searchParam); 114 SearchResult searchResult = searchCommonService.doSearch(indexName, searchParam);
118 SearchApiResult searchApiResult = new SearchApiResult(); 115 SearchApiResult searchApiResult = new SearchApiResult();
119 if (CollectionUtils.isNotEmpty(searchResult.getResultList())) { 116 if (CollectionUtils.isNotEmpty(searchResult.getResultList())) {
120 List<Map<String, Object>> sknList = getSknImageMap(searchResult.getResultList()); 117 List<Map<String, Object>> sknList = getSknImageMap(searchResult.getResultList());
121 - List<String> querySknList = new ArrayList<>(); 118 + String sknFilterString = "";
122 for (Map<String, Object> map : sknList) { 119 for (Map<String, Object> map : sknList) {
123 - querySknList.add(map.get("productSkn").toString()); 120 + sknFilterString += map.get("productSkn").toString() + ",";
  121 + }
  122 + //2.2第二次查询,用128维向量粗查询这些skn
  123 + searchParam.setPage(page);
  124 + searchParam.setOffset((page - 1) * pageSize);
  125 + searchParam.setSize(pageSize);
  126 + //todo 先用32维的，后面要改成128的
  127 + searchParam.setQuery(buildFunctionScoreQueryBuilder(VECTORS_32_KEY, vectors_32, sknFilterString));
  128 + searchResult = searchCommonService.doSearch(indexName, searchParam);
  129 + if (CollectionUtils.isNotEmpty(searchResult.getResultList())) {
  130 + sknList = getSknImageMap(searchResult.getResultList());
  131 + List<String> querySknList = new ArrayList<>();
  132 + for (Map<String, Object> map : sknList) {
  133 + querySknList.add(map.get("productSkn").toString());
  134 + }
  135 + //3.根据返回的skn列表，查询ProductIndex
  136 + searchApiResult = searchProductList(paramMap, querySknList, page, pageSize);
  137 + logger.info("[func=searchSknByPhoto][cost={}]", System.currentTimeMillis() - begin);
124 } 138 }
125 - //3.根据返回的skn列表，查询ProductIndex  
126 - searchApiResult = searchProductList(paramMap, querySknList, page, pageSize);  
127 } 139 }
128 - logger.info("[func=searchSknByPhoto][cost={}]", System.currentTimeMillis() - begin);  
129 return searchApiResult; 140 return searchApiResult;
130 } catch (Exception e) { 141 } catch (Exception e) {
131 publisher.publishEvent(new SearchEvent(EventReportEnum.SEARCHCONTROLLER_SEARCHSKNBYPHOTO.getEventName(), EventReportEnum.SEARCHCONTROLLER_SEARCHSKNBYPHOTO.getFunctionName(), 142 publisher.publishEvent(new SearchEvent(EventReportEnum.SEARCHCONTROLLER_SEARCHSKNBYPHOTO.getEventName(), EventReportEnum.SEARCHCONTROLLER_SEARCHSKNBYPHOTO.getFunctionName(),
@@ -173,12 +184,10 @@ public class SknImageVectorsServiceImpl implements ISknImageVectorsService, Appl @@ -173,12 +184,10 @@ public class SknImageVectorsServiceImpl implements ISknImageVectorsService, Appl
173 return new SearchApiResult().setData(photoListData); 184 return new SearchApiResult().setData(photoListData);
174 } 185 }
175 186
176 - private FunctionScoreQueryBuilder buildFunctionScoreQueryBuilder(String vectorsFieldName,String vectorsValue) { 187 + private FunctionScoreQueryBuilder buildFunctionScoreQueryBuilder(String vectorsFieldName, String vectorsValue, String sknFilterString) {
177 MatchAllQueryBuilder matchAllQueryBuilder = QueryBuilders.matchAllQuery().boost(1.0f); 188 MatchAllQueryBuilder matchAllQueryBuilder = QueryBuilders.matchAllQuery().boost(1.0f);
178 //query 189 //query
179 FunctionScoreQueryBuilder functionScoreQueryBuilder = new FunctionScoreQueryBuilder(matchAllQueryBuilder); 190 FunctionScoreQueryBuilder functionScoreQueryBuilder = new FunctionScoreQueryBuilder(matchAllQueryBuilder);
180 - //filter  
181 - QueryBuilder filter = QueryBuilders.matchAllQuery().boost(1.0f);  
182 //script_score 191 //script_score
183 String[] feaArray = vectorsValue.split(","); 192 String[] feaArray = vectorsValue.split(",");
184 Long[] fea = new Long[feaArray.length]; 193 Long[] fea = new Long[feaArray.length];
@@ -194,7 +203,16 @@ public class SknImageVectorsServiceImpl implements ISknImageVectorsService, Appl @@ -194,7 +203,16 @@ public class SknImageVectorsServiceImpl implements ISknImageVectorsService, Appl
194 Script script = new Script(inlineScript, ScriptService.ScriptType.INLINE, "native", params); 203 Script script = new Script(inlineScript, ScriptService.ScriptType.INLINE, "native", params);
195 ScriptScoreFunctionBuilder scriptBuilder = ScoreFunctionBuilders.scriptFunction(script); 204 ScriptScoreFunctionBuilder scriptBuilder = ScoreFunctionBuilders.scriptFunction(script);
196 //function_score 205 //function_score
197 - functionScoreQueryBuilder.add(filter, scriptBuilder); 206 + //filter,sknFilterString为空是第一次粗查询，不为空是第二次精查询
  207 + if (StringUtils.isBlank(sknFilterString)) {
  208 + QueryBuilder filter = QueryBuilders.matchAllQuery().boost(1.0f);
  209 + functionScoreQueryBuilder.add(filter, scriptBuilder);
  210 + } else {
  211 + int[] sknArray = ConvertUtils.stringToIntArray(sknFilterString, ",");
  212 + BoolQueryBuilder boolFilter = QueryBuilders.boolQuery();
  213 + boolFilter.must(QueryBuilders.termsQuery("productSkn", sknArray));
  214 + functionScoreQueryBuilder.add(boolFilter, scriptBuilder);
  215 + }
198 functionScoreQueryBuilder.scoreMode("sum"); 216 functionScoreQueryBuilder.scoreMode("sum");
199 functionScoreQueryBuilder.boostMode(CombineFunction.REPLACE); 217 functionScoreQueryBuilder.boostMode(CombineFunction.REPLACE);
200 functionScoreQueryBuilder.maxBoost(3.4028235E38F); 218 functionScoreQueryBuilder.maxBoost(3.4028235E38F);