yoho-search / yoho-search-service · Commits

Go to a project

Authored by hugufei
Commit 00ebaebd9afe097c4ec8a3b6bd3f548f53fdca5c 1 parent 63c5f7b0

代码优化

Inline Side-by-side
Showing 16 changed files with 592 additions and 339 deletions
set JAVA_MEM_OPT= -Xms1024m -Xmx1024m -XX:MaxPermSize=128m -Djava.net.preferIPv4Stack=true
set JAVA_HOME=/usr/local/java
set WEB_APP_HOME=/home/dev/yoho-search-service/deploy
set WEB_APP_HOME=/home/master/yoho-search-service/deploy
set CATALINA_HOME=/usr/local/tomcat
set CATALINA_BASE=/home/dev/yoho-search-service/deploy/tomcat_server
set CATALINA_BASE=/home/master/yoho-search-service/deploy/tomcat_server
rem set JAVA_DEBUG_OPT= -server -Xdebug -Xnoagent -Djava.compiler=NONE -Xrunjdwp:transport=dt_socket,address=8082,server=y,suspend=n
... ...
... ... @@ -3,16 +3,15 @@ package com.yoho.search.recall.scene;
import com.alibaba.fastjson.JSONObject;
import com.yoho.search.models.SearchApiResult;
import com.yoho.search.recall.scene.helper.RecallCommonService;
import com.yoho.search.recall.scene.models.RecallRequest;
import com.yoho.search.recall.scene.models.*;
import com.yoho.search.recall.scene.request.BrandRecallRequestBuilder;
import com.yoho.search.recall.scene.request.OtherRecallRequestBuilder;
import com.yoho.search.recall.scene.models.RecallResponse;
import com.yoho.search.recall.scene.models.RecallRequestTypeEnum;
import com.yoho.search.recall.scene.request.CommonRecallRequestBuilder;
import com.yoho.search.recall.scene.persional.PersionalFactor;
import com.yoho.search.recall.scene.persional.RecallPersionalService;
import com.yoho.search.recall.sort.helper.RecallServiceHelper;
import com.yoho.search.service.helper.SearchCommonHelper;
import com.yoho.search.service.helper.SearchServiceHelper;
import org.apache.commons.collections.MapUtils;
import org.apache.commons.lang.StringUtils;
import org.elasticsearch.index.query.BoolQueryBuilder;
import org.elasticsearch.index.query.QueryBuilder;
... ... @@ -29,9 +28,9 @@ public class SceneRecallService {
@Autowired
private RecallPersionalService recallPersionalService;
@Autowired
private OtherRecallRequestBuilder otherRecallRequest;
private CommonRecallRequestBuilder commonRecallRequestBuilder;
@Autowired
private BrandRecallRequestBuilder brandRecallRequest;
private BrandRecallRequestBuilder brandRecallRequestBuilder;
@Autowired
private RecallCommonService recallCommonService;
@Autowired
... ... @@ -39,7 +38,7 @@ public class SceneRecallService {
@Autowired
private SearchCommonHelper searchCommonHelper;
public SearchApiResult sceneRecall(Map<String, String> paramMap){
public SearchApiResult sceneRecall(Map<String, String> paramMap) {
try {
//1、分页参数验证
int pageSize = StringUtils.isBlank(paramMap.get("viewNum")) ? 10 : Integer.parseInt(paramMap.get("viewNum"));
... ... @@ -47,27 +46,18 @@ public class SceneRecallService {
if (page < 1 || pageSize < 0 || page * pageSize > 1000000) {
return new SearchApiResult().setCode(400).setMessage("分页参数不合法");
}
//2、构造queryAndFilter
QueryBuilder query = searchServiceHelepr.constructQueryBuilder(paramMap);
BoolQueryBuilder filter = searchServiceHelepr.constructFilterBuilder(paramMap,null);
List<String> firstProductSkns =recallServiceHelper.getFirstProductSkns(paramMap);
//3、构造请求参数
List<RecallRequest> otherRecallRequests = otherRecallRequest.buildOtherRecallRequests(query,filter,firstProductSkns,pageSize);
PersionalFactor persionalFactor = recallPersionalService.getPersionalUseFactor(query,filter,1,null);
List<RecallRequest> brandRecallRequests = brandRecallRequest.buildBrandRecallRequests(query,filter,persionalFactor.getBrandIds());
//4、合并召回结果
RecallResponse results = this.batchRecall(otherRecallRequests,brandRecallRequests);
//5、构造返回结果
//2、构造召回相关参数
RecallParams recallParams = this.buildRecallParams(paramMap);
//3、执行召回
RecallResponse response = this.doBatchRecall(recallParams);
//4、构造返回结果
JSONObject dataMap = new JSONObject();
dataMap.put("total", results.getTotal());
dataMap.put("total", response.getTotal());
dataMap.put("page", page);
dataMap.put("page_size", pageSize);
dataMap.put("page_total", searchCommonHelper.getTotalPage(results.getTotal(), pageSize));
dataMap.put("product_list", results.getRecallSkns());
dataMap.put("page_total", searchCommonHelper.getTotalPage(response.getTotal(), pageSize));
dataMap.put("product_list", response.getRecallSkns());
dataMap.put("product_list_size", response.getRecallSkns().size());
return new SearchApiResult().setData(dataMap);
} catch (Exception e) {
e.printStackTrace();
... ... @@ -75,27 +65,78 @@ public class SceneRecallService {
}
}
private RecallResponse batchRecall(List<RecallRequest> otherRecallRequests, List<RecallRequest> brandRecallRequests){
List<RecallRequest> allRequests = new ArrayList<>(otherRecallRequests.size() + brandRecallRequests.size());
allRequests.addAll(otherRecallRequests);
allRequests.addAll(brandRecallRequests);
Map<String, RecallResponse> map = recallCommonService.batchRecall(allRequests);
private RecallParams buildRecallParams(Map<String, String> paramMap) throws Exception {
QueryBuilder query = searchServiceHelepr.constructQueryBuilder(paramMap);
BoolQueryBuilder filter = searchServiceHelepr.constructFilterBuilder(paramMap, null);
List<String> firstProductSkns = recallServiceHelper.getFirstProductSkns(paramMap);
int pageSize = MapUtils.getIntValue(paramMap, "viewNum", 10);
int uid = MapUtils.getIntValue(paramMap, "uid", 1);
String udid = MapUtils.getString(paramMap, "udid", "");
return new RecallParams(query,filter,firstProductSkns,pageSize,uid,udid);
}
private RecallResponse doBatchRecall(RecallParams param){
//1、构造召回请求
List<RecallRequest> allRequests = new ArrayList<>();
//2、构造非个性化的请求
List<RecallRequest> commonRequests = commonRecallRequestBuilder.buildCommonRecallRequests(param.getQuery(), param.getFilter(), param.getFirstProductSkns(), param.getPageSize());
allRequests.addAll(commonRequests);
//3、获取个性化因子
PersionalFactor persionalFactor = recallPersionalService.queryPersionalFactor(param.getQuery(), param.getFilter(), param.getUid(), param.getUdid());
//4、构建个性化品牌的召回请求
List<RecallRequest> brandRequests = brandRecallRequestBuilder.buildBrandRecallRequests(param.getQuery(), param.getFilter(), persionalFactor.getBrandIds());
allRequests.addAll(brandRequests);
//5、批量召回
List<RecallRequestResponse> requestResponses = recallCommonService.batchRecallAndCache(allRequests);
//6、从兜底类型中获取总数
long total = this.getTotalCount(requestResponses);
//7、获取召回的skn
List<RecallResponse.RecallSkn> recallSkns = this.distinctRecallSkn(requestResponses);
//8、构造返回结果
return new RecallResponse(RecallRequestTypeEnum.BATCH_RESULTS,total,recallSkns);
}
/**
* 从兜底类型中获取总数
* @param requestResponses
* @return
*/
private long getTotalCount(List<RecallRequestResponse> requestResponses) {
long total = 0;
for (RecallRequestResponse requestResponse : requestResponses) {
RecallResponse response = requestResponse.getResponse();
if(response==null){
continue;
}
if (RecallRequestTypeEnum.COMMON.name().equalsIgnoreCase(response.getRequestType())){
total = response.getTotal();
break;
}
}
return total;
}
/**
* 召回结果去重
* @param requestResponses
* @return
*/
private List<RecallResponse.RecallSkn> distinctRecallSkn(List<RecallRequestResponse> requestResponses) {
List<RecallResponse.RecallSkn> sknResults = new ArrayList<>();
long total = 0 ;
Set<Integer> existProductSkns = new HashSet<Integer>();
for (RecallResponse response : map.values()) {
if(total ==0 && response.getRequestType().equalsIgnoreCase(RecallRequestTypeEnum.COMMON.name())){
total = response.getTotal();
for (RecallRequestResponse requestResponse : requestResponses) {
RecallResponse response = requestResponse.getResponse();
if(response==null){
continue;
}
for (RecallResponse.RecallSkn recallSkn: response.getRecallSkns()) {
if(!existProductSkns.contains(recallSkn.getProductSkn())){
for (RecallResponse.RecallSkn recallSkn : response.getRecallSkns()) {
if (!existProductSkns.contains(recallSkn.getProductSkn())) {
sknResults.add(recallSkn);
existProductSkns.add(recallSkn.getProductSkn());
}
}
}
return new RecallResponse(RecallRequestTypeEnum.TOTAL_BATCH_RESULTS,total,sknResults);
return sknResults;
}
}
... ...
package com.yoho.search.recall.scene.helper;
import com.alibaba.fastjson.JSON;
import com.yoho.core.redis.cluster.operations.serializer.RedisKeyBuilder;
import com.yoho.search.base.utils.CollectionUtils;
import com.yoho.search.base.utils.ISearchConstants;
import com.yoho.search.base.utils.ProductIndexEsField;
import com.yoho.search.base.utils.Transfer;
import com.yoho.search.common.cache.impls.SearchRedis;
import com.yoho.search.core.es.model.SearchParam;
import com.yoho.search.core.es.model.SearchResult;
import com.yoho.search.recall.scene.models.RecallRequest;
import com.yoho.search.recall.scene.models.RecallRequestResponse;
import com.yoho.search.recall.scene.models.RecallResponse;
import com.yoho.search.recall.scene.persional.PersionalFactor;
import com.yoho.search.recall.scene.models.RecallResponseHelper;
import com.yoho.search.service.base.SearchCommonService;
import org.apache.commons.collections.MapUtils;
import org.elasticsearch.index.query.BoolQueryBuilder;
import org.elasticsearch.index.query.QueryBuilder;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Component;
import java.util.*;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
@Component
public class RecallCommonService {
private static final long cacheTimeInSecond = 10 * 60 ;//10分钟缓存
@Autowired
private SearchRedis searchRedis;
@Autowired
private SearchCommonService searchCommonService;
private static final int maxRequestCountPreTime = 10;
/**
* 批量召回入口
*
* @param requests
* @return
*/
public Map<String, RecallResponse> batchRecall(List<RecallRequest> requests) {
//1、从缓存中获取
int totalCount = requests.size();
Map<String, RecallResponse> cachedResults = this.mutiGetFromCache(requests);
//2、过滤缓存命中的请求
Iterator<RecallRequest> requestIterator = requests.iterator();
while (requestIterator.hasNext()) {
RecallRequest request = requestIterator.next();
if (cachedResults.containsKey(request.redisKeyBuilder().getKey())){
requestIterator.remove();
}
}
int requestCount = requests.size();
//3、判断是否全部命中了缓存
if (requests.isEmpty()) {
return cachedResults;
public List<RecallRequestResponse> batchRecallAndCache(final List<RecallRequest> requests) {
//1、先从缓存中获取数据，并且构建返回结果对象
final List<RecallRequestResponse> results = this.queryResultFromCache(requests);
//2、构造未命中缓存的请求-最多透传10个
final List<RecallRequest> notCachedRequests = this.buildNotCachedRequests(results,maxRequestCountPreTime);
//3、如果remainRequests为空，则说明全部命中了缓存，直接返回即可
if (notCachedRequests.isEmpty()) {
return results;
}
System.out.println("totalCount:"+totalCount+"_requestCount:"+requestCount);
//4、执行查询并加入缓存
this.queryAndAddToCache(requests,cachedResults);
//5、返回结果
return cachedResults;
System.out.println("totalRequestSize:" + requests.size() + "notCachedRequestSize:" + notCachedRequests.size());
//4、处理剩余请求
List<RecallRequestResponse> notCachedResults = this.queryNotCachedResult(notCachedRequests);
//5、将查出来对象加入缓存
this.addResultsToCache(notCachedResults);
//6、填充results
this.fillResults(results,notCachedResults);
return results;
}
/**
* 从缓存中批量获取
*
* @param requests
* @return
*/
private Map<String, RecallResponse> mutiGetFromCache(List<RecallRequest> requests) {
private List<RecallRequestResponse> queryResultFromCache(final List<RecallRequest> requests) {
List<RecallRequestResponse> results = new ArrayList<>(requests.size());
List<RedisKeyBuilder> redisKeyBuilders = new ArrayList<>();
for (RecallRequest req : requests) {
redisKeyBuilders.add(req.redisKeyBuilder());
}
Map<String, RecallResponse> results = new HashMap<>(requests.size());
List<String> cacheValues = searchRedis.searchValueOperations.multiGet(redisKeyBuilders);
for (int i = 0; i < redisKeyBuilders.size(); i++) {
for (int i = 0; i < requests.size(); i++) {
RecallRequest request = requests.get(i);
String cacheValue = cacheValues.get(i);
if (cacheValue != null) {
results.put(redisKeyBuilders.get(i).getKey(), this.fromString(cacheValue));
}
RecallResponse response = RecallResponseHelper.buildResonse(cacheValue);
results.add(new RecallRequestResponse(request, response));
}
return results;
}
/**
* 查询未命中缓存的请求，并加入缓存
* @param requests
* 获取未命中缓存的请求
* @param requestResponseResults
* @return
*/
private List<RecallRequest> buildNotCachedRequests(List<RecallRequestResponse> requestResponseResults,int maxCount){
List<RecallRequest> remainRequests = new ArrayList<>();
for (RecallRequestResponse requestResponse : requestResponseResults) {
if (requestResponse.getResponse() == null) {
remainRequests.add(requestResponse.getRequest());
}
if(remainRequests.size()>=maxCount){
break;
}
}
return remainRequests;
}
/**
* 查询命中缓存的请求
* @param notCachedRequests
* @return
*/
private void queryAndAddToCache(List<RecallRequest> requests, Map<String, RecallResponse> cachedResults) {
private List<RecallRequestResponse> queryNotCachedResult(List<RecallRequest> notCachedRequests) {
//1、构造请求参数
List<SearchParam> searchParams = new ArrayList<>();
for (RecallRequest request : requests) {
for (RecallRequest request : notCachedRequests) {
searchParams.add(request.searchParam());
}
//2、执行搜索
List<SearchResult> searchResults = searchCommonService.doMutiSearch(ISearchConstants.INDEX_NAME_PRODUCT_INDEX, searchParams);
//3、构造结果和缓存结果
Map<RedisKeyBuilder, String> toCacheResults = new HashMap<RedisKeyBuilder, String>();
for (int i = 0; i < requests.size(); i++) {
RecallRequest request = requests.get(i);
//3、构造返回结果
List<RecallRequestResponse> results = new ArrayList<>();
for (int i = 0; i < notCachedRequests.size(); i++) {
RecallRequest request = notCachedRequests.get(i);
SearchResult searchResult = searchResults.get(i);
RecallResponse response = this.genRecallRspFromSearchResult(request,searchResult);
cachedResults.put(request.redisKeyBuilder().getKey(), response);
toCacheResults.put(request.redisKeyBuilder(),this.recallRspToString(response));
RecallResponse response = RecallResponseHelper.buildResonse(request, searchResult);
results.add(new RecallRequestResponse(request,response));
}
//4、加入缓存
searchRedis.searchRedisTemplate.mset(toCacheResults,cacheTimeInSecond);
return results;
}
private String recallRspToString(RecallResponse response) {
return JSON.toJSONString(response);
/**
* 将请求结果加入缓存
* @param notCachedResults
*/
private void addResultsToCache(List<RecallRequestResponse> notCachedResults){
//1、按缓存时间分组
Map<Integer,List<RecallRequestResponse>> groupMap = CollectionUtils.toListMap(notCachedResults, new Transfer<RecallRequestResponse, Integer>() {
@Override
public Integer transfer(RecallRequestResponse recallRequestResponse) {
return recallRequestResponse.getRequest().getCacheTimeInSecond();
}
});
//2、按缓存时间大小直接加入缓存
for (Map.Entry<Integer,List<RecallRequestResponse>> entry: groupMap.entrySet()) {
this.addRequestResponseToCache(entry.getValue(),entry.getKey());
}
}
private RecallResponse fromString(String value) {
return JSON.parseObject(value, RecallResponse.class);
private void addRequestResponseToCache(List<RecallRequestResponse> requestResponseList,int cacheTimeInSecond){
//1、构造缓存结果
Map<RedisKeyBuilder, String> toCacheResults = new HashMap<>();
for (RecallRequestResponse requestResponse :requestResponseList) {
RecallRequest request = requestResponse.getRequest();
RecallResponse response = requestResponse.getResponse();
toCacheResults.put(request.redisKeyBuilder(), RecallResponseHelper.serializerToString(response));
}
//2、加入缓存
searchRedis.searchRedisTemplate.mset(toCacheResults, cacheTimeInSecond);
}
private RecallResponse genRecallRspFromSearchResult(RecallRequest request, SearchResult searchResult) {
List<Map<String, Object>> results = searchResult.getResultList();
List<RecallResponse.RecallSkn> recallSkns = new ArrayList<>();
for (Map<String, Object> result : results) {
String reqType = request.getRequestType();
Integer productSkn = MapUtils.getInteger(result, ProductIndexEsField.productSkn, 0);
Integer brandId = MapUtils.getInteger(result, ProductIndexEsField.brandId, 0);
Integer middleSortId = MapUtils.getInteger(result, ProductIndexEsField.middleSortId, 0);
recallSkns.add(new RecallResponse.RecallSkn(reqType,productSkn,brandId,middleSortId));
/**
* 填充返回结果
* @param results
* @param remainRequestResults
*/
private void fillResults(List<RecallRequestResponse> results,List<RecallRequestResponse> remainRequestResults){
Map<String,RecallRequestResponse> remainRequestResponseMap = CollectionUtils.toMap(remainRequestResults, new Transfer<RecallRequestResponse,String>(){
@Override
public String transfer(RecallRequestResponse requestResponse) {
return requestResponse.getRequest().redisKeyBuilder().getKey();
}
});
for (RecallRequestResponse result: results) {
if(result.getResponse()!=null){
continue;
}
RecallRequestResponse requestResponse = remainRequestResponseMap.get(result.getRequest().redisKeyBuilder().getKey());
if(requestResponse==null){
continue;
}
result.setResponse(requestResponse.getResponse());
}
return new RecallResponse(request.getRequestType(),searchResult.getTotal(),recallSkns);
}
}
... ...
package com.yoho.search.recall.scene.models;
import org.elasticsearch.index.query.BoolQueryBuilder;
import org.elasticsearch.index.query.QueryBuilder;
import java.util.List;
public class RecallParams {
private QueryBuilder query;
private BoolQueryBuilder filter;
private List<String> firstProductSkns;
private int pageSize;
private int uid;
private String udid;
public RecallParams(QueryBuilder query, BoolQueryBuilder filter, List<String> firstProductSkns, int pageSize, int uid, String udid) {
this.query = query;
this.filter = filter;
this.firstProductSkns = firstProductSkns;
this.pageSize = pageSize;
this.uid = uid;
this.udid = udid;
}
public QueryBuilder getQuery() {
return query;
}
public BoolQueryBuilder getFilter() {
return filter;
}
public List<String> getFirstProductSkns() {
return firstProductSkns;
}
public int getPageSize() {
return pageSize;
}
public int getUid() {
return uid;
}
public String getUdid() {
return udid;
}
}
... ...
... ... @@ -25,8 +25,10 @@ public class RecallRequest implements IRecallRequest{
private Integer size;
private String requestType;
private RedisKeyBuilder redisKeyBuilder;
private int cacheTimeInSecond;
public RecallRequest(QueryBuilder queryInParam, QueryBuilder filterInParam, QueryBuilder extendFilter, SortBuilder<?> sortBuilder, Integer size, RecallRequestTypeEnum requestTypeEnum) {
public RecallRequest(QueryBuilder queryInParam, QueryBuilder filterInParam, QueryBuilder extendFilter, SortBuilder<?> sortBuilder, Integer size, RecallRequestTypeEnum requestTypeEnum,int cacheTimeInSecond) {
this.filterInParam = filterInParam;
this.queryInParam = queryInParam;
this.extendFilter = extendFilter;
... ... @@ -34,6 +36,7 @@ public class RecallRequest implements IRecallRequest{
this.size = size;
this.requestType = requestTypeEnum.name();
this.redisKeyBuilder = genRedisKeyBuilder();
this.cacheTimeInSecond = cacheTimeInSecond;
}
private RedisKeyBuilder genRedisKeyBuilder(){
... ... @@ -78,4 +81,8 @@ public class RecallRequest implements IRecallRequest{
public String getRequestType() {
return this.requestType;
}
public int getCacheTimeInSecond() {
return cacheTimeInSecond;
}
}
... ...
package com.yoho.search.recall.scene.models;
public class RecallRequestResponse {
private RecallRequest request;
private RecallResponse response;
public RecallRequestResponse(RecallRequest request, RecallResponse response) {
this.request = request;
this.response = response;
}
public RecallRequest getRequest() {
return request;
}
public void setResponse(RecallResponse response) {
this.response = response;
}
public RecallResponse getResponse() {
return response;
}
}
... ...
... ... @@ -2,7 +2,7 @@ package com.yoho.search.recall.scene.models;
public enum RecallRequestTypeEnum {
TOTAL_BATCH_RESULTS,
BATCH_RESULTS,
COMMON,
FIRST_PRODUCT_SKN,
... ...
package com.yoho.search.recall.scene.models;
import com.alibaba.fastjson.JSON;
import com.yoho.search.base.utils.ProductIndexEsField;
import com.yoho.search.core.es.model.SearchResult;
import org.apache.commons.collections.MapUtils;
import org.apache.commons.lang.StringUtils;
import java.util.ArrayList;
import java.util.List;
import java.util.Map;
public class RecallResponseHelper {
public static String serializerToString(RecallResponse response) {
return JSON.toJSONString(response);
}
public static RecallResponse buildResonse(String value) {
if(StringUtils.isBlank(value)){
return null;
}
try {
return JSON.parseObject(value, RecallResponse.class);
} catch (Exception e) {
return null;
}
}
public static RecallResponse buildResonse(RecallRequest request, SearchResult searchResult) {
List<Map<String, Object>> results = searchResult.getResultList();
List<RecallResponse.RecallSkn> recallSkns = new ArrayList<>();
for (Map<String, Object> result : results) {
String reqType = request.getRequestType();
Integer productSkn = MapUtils.getInteger(result, ProductIndexEsField.productSkn, 0);
Integer brandId = MapUtils.getInteger(result, ProductIndexEsField.brandId, 0);
Integer middleSortId = MapUtils.getInteger(result, ProductIndexEsField.middleSortId, 0);
recallSkns.add(new RecallResponse.RecallSkn(reqType, productSkn, brandId, middleSortId));
}
return new RecallResponse(request.getRequestType(), searchResult.getTotal(), recallSkns);
}
}
... ...
... ... @@ -8,12 +8,12 @@ public class PersionalFactor implements Serializable{
private static final long serialVersionUID = 89030356435559223L;
private List<Integer> brandIds;
private List<PersionalSortPriceArea> sortPriceAreas;
private List<SortPriceArea> sortPriceAreas;
public PersionalFactor() {
}
public PersionalFactor(List<Integer> brandIds, List<PersionalSortPriceArea> sortPriceAreas) {
public PersionalFactor(List<Integer> brandIds, List<SortPriceArea> sortPriceAreas) {
this.brandIds = brandIds;
this.sortPriceAreas = sortPriceAreas;
}
... ... @@ -22,7 +22,7 @@ public class PersionalFactor implements Serializable{
this.brandIds = brandIds;
}
public void setSortPriceAreas(List<PersionalSortPriceArea> sortPriceAreas) {
public void setSortPriceAreas(List<SortPriceArea> sortPriceAreas) {
this.sortPriceAreas = sortPriceAreas;
}
... ... @@ -30,8 +30,47 @@ public class PersionalFactor implements Serializable{
return brandIds;
}
public List<PersionalSortPriceArea> getSortPriceAreas() {
public List<SortPriceArea> getSortPriceAreas() {
return sortPriceAreas;
}
public static class SortPriceArea implements Serializable{
private static final long serialVersionUID = -7973209074997705083L;
private Integer middleSortId;
private Integer priceArea;
public SortPriceArea() {
}
public SortPriceArea(Integer middleSortId, Integer priceArea) {
this.middleSortId = middleSortId;
this.priceArea = priceArea;
}
public Integer getMiddleSortId() {
return middleSortId;
}
public void setMiddleSortId(Integer middleSortId) {
this.middleSortId = middleSortId;
}
public Integer getPriceArea() {
return priceArea;
}
public void setPriceArea(Integer priceArea) {
this.priceArea = priceArea;
}
@Override
public boolean equals(Object o) {
SortPriceArea area = (SortPriceArea)o;
return this.middleSortId.equals(area.getMiddleSortId()) && this.priceArea.equals(area.getPriceArea());
}
}
}
... ...
package com.yoho.search.recall.scene.persional;
import com.yoho.search.base.utils.ISearchConstants;
import com.yoho.search.base.utils.ProductIndexEsField;
import com.yoho.search.common.cache.SearchCacheFactory;
import com.yoho.search.core.es.model.SearchParam;
import com.yoho.search.core.es.model.SearchResult;
import com.yoho.search.service.base.SearchCacheService;
import com.yoho.search.service.base.SearchCommonService;
import org.elasticsearch.index.query.BoolQueryBuilder;
import org.elasticsearch.index.query.QueryBuilder;
import org.elasticsearch.search.aggregations.AbstractAggregationBuilder;
import org.elasticsearch.search.aggregations.Aggregation;
import org.elasticsearch.search.aggregations.AggregationBuilders;
import org.elasticsearch.search.aggregations.bucket.MultiBucketsAggregation;
import org.elasticsearch.search.aggregations.bucket.terms.TermsAggregationBuilder;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Component;
import java.util.ArrayList;
import java.util.Iterator;
import java.util.List;
import java.util.Map;
@Component
class PersionalFactorPageComponent {
@Autowired
private SearchCommonService searchCommonService;
@Autowired
private SearchCacheService searchCacheService;
@Autowired
private SearchCacheFactory searchCacheFactory;
/**
* 获取链接中的个性化因子
*
* @param query
* @param filter
* @return
*/
public PersionalFactor queryPagePersionalFactor(QueryBuilder query, BoolQueryBuilder filter) {
//0、构造参数
SearchParam searchParam = new SearchParam();
searchParam.setQuery(query);
searchParam.setFiter(filter);
searchParam.setSize(0);
List<AbstractAggregationBuilder<?>> aggregationBuilders = new ArrayList<>();
//1、品牌id聚合
aggregationBuilders.add(AggregationBuilders.terms("brandIdAgg").field(ProductIndexEsField.brandId).size(200));//品牌id聚合
//2、品类价格带聚合
TermsAggregationBuilder middleAggBuilder = AggregationBuilders.terms("middleSortIdAgg").field(ProductIndexEsField.middleSortId).size(100);
TermsAggregationBuilder genderAggBuilder = AggregationBuilders.terms("genderAgg").field(ProductIndexEsField.gender).size(5);
middleAggBuilder.subAggregation(genderAggBuilder);
aggregationBuilders.add(middleAggBuilder);//品类性格聚合
searchParam.setAggregationBuilders(aggregationBuilders);
//3、缓存中获取
String cacheKey = searchCacheService.genSearchParamString(ISearchConstants.INDEX_NAME_PRODUCT_INDEX,searchParam);
PersionalFactor pagePersionalFactor = searchCacheService.getSerializableObjectFromCache(searchCacheFactory.getAggregationSearchCache(),cacheKey,PersionalFactor.class,true);
if(pagePersionalFactor!=null) {
return pagePersionalFactor;
}
//4、执行查询
SearchResult searchResult = searchCommonService.doSearch(ISearchConstants.INDEX_NAME_PRODUCT_INDEX, searchParam);
//5、构造结果
Map<String, Aggregation> aggregationMap = searchResult.getAggMaps();
List<Integer> brandIds = this.getBrandIdsFromAggregationMap(aggregationMap);
List<PersionalFactor.SortPriceArea> sortPriceAreas = this.getSortPriceAreasFromAggregationMap(aggregationMap);
pagePersionalFactor = new PersionalFactor(brandIds, sortPriceAreas);
//6、加入缓存
searchCacheService.addSerializableObjectToCache(searchCacheFactory.getAggregationSearchCache(),cacheKey,pagePersionalFactor,true);
return pagePersionalFactor;
}
private List<Integer> getBrandIdsFromAggregationMap(Map<String, Aggregation> aggregationMap) {
List<Integer> brandIds = new ArrayList<Integer>();
MultiBucketsAggregation aggregation = (MultiBucketsAggregation) aggregationMap.get("brandIdAgg");
if (aggregation == null) {
return brandIds;
}
Iterator<? extends MultiBucketsAggregation.Bucket> iteratorterator = aggregation.getBuckets().iterator();
while (iteratorterator.hasNext()) {
MultiBucketsAggregation.Bucket bucket = iteratorterator.next();
Integer brandId = Integer.valueOf(bucket.getKeyAsString());
brandIds.add(brandId);
}
return brandIds;
}
private List<PersionalFactor.SortPriceArea> getSortPriceAreasFromAggregationMap(Map<String, Aggregation> aggregationMap) {
List<PersionalFactor.SortPriceArea> sortPrices = new ArrayList<>();
MultiBucketsAggregation aggregation = (MultiBucketsAggregation) aggregationMap.get("middleSortIdAgg");
if (aggregation == null) {
return new ArrayList<>();
}
Iterator<? extends MultiBucketsAggregation.Bucket> iterator = aggregation.getBuckets().iterator();
while (iterator.hasNext()) {
MultiBucketsAggregation.Bucket middleSortIdBucket = iterator.next();
Integer middleSortId = Integer.valueOf(middleSortIdBucket.getKeyAsString());
MultiBucketsAggregation genderAggregation = ((MultiBucketsAggregation) middleSortIdBucket.getAggregations().asMap().get("genderAgg"));
if(genderAggregation==null){
continue;
}
Iterator<? extends MultiBucketsAggregation.Bucket> genderIterator = genderAggregation.getBuckets().iterator();
while (genderIterator.hasNext()) {
MultiBucketsAggregation.Bucket genderBucket = genderIterator.next();
Integer gender = Integer.valueOf(genderBucket.getKeyAsString());
sortPrices.add(new PersionalFactor.SortPriceArea(middleSortId, gender));
}
}
return sortPrices;
}
}
... ...
package com.yoho.search.recall.scene.persional;
import org.springframework.stereotype.Component;
import java.util.ArrayList;
import java.util.List;
@Component
class PersionalFactorUserComponent {
/**
* 获取用户的个性化因子
*
* @param uid
* @param udid
* @return
*/
public PersionalFactor queryUserPersionalFactor(int uid, String udid) {
List<Integer> brandIds = new ArrayList<Integer>();
for (int i = 0; i < 100; i++) {
brandIds.add((int) (Math.random() * 3000));
}
List<PersionalFactor.SortPriceArea> sortPriceArea = new ArrayList<PersionalFactor.SortPriceArea>();
for (int i = 0; i < 50; i++) {
sortPriceArea.add(new PersionalFactor.SortPriceArea((int) (Math.random() * 1000), (int) (Math.random() * 3)));
}
return new PersionalFactor(brandIds, sortPriceArea);
}
}
... ...
package com.yoho.search.recall.scene.persional;
import java.io.Serializable;
public class PersionalSortPriceArea implements Serializable{
private static final long serialVersionUID = -7973209074997705083L;
private Integer middleSortId;
private Integer priceArea;
public PersionalSortPriceArea() {
}
public PersionalSortPriceArea(Integer middleSortId, Integer priceArea) {
this.middleSortId = middleSortId;
this.priceArea = priceArea;
}
public Integer getMiddleSortId() {
return middleSortId;
}
public void setMiddleSortId(Integer middleSortId) {
this.middleSortId = middleSortId;
}
public Integer getPriceArea() {
return priceArea;
}
public void setPriceArea(Integer priceArea) {
this.priceArea = priceArea;
}
@Override
public boolean equals(Object o) {
PersionalSortPriceArea area = (PersionalSortPriceArea)o;
return this.middleSortId.equals(area.getMiddleSortId())
&& this.priceArea.equals(area.getPriceArea());
}
}
package com.yoho.search.recall.scene.persional;
import com.alibaba.fastjson.JSONObject;
import com.yoho.search.base.utils.ISearchConstants;
import com.yoho.search.base.utils.ProductIndexEsField;
import com.yoho.search.common.cache.SearchCacheFactory;
import com.yoho.search.common.cache.impls.SearchRedis;
import com.yoho.search.common.cache.model.CacheObject;
import com.yoho.search.common.cache.model.SearchCache;
import com.yoho.search.core.es.model.SearchParam;
import com.yoho.search.core.es.model.SearchResult;
import com.yoho.search.service.base.SearchCacheService;
import com.yoho.search.service.base.SearchCommonService;
import org.elasticsearch.index.query.BoolQueryBuilder;
import org.elasticsearch.index.query.QueryBuilder;
import org.elasticsearch.search.aggregations.AbstractAggregationBuilder;
import org.elasticsearch.search.aggregations.Aggregation;
import org.elasticsearch.search.aggregations.AggregationBuilders;
import org.elasticsearch.search.aggregations.bucket.MultiBucketsAggregation;
import org.elasticsearch.search.aggregations.bucket.terms.TermsAggregationBuilder;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Component;
import java.util.ArrayList;
import java.util.Iterator;
import java.util.List;
import java.util.Map;
@Component
public class RecallPersionalService {
private static final Logger logger = LoggerFactory.getLogger(RecallPersionalService.class);
private static final int RETURN_BRAND_COUNT = 10;
private static final int RETURN_SORT_PRICE_AREA_COUNT = 5;
private static final int RETURN_MAX_BRAND_COUNT = 10;
private static final int RETURN_MAX_SORT_PRICE_AREA_COUNT = 5;
@Autowired
private SearchCommonService searchCommonService;
private PersionalFactorPageComponent pageComponent;
@Autowired
private SearchCacheService searchCacheService;
@Autowired
private SearchCacheFactory searchCacheFactory;
private PersionalFactorUserComponent userComponent;
/**
* 获取个性化品牌id
* 获取个性化因子
*
* @param query
* @param filter
* @param udid
* @return
*/
public PersionalFactor getPersionalUseFactor(QueryBuilder query, BoolQueryBuilder filter, int uid, String udid) {
PersionalFactor pageFactor = this.getPageFactor(query, filter);
PersionalFactor userFactor = this.getUserFactor(uid, udid);
List<Integer> brandIds = this.joinBrandIds(pageFactor, userFactor, RETURN_BRAND_COUNT);
List<PersionalSortPriceArea> joinSortPrices = this.joinSortPriceAreas(pageFactor, userFactor, RETURN_SORT_PRICE_AREA_COUNT);
return new PersionalFactor(brandIds, joinSortPrices);
}
/**
* 获取链接中的个性化因子
* <p>
* //TODO 添加缓存
*
* @param query
* @param filter
* @return
*/
private PersionalFactor getPageFactor(QueryBuilder query, BoolQueryBuilder filter) {
//0、构造参数
SearchParam searchParam = new SearchParam();
searchParam.setQuery(query);
searchParam.setFiter(filter);
searchParam.setSize(0);
List<AbstractAggregationBuilder<?>> aggregationBuilders = new ArrayList<>();
//1、品牌id聚合
aggregationBuilders.add(AggregationBuilders.terms("brandIdAgg").field(ProductIndexEsField.brandId).size(200));//品牌id聚合
//2、品类价格带聚合
TermsAggregationBuilder middleAggBuilder = AggregationBuilders.terms("middleSortIdAgg").field(ProductIndexEsField.middleSortId).size(100);
TermsAggregationBuilder genderAggBuilder = AggregationBuilders.terms("genderAgg").field(ProductIndexEsField.gender).size(5);
middleAggBuilder.subAggregation(genderAggBuilder);
aggregationBuilders.add(middleAggBuilder);//品类性格聚合
searchParam.setAggregationBuilders(aggregationBuilders);
//3、缓存中获取
String cacheKey = searchCacheService.genSearchParamString(ISearchConstants.INDEX_NAME_PRODUCT_INDEX,searchParam);
PersionalFactor pagePersionalFactor = searchCacheService.getSerializableObjectFromCache(searchCacheFactory.getAggregationSearchCache(),cacheKey,PersionalFactor.class,true);
if(pagePersionalFactor!=null) {
return pagePersionalFactor;
}
//4、执行查询
SearchResult searchResultearchResult = searchCommonService.doSearch(ISearchConstants.INDEX_NAME_PRODUCT_INDEX, searchParam);
//5、构造结果
Map<String, Aggregation> aggregationMap = searchResultearchResult.getAggMaps();
List<Integer> brandIds = this.getBrandId(aggregationMap);
List<PersionalSortPriceArea> sortPriceAreas = this.getSortPriceAreas(aggregationMap);
pagePersionalFactor = new PersionalFactor(brandIds, sortPriceAreas);
//6、加入缓存
searchCacheService.addSerializableObjectToCache(searchCacheFactory.getAggregationSearchCache(),cacheKey,pagePersionalFactor,true);
return pagePersionalFactor;
}
/**
* 获取用户的个性化因子
*
* @param uid
* @param udid
* @return
*/
private PersionalFactor getUserFactor(int uid, String udid) {
List<Integer> brandIds = new ArrayList<Integer>();
for (int i = 0; i < 100; i++) {
brandIds.add((int) (Math.random() * 3000));
}
List<PersionalSortPriceArea> sortPriceArea = new ArrayList<PersionalSortPriceArea>();
for (int i = 0; i < 50; i++) {
sortPriceArea.add(new PersionalSortPriceArea((int) (Math.random() * 1000), (int) (Math.random() * 3)));
}
return new PersionalFactor(brandIds, sortPriceArea);
}
private List<Integer> getBrandId(Map<String, Aggregation> aggregationMap) {
List<Integer> brandIds = new ArrayList<Integer>();
MultiBucketsAggregation aggregation = (MultiBucketsAggregation) aggregationMap.get("brandIdAgg");
if (aggregation == null) {
return brandIds;
}
Iterator<? extends MultiBucketsAggregation.Bucket> iteratorterator = aggregation.getBuckets().iterator();
while (iteratorterator.hasNext()) {
MultiBucketsAggregation.Bucket bucket = iteratorterator.next();
Integer brandId = Integer.valueOf(bucket.getKeyAsString());
brandIds.add(brandId);
}
return brandIds;
}
private List<PersionalSortPriceArea> getSortPriceAreas(Map<String, Aggregation> aggregationMap) {
List<PersionalSortPriceArea> sortPrices = new ArrayList<>();
MultiBucketsAggregation aggregation = (MultiBucketsAggregation) aggregationMap.get("middleSortIdAgg");
if (aggregation == null) {
return new ArrayList<>();
}
Iterator<? extends MultiBucketsAggregation.Bucket> iterator = aggregation.getBuckets().iterator();
while (iterator.hasNext()) {
MultiBucketsAggregation.Bucket middleSortIdBucket = iterator.next();
Integer middleSortId = Integer.valueOf(middleSortIdBucket.getKeyAsString());
MultiBucketsAggregation genderAggregation = ((MultiBucketsAggregation) middleSortIdBucket.getAggregations().asMap().get("genderAgg"));
if(genderAggregation==null){
continue;
}
Iterator<? extends MultiBucketsAggregation.Bucket> genderIterator = genderAggregation.getBuckets().iterator();
while (genderIterator.hasNext()) {
MultiBucketsAggregation.Bucket genderBucket = genderIterator.next();
Integer gender = Integer.valueOf(genderBucket.getKeyAsString());
sortPrices.add(new PersionalSortPriceArea(middleSortId, gender));
}
}
return sortPrices;
}
private List<Integer> joinBrandIds(PersionalFactor pageFactor, PersionalFactor persionalFactor, int count) {
List<Integer> allBrandIds = pageFactor.getBrandIds();
List<Integer> userBrandIds = persionalFactor.getBrandIds();
List<Integer> results = new ArrayList<>();
for (Integer userBrandId : userBrandIds) {
if (allBrandIds.contains(userBrandId)) {
results.add(userBrandId);
}
if (results.size() >= count) {
break;
}
}
return results;
public PersionalFactor queryPersionalFactor(QueryBuilder query, BoolQueryBuilder filter, int uid, String udid) {
//1、获取页面上的个性化因子
PersionalFactor pageFactor = pageComponent.queryPagePersionalFactor(query, filter);
//2、获取用户的个性化因子
PersionalFactor userFactor = userComponent.queryUserPersionalFactor(uid, udid);
//3、join获取最终的结果
List<Integer> brandIds = this.innerJoin(pageFactor.getBrandIds(),userFactor.getBrandIds(),RETURN_MAX_BRAND_COUNT);
List<PersionalFactor.SortPriceArea> sortPriceAreas = this.innerJoin(pageFactor.getSortPriceAreas(),userFactor.getSortPriceAreas(),RETURN_MAX_SORT_PRICE_AREA_COUNT);
//logger.info("getPersionalUseFactor,brandIds is[{}],sortPriceAreas is [{}]",brandIds,sortPriceAreas);
return new PersionalFactor(brandIds, sortPriceAreas);
}
private List<PersionalSortPriceArea> joinSortPriceAreas(PersionalFactor pageFactor, PersionalFactor persionalFactor, int count) {
List<PersionalSortPriceArea> allSortPriceAreas = pageFactor.getSortPriceAreas();
List<PersionalSortPriceArea> userSortPriceAreas = persionalFactor.getSortPriceAreas();
List<PersionalSortPriceArea> results = new ArrayList<>();
for (PersionalSortPriceArea userSortPriceArea : userSortPriceAreas) {
if (allSortPriceAreas.contains(userSortPriceArea)) {
results.add(userSortPriceArea);
private <T> List<T> innerJoin(List<T> aList,List<T> bList,int size) {
List<T> results = new ArrayList<T>();
for (T a : aList) {
if (bList.contains(a)) {
results.add(a);
}
if (results.size() >= count) {
if (results.size() >= size) {
break;
}
}
... ...
... ... @@ -5,13 +5,16 @@ import com.yoho.search.recall.scene.models.RecallRequestTypeEnum;
import com.yoho.search.recall.sort.strategy.IRecallStrategy;
import org.elasticsearch.index.query.BoolQueryBuilder;
import org.elasticsearch.index.query.QueryBuilder;
import org.springframework.stereotype.Component;
@Component
class BaseRecallRequest {
public abstract class BaseRecallRequest {
protected RecallRequest buildRecallRequest(QueryBuilder query, BoolQueryBuilder filter, IRecallStrategy strategy, int size, RecallRequestTypeEnum recallRequestTypeEnum){
return new RecallRequest(query, filter, strategy.filter() , strategy.firstSortBuilder(), size, recallRequestTypeEnum);
protected static final int COMMON_CACHE_TIME_IN_SECOND = 5 * 60;//5分钟缓存
protected static final int BRAND_CACHE_TIME_IN_SECOND = 60 * 60;//60分钟缓存
protected RecallRequest buildRecallRequest(QueryBuilder query, BoolQueryBuilder filter, IRecallStrategy strategy, int size, RecallRequestTypeEnum requestType) {
return new RecallRequest(query, filter, strategy.filter(), strategy.firstSortBuilder(), size, requestType, this.cacheTimeInSecond());
}
protected abstract int cacheTimeInSecond();
}
... ...
... ... @@ -17,6 +17,11 @@ public class BrandRecallRequestBuilder extends BaseRecallRequest{
private static final int RECALL_BRAND_PER_BRAND = 5;
@Override
protected int cacheTimeInSecond() {
return BRAND_CACHE_TIME_IN_SECOND;
}
/**
* 按偏好品牌批量召回
* @param filter
... ... @@ -26,21 +31,41 @@ public class BrandRecallRequestBuilder extends BaseRecallRequest{
*/
public List<RecallRequest> buildBrandRecallRequests(QueryBuilder query, BoolQueryBuilder filter, List<Integer> brandIds) {
//1、构造召回请求
List<RecallRequest> requests = new ArrayList<RecallRequest>();
List<RecallRequest> requests = new ArrayList<>();
// 1) 人气
for (Integer brandId : brandIds) {
requests.add(this.buildBrandHeatValueRequest(query, filter, brandId, RECALL_BRAND_PER_BRAND));
}
// 2) 新品
for (Integer brandId : brandIds) {
// 1) 新品召回
requests.add(this.buildBrandNewShelveRequest(query, filter, brandId, RECALL_BRAND_PER_BRAND));
// 2) 新降价
}
// 3) 新降价
for (Integer brandId : brandIds) {
requests.add(this.buildBrandNewReducePriceRequest(query, filter, brandId, RECALL_BRAND_PER_BRAND));
// 3) 新开促销
}
// 4) 新开促销
for (Integer brandId : brandIds) {
requests.add(this.buildBrandNewPromotionRequest(query, filter, brandId, RECALL_BRAND_PER_BRAND));
// 4) 人气
requests.add(this.buildBrandHeatValueRequest(query, filter, brandId, RECALL_BRAND_PER_BRAND));
}
return requests;
}
/**
* 构造【按品牌id召回人气商品】的请求参数
*
* @param query
* @param filter
* @param brandId
* @param size
* @return
*/
private RecallRequest buildBrandHeatValueRequest(QueryBuilder query, BoolQueryBuilder filter, Integer brandId, int size) {
BrandHeatValueStrategy strage = new BrandHeatValueStrategy(Arrays.asList(brandId), size);
return this.buildRecallRequest(query, filter, strage, size, RecallRequestTypeEnum.BRAND_HEAT_VALUE);
}
/**
* 构造【按品牌id召回新品】的请求参数
*
* @param query
... ... @@ -82,18 +107,5 @@ public class BrandRecallRequestBuilder extends BaseRecallRequest{
return this.buildRecallRequest(query, filter, strage, size, RecallRequestTypeEnum.BRAND_PROMOTION);
}
/**
* 构造【按品牌id召回人气商品】的请求参数
*
* @param query
* @param filter
* @param brandId
* @param size
* @return
*/
private RecallRequest buildBrandHeatValueRequest(QueryBuilder query, BoolQueryBuilder filter, Integer brandId, int size) {
BrandHeatValueStrategy strage = new BrandHeatValueStrategy(Arrays.asList(brandId), size);
return this.buildRecallRequest(query, filter, strage, size, RecallRequestTypeEnum.BRAND_HEAT_VALUE);
}
}
... ...
... ... @@ -13,7 +13,12 @@ import java.util.ArrayList;
import java.util.List;
@Component
public class OtherRecallRequestBuilder extends BaseRecallRequest{
public class CommonRecallRequestBuilder extends BaseRecallRequest{
@Override
protected int cacheTimeInSecond() {
return COMMON_CACHE_TIME_IN_SECOND;
}
/**
* 批量召回业务需求
... ... @@ -21,7 +26,7 @@ public class OtherRecallRequestBuilder extends BaseRecallRequest{
* @param filter
* @return
*/
public List<RecallRequest> buildOtherRecallRequests(QueryBuilder query, BoolQueryBuilder filter, List<String> firstProductSkns, int pageSize){
public List<RecallRequest> buildCommonRecallRequests(QueryBuilder query, BoolQueryBuilder filter, List<String> firstProductSkns, int pageSize){
//1、构造召回请求
List<RecallRequest> requests = new ArrayList<RecallRequest>();
//1.1) 人气的召回
... ...