...
|
...
|
@@ -2,26 +2,19 @@ package com.yoho.search.recall.common; |
|
|
|
|
|
import com.alibaba.fastjson.JSONObject;
|
|
|
import com.yoho.search.base.utils.CollectionUtils;
|
|
|
import com.yoho.search.base.utils.ISearchConstants;
|
|
|
import com.yoho.search.base.utils.ProductIndexEsField;
|
|
|
import com.yoho.search.common.cache.CacheType;
|
|
|
import com.yoho.search.common.cache.aop.SearchCacheAble;
|
|
|
import com.yoho.search.core.es.model.SearchParam;
|
|
|
import com.yoho.search.core.es.model.SearchResult;
|
|
|
import com.yoho.search.core.personalized.PersonalizedSearch;
|
|
|
import com.yoho.search.models.SearchApiResult;
|
|
|
import com.yoho.search.recall.common.beans.ProductFeatureFactorHepler;
|
|
|
import com.yoho.search.recall.common.cacheable.CacheAbleServiceHelper;
|
|
|
import com.yoho.search.recall.common.cacheable.CommonPageRecallService;
|
|
|
import com.yoho.search.recall.common.model.CommonRecallParam;
|
|
|
import com.yoho.search.recall.common.model.CommonRecallResult;
|
|
|
import com.yoho.search.recall.common.model.CommonRecallSkn;
|
|
|
import com.yoho.search.recall.common.model.UserFeatureFactor;
|
|
|
import com.yoho.search.service.base.ProductListSortKey;
|
|
|
import com.yoho.search.service.base.ProductListSortService;
|
|
|
import com.yoho.search.service.base.SearchCommonService;
|
|
|
import com.yoho.search.service.base.SearchRequestParams;
|
|
|
import com.yoho.search.service.base.index.ProductIndexBaseService;
|
|
|
import com.yoho.search.service.helper.SearchParamHelper;
|
|
|
import com.yoho.search.service.helper.SearchSortHelper;
|
|
|
import com.yoho.search.service.scorer.personal.PersonalVectorFeatureSearch;
|
|
|
import org.apache.commons.collections.MapUtils;
|
|
|
import org.apache.commons.lang.StringUtils;
|
...
|
...
|
@@ -33,233 +26,165 @@ import org.elasticsearch.search.sort.SortOrder; |
|
|
import org.slf4j.Logger;
|
|
|
import org.slf4j.LoggerFactory;
|
|
|
import org.springframework.beans.factory.annotation.Autowired;
|
|
|
import org.springframework.stereotype.Service;
|
|
|
import org.springframework.stereotype.Component;
|
|
|
|
|
|
import java.util.*;
|
|
|
|
|
|
@Service
|
|
|
public class ProductListServiceHelper {
|
|
|
@Component
|
|
|
public class CommonRecallProductListService {
|
|
|
|
|
|
private static final Logger logger = LoggerFactory.getLogger(ProductListServiceHelper.class);
|
|
|
private static final Logger logger = LoggerFactory.getLogger(CommonRecallProductListService.class);
|
|
|
|
|
|
@Autowired
|
|
|
private CommonPageRecallService commonRecallSceneService;
|
|
|
@Autowired
|
|
|
private ProductListSortService productListSortService;
|
|
|
@Autowired
|
|
|
private SearchParamHelper searchParamHelper;
|
|
|
@Autowired
|
|
|
private SearchSortHelper searchSortHelper;
|
|
|
@Autowired
|
|
|
private ProductIndexBaseService productIndexBaseService;
|
|
|
@Autowired
|
|
|
private SearchCommonService searchCommonService;
|
|
|
@Autowired
|
|
|
private PersonalVectorFeatureSearch personalVectorFeatureSearch;
|
|
|
@Autowired
|
|
|
private ProductFeatureFactorHepler productFeatureFactorHepler;
|
|
|
@Autowired
|
|
|
private CacheAbleServiceHelper cacheAbleServiceHelper;
|
|
|
@Autowired
|
|
|
private CommonPageRecallService commonRecallSceneService;
|
|
|
@Autowired
|
|
|
private PersonalVectorFeatureSearch personalVectorFeatureSearch;
|
|
|
@Autowired
|
|
|
private ProductFeatureFactorHepler productFeatureFactorHepler;
|
|
|
@Autowired
|
|
|
private CacheAbleServiceHelper cacheAbleServiceHelper;
|
|
|
@Autowired
|
|
|
private ProductListSortService productListSortService;
|
|
|
|
|
|
/**
|
|
|
* 个性化的列表接口
|
|
|
*
|
|
|
* @order为空并且有uid
|
|
|
* @param paramMap
|
|
|
* @return
|
|
|
*/
|
|
|
public SearchApiResult productListForNewPersional(Map<String, String> paramMap) {
|
|
|
// 1、召回整数页个skn[需要new一个对象出来,不然原有数据的顺序会变]
|
|
|
CommonRecallResult cacheObject = commonRecallSceneService.doCommonPageRecallBatch(paramMap);
|
|
|
CommonRecallResult commonRecallResult = new CommonRecallResult(cacheObject);
|
|
|
// 2、获取分页参数
|
|
|
int page = StringUtils.isBlank(paramMap.get("page")) ? 1 : Integer.parseInt(paramMap.get("page"));
|
|
|
int viewNum = StringUtils.isBlank(paramMap.get("viewNum")) ? 10 : Integer.parseInt(paramMap.get("viewNum"));
|
|
|
long total = commonRecallResult.getTotalCount();
|
|
|
// 3、为用户做列表截取-保留第一页数据
|
|
|
int recallMaxPage = (int) (commonRecallResult.listCount() / viewNum);
|
|
|
if (recallMaxPage == 0) {
|
|
|
recallMaxPage = 1;
|
|
|
}
|
|
|
// 4、构造分页结果参数
|
|
|
JSONObject results = new JSONObject();
|
|
|
results.put("total", total);
|
|
|
results.put("page", page);
|
|
|
results.put("page_size", viewNum);
|
|
|
results.put("page_total", this.getTotalPage(total, viewNum));
|
|
|
// 5、构造productList
|
|
|
if (page <= recallMaxPage) {
|
|
|
results.put("product_list", this.queryProductListForPersional(commonRecallResult, paramMap, page, viewNum));
|
|
|
} else {
|
|
|
results.put("product_list", this.queryProductListForOtherPageIndex(paramMap, commonRecallResult, page - recallMaxPage, viewNum).getData());
|
|
|
}
|
|
|
return new SearchApiResult().setData(results);
|
|
|
}
|
|
|
/**
|
|
|
* 高性能的个性化的列表接口
|
|
|
*
|
|
|
* @order为空并且有uid
|
|
|
* @param paramMap
|
|
|
* @return
|
|
|
*/
|
|
|
public SearchApiResult productList(Map<String, String> paramMap) {
|
|
|
// 1、召回整数页个skn[需要new一个对象出来,不然原有数据的顺序会变]
|
|
|
CommonRecallResult cacheObject = commonRecallSceneService.doCommonPageRecallBatch(paramMap);
|
|
|
CommonRecallResult commonRecallResult = new CommonRecallResult(cacheObject);
|
|
|
// 2、获取分页参数
|
|
|
int page = StringUtils.isBlank(paramMap.get("page")) ? 1 : Integer.parseInt(paramMap.get("page"));
|
|
|
int viewNum = StringUtils.isBlank(paramMap.get("viewNum")) ? 10 : Integer.parseInt(paramMap.get("viewNum"));
|
|
|
long total = commonRecallResult.getTotalCount();
|
|
|
// 3、为用户做列表截取-保留第一页数据
|
|
|
int recallMaxPage = (int) (commonRecallResult.listCount() / viewNum);
|
|
|
if (recallMaxPage == 0) {
|
|
|
recallMaxPage = 1;
|
|
|
}
|
|
|
// 4、构造分页结果参数
|
|
|
JSONObject results = new JSONObject();
|
|
|
results.put("total", total);
|
|
|
results.put("page", page);
|
|
|
results.put("page_size", viewNum);
|
|
|
results.put("page_total", this.getTotalPage(total, viewNum));
|
|
|
// 5、构造productList
|
|
|
if (page <= recallMaxPage) {
|
|
|
results.put("product_list", this.queryProductListForPersional(commonRecallResult, paramMap, page, viewNum));
|
|
|
} else {
|
|
|
results.put("product_list", this.queryProductListForOtherPageIndex(paramMap, commonRecallResult, page - recallMaxPage, viewNum).getData());
|
|
|
}
|
|
|
return new SearchApiResult().setData(results);
|
|
|
}
|
|
|
|
|
|
/**
|
|
|
* 召回页的处理方式
|
|
|
*
|
|
|
* @param commonRecallResult
|
|
|
* @param page
|
|
|
* @param viewNum
|
|
|
* @return
|
|
|
*/
|
|
|
private List<? extends Map<?, ?>> queryProductListForPersional(CommonRecallResult commonRecallResult, Map<String, String> paramMap, int page, int viewNum) {
|
|
|
// 1、查询这整数页个skn的全部信息-包含了变价计划[在排序前查询才能命中缓存!~]
|
|
|
SearchApiResult productInfoMapResult = cacheAbleServiceHelper.queryProductInfoMap(commonRecallResult);
|
|
|
// 2、排序
|
|
|
commonRecallResult = this.callUserScoreAndRank(paramMap, commonRecallResult);
|
|
|
// 3、数量截取
|
|
|
JSONObject productInfoMap = (JSONObject) productInfoMapResult.getData();
|
|
|
List<JSONObject> results = new ArrayList<JSONObject>();
|
|
|
List<CommonRecallSkn> recallSknList = commonRecallResult.getRecallSknList();
|
|
|
int fromIndex = (page - 1) * viewNum;
|
|
|
int toIndex = page * viewNum;
|
|
|
if (toIndex > recallSknList.size()) {
|
|
|
toIndex = recallSknList.size();
|
|
|
}
|
|
|
recallSknList = CollectionUtils.safeSubList(recallSknList, fromIndex, toIndex);
|
|
|
for (CommonRecallSkn commonRecallSkn : recallSknList) {
|
|
|
results.add(productInfoMap.getJSONObject(String.valueOf(commonRecallSkn.getProductSkn())));
|
|
|
}
|
|
|
// 4、品牌打散
|
|
|
results = productListSortService.sortProductList(results, new ProductListSortKey<JSONObject>() {
|
|
|
@Override
|
|
|
public String getSortKey(JSONObject product) {
|
|
|
return MapUtils.getString(product, "brand_id", "0");
|
|
|
}
|
|
|
@Override
|
|
|
public int getMaxCount() {
|
|
|
return 2;
|
|
|
}
|
|
|
});
|
|
|
return results;
|
|
|
}
|
|
|
/**
|
|
|
* 召回页的处理方式
|
|
|
*
|
|
|
* @param commonRecallResult
|
|
|
* @param page
|
|
|
* @param viewNum
|
|
|
* @return
|
|
|
*/
|
|
|
private List<? extends Map<?, ?>> queryProductListForPersional(CommonRecallResult commonRecallResult, Map<String, String> paramMap, int page, int viewNum) {
|
|
|
// 1、查询这整数页个skn的全部信息-包含了变价计划[在排序前查询才能命中缓存!~]
|
|
|
SearchApiResult productInfoMapResult = cacheAbleServiceHelper.queryProductInfoMap(commonRecallResult);
|
|
|
// 2、排序
|
|
|
commonRecallResult = this.callUserScoreAndRank(paramMap, commonRecallResult);
|
|
|
// 3、数量截取
|
|
|
JSONObject productInfoMap = (JSONObject) productInfoMapResult.getData();
|
|
|
List<JSONObject> results = new ArrayList<JSONObject>();
|
|
|
List<CommonRecallSkn> recallSknList = commonRecallResult.getRecallSknList();
|
|
|
int fromIndex = (page - 1) * viewNum;
|
|
|
int toIndex = page * viewNum;
|
|
|
if (toIndex > recallSknList.size()) {
|
|
|
toIndex = recallSknList.size();
|
|
|
}
|
|
|
recallSknList = CollectionUtils.safeSubList(recallSknList, fromIndex, toIndex);
|
|
|
for (CommonRecallSkn commonRecallSkn : recallSknList) {
|
|
|
results.add(productInfoMap.getJSONObject(String.valueOf(commonRecallSkn.getProductSkn())));
|
|
|
}
|
|
|
// 4、品牌打散
|
|
|
results = productListSortService.sortProductList(results, new ProductListSortKey<JSONObject>() {
|
|
|
@Override
|
|
|
public String getSortKey(JSONObject product) {
|
|
|
return MapUtils.getString(product, "brand_id", "0");
|
|
|
}
|
|
|
@Override
|
|
|
public int getMaxCount() {
|
|
|
return 2;
|
|
|
}
|
|
|
});
|
|
|
return results;
|
|
|
}
|
|
|
|
|
|
/**
|
|
|
* 其他页码的处理方式
|
|
|
*
|
|
|
* @param paramMap
|
|
|
* @param commonRecallResult
|
|
|
* @param realPage
|
|
|
* @return
|
|
|
*/
|
|
|
private SearchApiResult queryProductListForOtherPageIndex(Map<String, String> paramMap, CommonRecallResult commonRecallResult, int realPage, int viewNum) {
|
|
|
try {
|
|
|
// 1、其他页码使用CommonRecallParamService去查询
|
|
|
CommonRecallParam commonRecallParam = new CommonRecallParam(paramMap, 0, realPage, viewNum);
|
|
|
// 2、前面几个已经召回的需要排除
|
|
|
BoolQueryBuilder extendMustFilter = QueryBuilders.boolQuery();
|
|
|
extendMustFilter.mustNot(QueryBuilders.termsQuery(ProductIndexEsField.productSkn, commonRecallResult.toSknList()));
|
|
|
commonRecallParam.setExtendMustFilter(extendMustFilter);
|
|
|
// 3、构造sort
|
|
|
List<SortBuilder<?>> sortBuilders = new ArrayList<SortBuilder<?>>();
|
|
|
sortBuilders.add(SortBuilders.fieldSort(ProductIndexEsField.sevendayMoney).order(SortOrder.DESC));
|
|
|
commonRecallParam.setSortBuilders(sortBuilders);
|
|
|
// 4、执行查询
|
|
|
return cacheAbleServiceHelper.queryProductListByRecallParam(commonRecallParam);
|
|
|
} catch (Exception e) {
|
|
|
logger.error(e.getMessage(), e);
|
|
|
return new SearchApiResult().setData(new ArrayList<>());
|
|
|
}
|
|
|
}
|
|
|
/**
|
|
|
* 其他页码的处理方式
|
|
|
*
|
|
|
* @param paramMap
|
|
|
* @param commonRecallResult
|
|
|
* @param realPage
|
|
|
* @return
|
|
|
*/
|
|
|
private SearchApiResult queryProductListForOtherPageIndex(Map<String, String> paramMap, CommonRecallResult commonRecallResult, int realPage, int viewNum) {
|
|
|
try {
|
|
|
// 1、其他页码使用CommonRecallParamService去查询
|
|
|
CommonRecallParam commonRecallParam = new CommonRecallParam(paramMap, 0, realPage, viewNum);
|
|
|
// 2、前面几个已经召回的需要排除
|
|
|
BoolQueryBuilder extendMustFilter = QueryBuilders.boolQuery();
|
|
|
extendMustFilter.mustNot(QueryBuilders.termsQuery(ProductIndexEsField.productSkn, commonRecallResult.toSknList()));
|
|
|
commonRecallParam.setExtendMustFilter(extendMustFilter);
|
|
|
// 3、构造sort
|
|
|
List<SortBuilder<?>> sortBuilders = new ArrayList<SortBuilder<?>>();
|
|
|
sortBuilders.add(SortBuilders.fieldSort(ProductIndexEsField.sevendayMoney).order(SortOrder.DESC));
|
|
|
commonRecallParam.setSortBuilders(sortBuilders);
|
|
|
// 4、执行查询
|
|
|
return cacheAbleServiceHelper.queryProductListByRecallParam(commonRecallParam);
|
|
|
} catch (Exception e) {
|
|
|
logger.error(e.getMessage(), e);
|
|
|
return new SearchApiResult().setData(new ArrayList<>());
|
|
|
}
|
|
|
}
|
|
|
|
|
|
private long getTotalPage(long total, int viewNum) {
|
|
|
if (viewNum == 0) {
|
|
|
return total;
|
|
|
}
|
|
|
long totalPage = total / viewNum;
|
|
|
if (total % viewNum > 0) {
|
|
|
totalPage = totalPage + 1;
|
|
|
}
|
|
|
return totalPage;
|
|
|
}
|
|
|
private long getTotalPage(long total, int viewNum) {
|
|
|
if (viewNum == 0) {
|
|
|
return total;
|
|
|
}
|
|
|
long totalPage = total / viewNum;
|
|
|
if (total % viewNum > 0) {
|
|
|
totalPage = totalPage + 1;
|
|
|
}
|
|
|
return totalPage;
|
|
|
}
|
|
|
|
|
|
/**
|
|
|
* 根据向量计算得分并排序
|
|
|
*
|
|
|
* @param paramMap
|
|
|
* @param commonRecallResult
|
|
|
* @return
|
|
|
*/
|
|
|
private CommonRecallResult callUserScoreAndRank(Map<String, String> paramMap, CommonRecallResult commonRecallResult) {
|
|
|
PersonalizedSearch personalizedSearch = personalVectorFeatureSearch.getPersonalizedSearch(paramMap);
|
|
|
UserFeatureFactor userFeatureFactor = new UserFeatureFactor(personalizedSearch);
|
|
|
List<String> firstProductSkns = Arrays.asList(MapUtils.getString(paramMap, SearchRequestParams.PARAM_SEARCH_FIRST_PRODUCRSKN, "").split(","));
|
|
|
for (CommonRecallSkn commonRecallSkn : commonRecallResult.getRecallSknList()) {
|
|
|
if (firstProductSkns.contains(String.valueOf(commonRecallSkn.getProductSkn()))) {
|
|
|
commonRecallSkn.setScore(10000d);// firstSkn排第一个
|
|
|
} else {
|
|
|
commonRecallSkn.setScore(productFeatureFactorHepler.calProductFeatureFactor(userFeatureFactor, commonRecallSkn.getProductFeatureFactor()));
|
|
|
}
|
|
|
}
|
|
|
Collections.sort(commonRecallResult.getRecallSknList(), new Comparator<CommonRecallSkn>() {
|
|
|
@Override
|
|
|
public int compare(CommonRecallSkn o1, CommonRecallSkn o2) {
|
|
|
return o2.getScore().compareTo(o1.getScore());// 大的排前面
|
|
|
}
|
|
|
});
|
|
|
return commonRecallResult;
|
|
|
}
|
|
|
|
|
|
/**
|
|
|
* 非个性化的列表接口-去除uid缓存
|
|
|
*
|
|
|
* @order不为空,或者无uid
|
|
|
* @param paramMap
|
|
|
* @return
|
|
|
*/
|
|
|
@SearchCacheAble(cacheName = "PRODUCT_LIST_NOT_PERSIONAL", cacheType = CacheType.SEARCH_REDIS, cacheInMinute = 10, excludeParams = { "uid", "firstProductSkn" })
|
|
|
public SearchApiResult productListNotPersional(Map<String, String> paramMap) {
|
|
|
return this.productList(paramMap);
|
|
|
}
|
|
|
|
|
|
/**
|
|
|
* 个性化的列表接口-基于向量的个性化缓存-估计命中率会很低
|
|
|
*
|
|
|
* @param paramMap
|
|
|
* @return
|
|
|
*/
|
|
|
@SearchCacheAble(cacheName = "PRODUCT_LIST_DEFAULT_PERSIONAL", cacheType = CacheType.SEARCH_REDIS, cacheInMinute = 10)
|
|
|
public SearchApiResult productListForDefaultPersional(Map<String, String> paramMap) {
|
|
|
return this.productList(paramMap);
|
|
|
}
|
|
|
|
|
|
private SearchApiResult productList(Map<String, String> paramMap){
|
|
|
try {
|
|
|
// 1)验证查询条数
|
|
|
int pageSize = StringUtils.isBlank(paramMap.get("viewNum")) ? 10 : Integer.parseInt(paramMap.get("viewNum"));
|
|
|
int page = StringUtils.isBlank(paramMap.get("page")) ? 1 : Integer.parseInt(paramMap.get("page"));
|
|
|
if (page < 1 || pageSize < 0 || page * pageSize > 1000000) {
|
|
|
return new SearchApiResult().setCode(400).setMessage("分页参数不合法");
|
|
|
}
|
|
|
if (pageSize > 100) {
|
|
|
pageSize = 100;
|
|
|
}
|
|
|
// 2)构建基本查询参数
|
|
|
SearchParam searchParam = searchParamHelper.buildWithPersional(paramMap, true);// 这个必须用buildWithPersional构造,有些特殊的逻辑-如firstSkn在里面
|
|
|
searchParam.setAggregationBuilders(null);
|
|
|
searchParam.setOffset((page - 1) * pageSize);
|
|
|
searchParam.setSize(pageSize);
|
|
|
// 3)设置排序字段
|
|
|
searchParam.setSortBuilders(searchSortHelper.buildSortList(paramMap));
|
|
|
// 4)设置返回的参数【节省带宽】
|
|
|
List<String> includeFields = productIndexBaseService.getProductIndexIncludeFields();
|
|
|
searchParam.setIncludeFields(includeFields);
|
|
|
SearchResult searchResult = searchCommonService.doSearch(ISearchConstants.INDEX_NAME_PRODUCT_INDEX, searchParam);
|
|
|
// 5)构造返回结果
|
|
|
JSONObject dataMap = new JSONObject();
|
|
|
dataMap.put("total", searchResult.getTotal());
|
|
|
dataMap.put("page", searchResult.getPage());
|
|
|
dataMap.put("page_size", searchParam.getSize());
|
|
|
dataMap.put("page_total", searchResult.getTotalPage());
|
|
|
List<Map<String, Object>> product_list = productIndexBaseService.getProductListWithPricePlan(searchResult.getResultList());
|
|
|
dataMap.put("product_list", productListSortService.sortProductList(product_list));// 处理一下商品的顺序;
|
|
|
return new SearchApiResult().setData(dataMap);
|
|
|
} catch (Exception e) {
|
|
|
logger.error(e.getMessage(), e);
|
|
|
return new SearchApiResult().setData(null).setCode(500);
|
|
|
}
|
|
|
}
|
|
|
/**
|
|
|
* 根据向量计算得分并排序
|
|
|
*
|
|
|
* @param paramMap
|
|
|
* @param commonRecallResult
|
|
|
* @return
|
|
|
*/
|
|
|
private CommonRecallResult callUserScoreAndRank(Map<String, String> paramMap, CommonRecallResult commonRecallResult) {
|
|
|
PersonalizedSearch personalizedSearch = personalVectorFeatureSearch.getPersonalizedSearch(paramMap);
|
|
|
UserFeatureFactor userFeatureFactor = new UserFeatureFactor(personalizedSearch);
|
|
|
List<String> firstProductSkns = Arrays.asList(MapUtils.getString(paramMap, SearchRequestParams.PARAM_SEARCH_FIRST_PRODUCRSKN, "").split(","));
|
|
|
for (CommonRecallSkn commonRecallSkn : commonRecallResult.getRecallSknList()) {
|
|
|
if (firstProductSkns.contains(String.valueOf(commonRecallSkn.getProductSkn()))) {
|
|
|
commonRecallSkn.setScore(10000d);// firstSkn排第一个
|
|
|
} else {
|
|
|
commonRecallSkn.setScore(productFeatureFactorHepler.calProductFeatureFactor(userFeatureFactor, commonRecallSkn.getProductFeatureFactor()));
|
|
|
}
|
|
|
}
|
|
|
Collections.sort(commonRecallResult.getRecallSknList(), new Comparator<CommonRecallSkn>() {
|
|
|
@Override
|
|
|
public int compare(CommonRecallSkn o1, CommonRecallSkn o2) {
|
|
|
return o2.getScore().compareTo(o1.getScore());// 大的排前面
|
|
|
}
|
|
|
});
|
|
|
return commonRecallResult;
|
|
|
}
|
|
|
|
|
|
} |
...
|
...
|
|