yoho-search / yoho-search-service · Commits

Go to a project

Authored by hugufei
Commit b0e6c885aac50947dc21fdc8ef3372c8ba072e0e 1 parent 151d17c2

CommonSceneService 切到CommonSceneProductListService

Inline Side-by-side
Showing 15 changed files with 655 additions and 205 deletions
package com.yoho.search.common.cache.aop;
public abstract class SearchCacheAbleParam {
public abstract String searchCacheKey();
}
... ...
... ... @@ -31,66 +31,6 @@ public class SearchCacheAspect {
@Autowired
private SearchCacheFactory searchCacheFactory;
// 获取请求的参数
private String getCacheKey(ProceedingJoinPoint pjp, SearchCacheAble searchCacheAble) {
Object[] arges = pjp.getArgs();
for (Object object : arges) {
if (object instanceof HttpServletRequest) {
return this.getCacheKey((HttpServletRequest) object, searchCacheAble);
}
if (object instanceof Map) {
return this.getCacheKey((Map<?, ?>) object, searchCacheAble);
}
}
return null;
}
// 获取请求的参数
private String getCacheKey(HttpServletRequest httpServletRequest, SearchCacheAble searchCacheAble) {
Map<String, String> paramMap = HttpServletRequestUtils.transParamType(httpServletRequest);
return this.getCacheKey(paramMap, searchCacheAble);
}
private String getCacheKey(Map<?, ?> paramMap, String cacheName, List<String> params, boolean inClude) {
StringBuilder paramStringBuilder = new StringBuilder();
for (Map.Entry<?, ?> entry : paramMap.entrySet()) {
String key = String.valueOf(entry.getKey());
if (inClude && !params.contains(key)) {
continue;
}
if (!inClude && params.contains(key)) {
continue;
}
Object value = entry.getValue();
if (value == null || StringUtils.isBlank(value.toString()) || value.toString().equals("null")) {
continue;
}
paramStringBuilder.append('&').append(key).append('=').append(value);
}
// System.err.println(paramStringBuilder);
String cacheKey = "YOHOSEARCH:" + cacheName + ":" + MD5Util.string2MD5(paramStringBuilder.toString());
return cacheKey;
}
// 获取请求的参数
private String getCacheKey(Map<?, ?> paramMap, SearchCacheAble searchCacheAble) {
List<String> includeParams = Arrays.asList(searchCacheAble.includeParams());
if (!includeParams.isEmpty()) {
return this.getCacheKey(paramMap, searchCacheAble.cacheName(), includeParams, true);
}
List<String> excludeParams = Arrays.asList(searchCacheAble.excludeParams());
return this.getCacheKey(paramMap, searchCacheAble.cacheName(), excludeParams, false);
}
public Class<? extends Serializable> getReturnClass(SearchCacheAble searchCacheAble, MethodSignature signature) {
final Class<?> returnType = signature.getMethod().getReturnType();
final Class<? extends Serializable> returnClass = searchCacheAble.returnClass();
if (!returnType.getName().equals(returnClass.getName())) {
return null;
}
return returnClass;
}
@Around("@annotation(com.yoho.search.common.cache.aop.SearchCacheAble)")
public Object doCache(ProceedingJoinPoint pjp) throws Throwable {
MethodSignature signature = (MethodSignature) pjp.getSignature();
... ... @@ -111,7 +51,6 @@ public class SearchCacheAspect {
}
SearchCache searchCache = searchCacheFactory.getAspectSearhCache(searchCacheAble);
Serializable cacheObject = searchCacheService.getSerializableObjectFromCache(searchCache, cacheKey, returnClass);
// System.err.println(cacheKey + "_" + JSON.toJSONString(cacheObject == null ? "" : cacheObject));
if (cacheObject != null) {
return cacheObject;
}
... ... @@ -127,4 +66,58 @@ public class SearchCacheAspect {
searchCacheService.addSerializableObjectToCache(searchCache, cacheKey, cacheObject);
return cacheObject;
}
// 校验返回类型是否一致
private Class<? extends Serializable> getReturnClass(SearchCacheAble searchCacheAble, MethodSignature signature) {
final Class<?> returnType = signature.getMethod().getReturnType();
final Class<? extends Serializable> returnClass = searchCacheAble.returnClass();
if (!returnType.getName().equals(returnClass.getName())) {
return null;
}
return returnClass;
}
// 获取请求的参数
private String getCacheKey(ProceedingJoinPoint pjp, SearchCacheAble searchCacheAble) {
Object[] arges = pjp.getArgs();
if (arges.length != 1) {
return null;
}
Object object = arges[0];
if (object instanceof SearchCacheAbleParam) {
return ((SearchCacheAbleParam) object).searchCacheKey();
}
if (object instanceof HttpServletRequest) {
return this.getCacheKey((HttpServletRequest) object, searchCacheAble);
}
if (object instanceof Map) {
return this.getCacheKey((Map<?, ?>) object, searchCacheAble);
}
return null;
}
// 获取请求的参数
private String getCacheKey(HttpServletRequest httpServletRequest, SearchCacheAble searchCacheAble) {
Map<String, String> paramMap = HttpServletRequestUtils.transParamType(httpServletRequest);
return this.getCacheKey(paramMap, searchCacheAble);
}
// 获取请求的参数
private String getCacheKey(Map<?, ?> paramMap, SearchCacheAble searchCacheAble) {
List<String> includeParams = Arrays.asList(searchCacheAble.includeParams());
List<String> excludeParams = Arrays.asList(searchCacheAble.excludeParams());
String paramKey = "";
if (!includeParams.isEmpty()) {
paramKey = HttpServletRequestUtils.genParamStringWithIncludeParams(paramMap, includeParams);
} else if (!excludeParams.isEmpty()) {
paramKey = HttpServletRequestUtils.genParamStringWithExcludeParams(paramMap, excludeParams);
} else {
paramKey = HttpServletRequestUtils.genParamString(paramMap);
}
StringBuilder realCacheKey = new StringBuilder("YOHOSEARCH:");
realCacheKey.append(searchCacheAble.cacheName());
realCacheKey.append(MD5Util.string2MD5(paramKey));
return realCacheKey.toString();
}
}
... ...
package com.yoho.search.common.utils;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import javax.servlet.http.HttpServletRequest;
import org.apache.commons.lang.StringUtils;
import org.apache.commons.lang3.ArrayUtils;
public class HttpServletRequestUtils {
public static Map<String, String> transParamType(HttpServletRequest request) {
Map<String, String> resultMap = new HashMap<>();
if(request.getParameterMap() != null) {
for (Map.Entry<String, String[]> entry : request.getParameterMap().entrySet()) {
String value = ArrayUtils.isEmpty(entry.getValue()) ? null : entry.getValue()[0];
resultMap.put(entry.getKey(), value);
}
}
if (request.getParameterMap() != null) {
for (Map.Entry<String, String[]> entry : request.getParameterMap().entrySet()) {
String value = ArrayUtils.isEmpty(entry.getValue()) ? null : entry.getValue()[0];
resultMap.put(entry.getKey(), value);
}
}
return resultMap;
}
public static String transParamParamStr(HttpServletRequest request) {
if(request.getParameterMap() != null) {
if (request.getParameterMap() != null) {
StringBuilder paramStringBuilder = new StringBuilder();
for (Map.Entry<String, String[]> entry : request.getParameterMap().entrySet()) {
String value = ArrayUtils.isEmpty(entry.getValue()) ? null : entry.getValue()[0];
... ... @@ -31,7 +33,7 @@ public class HttpServletRequestUtils {
}
return "";
}
public static String getRequestUrl(HttpServletRequest request) {
StringBuffer sb = new StringBuffer();
sb.append(request.getRequestURI());
... ... @@ -45,38 +47,79 @@ public class HttpServletRequestUtils {
}
return sb.toString();
}
public static String getIpAddress(HttpServletRequest request){
String ip = request.getHeader("x-forwarded-for");
if (ip == null || ip.length() == 0 || "unknown".equalsIgnoreCase(ip)) {
ip = request.getHeader("Proxy-Client-IP");
}
if (ip == null || ip.length() == 0 || "unknown".equalsIgnoreCase(ip)) {
ip = request.getHeader("WL-Proxy-Client-IP");
}
if (ip == null || ip.length() == 0 || "unknown".equalsIgnoreCase(ip)) {
ip = request.getHeader("HTTP_CLIENT_IP");
}
if (ip == null || ip.length() == 0 || "unknown".equalsIgnoreCase(ip)) {
ip = request.getHeader("HTTP_X_FORWARDED_FOR");
}
if (ip == null || ip.length() == 0 || "unknown".equalsIgnoreCase(ip)) {
ip = request.getRemoteAddr();
}
return ip;
public static String getIpAddress(HttpServletRequest request) {
String ip = request.getHeader("x-forwarded-for");
if (ip == null || ip.length() == 0 || "unknown".equalsIgnoreCase(ip)) {
ip = request.getHeader("Proxy-Client-IP");
}
if (ip == null || ip.length() == 0 || "unknown".equalsIgnoreCase(ip)) {
ip = request.getHeader("WL-Proxy-Client-IP");
}
if (ip == null || ip.length() == 0 || "unknown".equalsIgnoreCase(ip)) {
ip = request.getHeader("HTTP_CLIENT_IP");
}
if (ip == null || ip.length() == 0 || "unknown".equalsIgnoreCase(ip)) {
ip = request.getHeader("HTTP_X_FORWARDED_FOR");
}
if (ip == null || ip.length() == 0 || "unknown".equalsIgnoreCase(ip)) {
ip = request.getRemoteAddr();
}
return ip;
}
public static String genParamString(Map<String, String> paramMap) {
if(paramMap==null || paramMap.isEmpty()){
public static String genParamString(Map<?, ?> paramMap) {
if (paramMap == null || paramMap.isEmpty()) {
return "";
}
StringBuilder paramStringBuilder = new StringBuilder();
for (Map.Entry<String, String> entry : paramMap.entrySet()) {
String key = entry.getKey();
paramStringBuilder.append('&').append(key).append('=').append(entry.getValue());
for (Map.Entry<?, ?> entry : paramMap.entrySet()) {
String key = String.valueOf(entry.getKey());
String value = String.valueOf(entry.getValue());
if (StringUtils.isBlank(key) || StringUtils.isBlank(value)) {
continue;
}
paramStringBuilder.append('&').append(key).append('=').append(value);
}
return paramStringBuilder.toString().replaceFirst("&", "");
}
public static String genParamStringWithIncludeParams(Map<?, ?> paramMap, List<String> includeParams) {
if (paramMap == null || paramMap.isEmpty()) {
return "";
}
StringBuilder paramStringBuilder = new StringBuilder();
for (Map.Entry<?, ?> entry : paramMap.entrySet()) {
String key = String.valueOf(entry.getKey());
String value = String.valueOf(entry.getValue());
if (StringUtils.isBlank(key) || StringUtils.isBlank(value)) {
continue;
}
if (!includeParams.contains(key)) {
continue;
}
paramStringBuilder.append('&').append(key).append('=').append(value);
}
return paramStringBuilder.toString().replaceFirst("&", "");
}
public static String genParamStringWithExcludeParams(Map<?, ?> paramMap, List<String> excludeParams) {
if (paramMap == null || paramMap.isEmpty()) {
return "";
}
StringBuilder paramStringBuilder = new StringBuilder();
for (Map.Entry<?, ?> entry : paramMap.entrySet()) {
String key = String.valueOf(entry.getKey());
String value = String.valueOf(entry.getValue());
if (StringUtils.isBlank(key) || StringUtils.isBlank(value)) {
continue;
}
if (excludeParams.contains(key)) {
continue;
}
paramStringBuilder.append('&').append(key).append('=').append(value);
}
return paramStringBuilder.toString().replaceFirst("&", "");
}
}
... ...
... ... @@ -182,7 +182,7 @@ public class SearchSortHelper {
this.addSortBuildSorts(sortBuilders, filteredFieldNames, "firstShelveTime", SortOrder.DESC);
this.addSortBuildSorts(sortBuilders, filteredFieldNames, "id", SortOrder.DESC);
}
/**
* 构造排序方式
*
... ...
... ... @@ -53,6 +53,11 @@ public class BaseRecallService {
searchParam.setSortBuilders(sortBuilders);
return searchParam;
}
protected int getViewNum(Map<String,String> paramMap){
return MapUtils.getIntValue(paramMap, "viewNum", 60);
}
/**
* 合并召回结果，默认最后一个包含了总数量
... ... @@ -68,12 +73,10 @@ public class BaseRecallService {
productList.addAll(this.getCommonRecallSknList(searchResult, existsProductSkn));
}
long totalCount = searchResults.get(searchResults.size() - 1).getTotal();
long listCount = productList.size();
// 2、构造返回结果
CommonRecallResult commonRecallResults = new CommonRecallResult();
commonRecallResults.setTotalCount(totalCount);
commonRecallResults.setListCount(listCount);
commonRecallResults.setProductList(productList);
commonRecallResults.setRecallSknList(productList);
return commonRecallResults;
}
... ...
package com.yoho.search.service.recall;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collections;
import java.util.List;
import java.util.Map;
import org.apache.commons.collections.MapUtils;
import org.apache.commons.lang.StringUtils;
import org.elasticsearch.index.query.BoolQueryBuilder;
import org.elasticsearch.index.query.QueryBuilders;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Service;
import com.alibaba.fastjson.JSONObject;
import com.yoho.search.base.utils.ISearchConstants;
import com.yoho.search.base.utils.ProductIndexEsField;
import com.yoho.search.common.cache.aop.SearchCacheAble;
import com.yoho.search.core.es.model.SearchParam;
import com.yoho.search.core.es.model.SearchResult;
import com.yoho.search.core.personalized.PersonalizedSearch;
import com.yoho.search.models.SearchApiResult;
import com.yoho.search.service.base.ProductListSortService;
import com.yoho.search.service.base.SearchCommonService;
import com.yoho.search.service.base.SearchRequestParams;
import com.yoho.search.service.base.index.ProductIndexBaseService;
import com.yoho.search.service.helper.SearchParamHelper;
import com.yoho.search.service.helper.SearchSortHelper;
import com.yoho.search.service.recall.cacheable.CacheAbleServiceHelper;
import com.yoho.search.service.recall.cacheable.CommonPageRecallService;
import com.yoho.search.service.recall.model.CommonRecallParam;
import com.yoho.search.service.recall.model.CommonRecallResult;
import com.yoho.search.service.recall.model.CommonRecallSkn;
import com.yoho.search.service.scorer.personal.PersonalVectorFeatureSearch;
@Service
public class CommonSceneProductListService {
@Autowired
private CommonPageRecallService commonRecallSceneService;
@Autowired
private ProductListSortService productListSortService;
@Autowired
private SearchParamHelper searchParamHelper;
@Autowired
private SearchSortHelper searchSortHelper;
@Autowired
private ProductIndexBaseService productIndexBaseService;
@Autowired
private SearchCommonService searchCommonService;
@Autowired
private PersonalVectorFeatureSearch personalVectorFeatureSearch;
@Autowired
private ProductFeatureFactorHepler productFeatureFactorHepler;
@Autowired
private CacheAbleServiceHelper cacheAbleServiceHelper;
private long getTotalPage(long total, int viewNum) {
if (viewNum == 0) {
return total;
}
long totalPage = total / viewNum;
if (total % viewNum > 0) {
totalPage = totalPage + 1;
}
return totalPage;
}
/**
* 个性化的列表接口
*
* @order为空并且有uid
* @param paramMap
* @return
*/
public SearchApiResult productListForPersional(Map<String, String> paramMap) {
// 1、召回整数页个skn
CommonRecallResult commonRecallResult = commonRecallSceneService.doCommonPageRecallBatch(paramMap);
// 2、查询这整数页个skn的全部信息-包含了变价计划
SearchApiResult productInfoMapResult = cacheAbleServiceHelper.queryProductInfoMap(commonRecallResult);
// 3、计算这些skn得分并排序
commonRecallResult = this.callUserScoreAndRank(paramMap, commonRecallResult);
// 4、获取分页参数
int page = StringUtils.isBlank(paramMap.get("page")) ? 1 : Integer.parseInt(paramMap.get("page"));
int viewNum = StringUtils.isBlank(paramMap.get("viewNum")) ? 60 : Integer.parseInt(paramMap.get("viewNum"));
long total = commonRecallResult.getTotalCount();
// 5、为用户做列表截取-保留第一页数据
int recallMaxPage = (int) (commonRecallResult.listCount() / viewNum);
if (recallMaxPage == 0) {
recallMaxPage = 1;
}
// 6、构造分页结果参数
JSONObject results = new JSONObject();
results.put("total", total);
results.put("page", page);
results.put("page_size", viewNum);
results.put("page_total", this.getTotalPage(total, viewNum));
// 7、构造productList
if (page <= recallMaxPage) {
results.put("product_list", this.getProductList(productInfoMapResult, commonRecallResult, page, viewNum));
return new SearchApiResult().setData(results);
}
// 8、其他页面使用CommonRecallParamService去查询
CommonRecallParam commonRecallParam = new CommonRecallParam(paramMap);
BoolQueryBuilder extendMustFilter = QueryBuilders.boolQuery();
extendMustFilter.mustNot(QueryBuilders.termsQuery(ProductIndexEsField.productSkn, commonRecallResult.toSknList()));
commonRecallParam.setExtendMustFilter(extendMustFilter);
commonRecallParam.setPage(page - recallMaxPage);
commonRecallParam.setViewNum(viewNum);
results.put("product_list", cacheAbleServiceHelper.queryProductListByRecallParam(commonRecallParam).getData());
return new SearchApiResult().setData(results);
}
private List<? extends Map<?, ?>> getProductList(SearchApiResult productInfoMapResult, CommonRecallResult commonRecallResult, int page, int viewNum) {
JSONObject productInfoMap = (JSONObject) productInfoMapResult.getData();
List<JSONObject> results = new ArrayList<JSONObject>();
List<CommonRecallSkn> recallSknList = commonRecallResult.getRecallSknList();
int fromIndex = (page - 1) * viewNum;
int toIndex = page * viewNum;
if (toIndex > recallSknList.size()) {
toIndex = recallSknList.size();
}
recallSknList = recallSknList.subList(fromIndex, toIndex);
for (CommonRecallSkn commonRecallSkn : recallSknList) {
results.add(productInfoMap.getJSONObject(String.valueOf(commonRecallSkn.getProductSkn())));
}
return results;
}
/**
* 根据向量计算得分并排序
*
* @param paramMap
* @param commonRecallResult
* @return
*/
private CommonRecallResult callUserScoreAndRank(Map<String, String> paramMap, CommonRecallResult commonRecallResult) {
PersonalizedSearch personalizedSearch = personalVectorFeatureSearch.getPersonalizedSearch(paramMap);
if (personalizedSearch == null) {
return commonRecallResult;
}
String userVectorFeature = personalizedSearch.getUserVectorFeature();
String[] userFeatureFactorArr = userVectorFeature.split(",");
String vectorFeatureVersion = personalizedSearch.getVectorFeatureVersion();
List<String> firstProductSkns = Arrays.asList(MapUtils.getString(paramMap, SearchRequestParams.PARAM_SEARCH_FIRST_PRODUCRSKN, "").split(","));
for (CommonRecallSkn commonRecallSkn : commonRecallResult.getRecallSknList()) {
if (firstProductSkns.contains(String.valueOf(commonRecallSkn.getProductSkn()))) {
commonRecallSkn.setScore(10000);
} else {
commonRecallSkn.setScore(productFeatureFactorHepler.calProductFeatureFactor(userFeatureFactorArr, vectorFeatureVersion, commonRecallSkn.getProductFeatureFactor()));
}
}
Collections.sort(commonRecallResult.getRecallSknList());
return commonRecallResult;
}
/**
* 非个性化的列表接口
*
* @order不为空，或者无uid
* @param paramMap
* @return
*/
@SearchCacheAble(cacheName = "SCENE_PRODUCT_LIST_DEFAULT", cacheInMinute = 10, excludeParams = { "uid" })
public SearchApiResult productListForDefault(Map<String, String> paramMap) {
try {
// 1）验证查询条数
int pageSize = StringUtils.isBlank(paramMap.get("viewNum")) ? 10 : Integer.parseInt(paramMap.get("viewNum"));
int page = StringUtils.isBlank(paramMap.get("page")) ? 1 : Integer.parseInt(paramMap.get("page"));
if (page < 1 || pageSize < 0) {
throw new IllegalArgumentException("分页参数不合法");
}
if (pageSize > 100) {
pageSize = 100;
}
// 2)构建基本查询参数
SearchParam searchParam = searchParamHelper.buildWithPersional(paramMap, true);
searchParam.setAggregationBuilders(null);
searchParam.setOffset((page - 1) * pageSize);
searchParam.setSize(pageSize);
// 3）设置排序字段
searchParam.setSortBuilders(searchSortHelper.buildSortList(paramMap));
// 4)设置返回的参数【节省带宽】
List<String> includeFields = productIndexBaseService.getProductIndexIncludeFields();
searchParam.setIncludeFields(includeFields);
SearchResult searchResult = searchCommonService.doSearch(ISearchConstants.INDEX_NAME_PRODUCT_INDEX, searchParam);
// 5)构造返回结果
JSONObject dataMap = new JSONObject();
dataMap.put("total", searchResult.getTotal());
dataMap.put("page", searchResult.getPage());
dataMap.put("page_size", searchParam.getSize());
dataMap.put("page_total", searchResult.getTotalPage());
List<Map<String, Object>> product_list = productIndexBaseService.getProductListWithPricePlan(searchResult.getResultList());
dataMap.put("product_list", productListSortService.sortProductList(product_list, paramMap));// 处理一下商品的顺序;
return new SearchApiResult().setData(dataMap);
} catch (Exception e) {
return new SearchApiResult().setData(null).setCode(500);
}
}
}
... ...
package com.yoho.search.service.recall;
import org.springframework.stereotype.Component;
@Component
public class ProductFeatureFactorHepler {
private static double baseConstant = 1;
private static double factorConstant = 1;
public double calProductFeatureFactor(String[] userFeatureFactorArr,String vectorFeatureVersion,String productFeatureFactor){
if(userFeatureFactorArr==null || userFeatureFactorArr.length==0){
return baseConstant;
}
int dimensionOfFactors = userFeatureFactorArr.length;
double[] userFeatureFactors = new double[dimensionOfFactors];
double temp;
double userFeatureVectorNorm = 0.0D;
for (int index = 0; index < dimensionOfFactors; index++) {
temp = Double.parseDouble(userFeatureFactorArr[index].trim());
userFeatureFactors[index] = temp;
userFeatureVectorNorm += temp * temp;
}
String versionPrefix = vectorFeatureVersion + "|";
if (!productFeatureFactor.trim().startsWith(versionPrefix)) {
return baseConstant;
}
String[] productFeatureFactorArr = productFeatureFactor.trim().substring(versionPrefix.length()).split(",");
if (productFeatureFactorArr == null || productFeatureFactorArr.length != userFeatureFactors.length) {
return baseConstant;
}
double prodFeatureVectorNorm = 0.0D;
double productiveSum = 0.0D;
double tempProdFactor;
for (int i = 0; i < dimensionOfFactors; i++) {
tempProdFactor = Double.parseDouble(productFeatureFactorArr[i].trim());
productiveSum += tempProdFactor * userFeatureFactors[i];
prodFeatureVectorNorm += tempProdFactor * tempProdFactor;
}
if (prodFeatureVectorNorm == 0) {
return baseConstant;
}
double cosScore = productiveSum / (Math.sqrt(prodFeatureVectorNorm) * Math.sqrt(userFeatureVectorNorm));
double finalScore = baseConstant + factorConstant * cosScore;
return finalScore;
}
}
... ...
package com.yoho.search.service.recall;
import org.springframework.stereotype.Component;
import com.yoho.search.base.utils.ISearchConstants;
import com.yoho.search.common.cache.aop.SearchCacheAble;
import com.yoho.search.core.es.model.SearchParam;
import com.yoho.search.core.es.model.SearchResult;
import com.yoho.search.service.recall.model.CommonRecallParam;
import com.yoho.search.service.recall.model.CommonRecallResult;
@Component
public class SortRecallSceneService extends BaseRecallService {
/**
* 单条件召回策略
*
* @param commonRecallParam
* @return
*/
@SearchCacheAble(cacheInMinute = 10, cacheName = "COMMON_RECALL_SINGLE", returnClass = CommonRecallResult.class, excludeParams = { "uid", "order", "page", "viewNum" })
public CommonRecallResult doCommonRecallSingel(CommonRecallParam commonRecallParam) {
try {
// 1、构造SearchParam
SearchParam searchParam = this.buildSearchParam(commonRecallParam.getParamMap(), commonRecallParam.getExtendMustFilter(), commonRecallParam.getSortBuilders(),
commonRecallParam.getSize());
// 2、查询es
SearchResult searchResult = searchCommonService.doSearch(ISearchConstants.INDEX_NAME_PRODUCT_INDEX, searchParam);
// 3、获取值
return this.getCommonRecallResult(searchResult);
} catch (Exception e) {
return new CommonRecallResult();
}
}
}
package com.yoho.search.service.recall.cacheable;
import java.util.List;
import java.util.Map;
import org.apache.commons.collections.MapUtils;
import org.apache.commons.lang.StringUtils;
import org.elasticsearch.index.query.BoolQueryBuilder;
import org.elasticsearch.index.query.QueryBuilders;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Service;
import com.alibaba.fastjson.JSONObject;
import com.yoho.search.base.utils.ISearchConstants;
import com.yoho.search.base.utils.ProductIndexEsField;
import com.yoho.search.common.cache.aop.SearchCacheAble;
import com.yoho.search.core.es.model.SearchParam;
import com.yoho.search.core.es.model.SearchResult;
import com.yoho.search.models.SearchApiResult;
import com.yoho.search.service.base.ProductListSortService;
import com.yoho.search.service.base.SearchCommonService;
import com.yoho.search.service.base.index.ProductIndexBaseService;
import com.yoho.search.service.helper.SearchParamHelper;
import com.yoho.search.service.helper.SearchSortHelper;
import com.yoho.search.service.recall.model.CommonRecallParam;
import com.yoho.search.service.recall.model.CommonRecallResult;
@Service
public class CacheAbleServiceHelper {
private static final Logger logger = LoggerFactory.getLogger(CacheAbleServiceHelper.class);
@Autowired
private ProductIndexBaseService productIndexBaseService;
@Autowired
private SearchCommonService searchCommonService;
@Autowired
private SearchParamHelper searchParamHelper;
@Autowired
private ProductListSortService productListSortService;
@Autowired
private SearchSortHelper searchSortHelper;
/**
* 根据skn批量查询skn信息-包含了变价计划
*
* @param commonRecallResult
* @return
*/
@SearchCacheAble(cacheName = "COMMON_SCENE_QUERY_BY_SKN", cacheInMinute = 10)
public SearchApiResult queryProductInfoMap(CommonRecallResult commonRecallResult) {
try {
// 1)获取skn
List<Integer> productSkns = commonRecallResult.toSknList();
// 2)构建基本查询参数
BoolQueryBuilder boolQueryBuilder = QueryBuilders.boolQuery();
boolQueryBuilder.must(QueryBuilders.termsQuery(ProductIndexEsField.productSkn, productSkns));
SearchParam searchParam = new SearchParam();
searchParam.setFiter(boolQueryBuilder);
searchParam.setSize(productSkns.size());
// 3)设置返回的参数【节省带宽】
List<String> includeFields = productIndexBaseService.getProductIndexIncludeFields();
searchParam.setIncludeFields(includeFields);
SearchResult searchResult = searchCommonService.doSearch(ISearchConstants.INDEX_NAME_PRODUCT_INDEX, searchParam);
// 4)构造返回结果-带变价计划
List<Map<String, Object>> product_list = productIndexBaseService.getProductListWithPricePlan(searchResult.getResultList());
JSONObject results = new JSONObject();
for (Map<String, Object> map : product_list) {
results.put(MapUtils.getString(map, "product_skn"), new JSONObject(map));
}
return new SearchApiResult().setData(results);
} catch (Exception e) {
logger.error(e.getMessage(),e);
return new SearchApiResult().setData(null).setCode(500);
}
}
/**
* 非个性化的列表接口
*
* @order不为空，或者无uid
* @param paramMap
* @return
*/
@SearchCacheAble(cacheName = "COMMON_SCENE_QUERY_BY_PARAM", cacheInMinute = 10, excludeParams = { "uid" })
public SearchApiResult queryProductListByRecallParam(CommonRecallParam commonRecallParam) {
try {
// 1、获取参数
Map<String, String> paramMap = commonRecallParam.getParamMap();
int page = commonRecallParam.getPage();
if (page == 0) {
page = StringUtils.isBlank(paramMap.get("page")) ? 1 : Integer.parseInt(paramMap.get("page"));
}
int pageSize = commonRecallParam.getViewNum();
if (pageSize == 0) {
pageSize = StringUtils.isBlank(paramMap.get("viewNum")) ? 10 : Integer.parseInt(paramMap.get("viewNum"));
}
// 2、构造核心参数
SearchParam searchParam = searchParamHelper.buildWithMustFilter(paramMap, commonRecallParam.getExtendMustFilter());
searchParam.setAggregationBuilders(null);
searchParam.setOffset((page - 1) * pageSize);
searchParam.setSize(pageSize);
// 2）设置排序字段
searchParam.setSortBuilders(commonRecallParam.getSortBuilders());
// 4)设置返回的参数【节省带宽】
List<String> includeFields = productIndexBaseService.getProductIndexIncludeFields();
searchParam.setIncludeFields(includeFields);
SearchResult searchResult = searchCommonService.doSearch(ISearchConstants.INDEX_NAME_PRODUCT_INDEX, searchParam);
// 5)构造返回结果
List<Map<String, Object>> product_list = productIndexBaseService.getProductListWithPricePlan(searchResult.getResultList());
product_list = productListSortService.sortProductList(product_list, paramMap);// 处理一下商品的顺序;
return new SearchApiResult().setData(product_list);
} catch (Exception e) {
logger.error(e.getMessage(),e);
return new SearchApiResult().setData(null).setCode(500);
}
}
}
... ...
package com.yoho.search.service.recall;
package com.yoho.search.service.recall.cacheable;
import java.util.ArrayList;
import java.util.List;
... ... @@ -15,6 +15,7 @@ import org.elasticsearch.search.sort.SortBuilders;
import org.elasticsearch.search.sort.SortOrder;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Component;
import com.yoho.search.base.utils.ISearchConstants;
... ... @@ -22,38 +23,59 @@ import com.yoho.search.base.utils.ProductIndexEsField;
import com.yoho.search.common.cache.aop.SearchCacheAble;
import com.yoho.search.core.es.model.SearchParam;
import com.yoho.search.core.es.model.SearchResult;
import com.yoho.search.service.base.ProductListSortService;
import com.yoho.search.service.base.SearchRequestParams;
import com.yoho.search.service.base.index.ProductIndexBaseService;
import com.yoho.search.service.helper.SearchSortHelper;
import com.yoho.search.service.recall.BaseRecallService;
import com.yoho.search.service.recall.model.CommonRecallResult;
import com.yoho.search.service.recall.model.CommonRecallSkn;
@Component
public class CommonRecallSceneService extends BaseRecallService {
private static final Logger logger = LoggerFactory.getLogger(CommonRecallSceneService.class);
public class CommonPageRecallService extends BaseRecallService {
private static final Logger logger = LoggerFactory.getLogger(CommonPageRecallService.class);
@Autowired
private SearchSortHelper searchSortHelper;
@Autowired
private ProductIndexBaseService productIndexBaseService;
@Autowired
private ProductListSortService productListSortService;
/**
* 默认召回首页skn的缓存
* 通用页面默认的召回机制
*
* @param paramMap
* @return
*/
@SearchCacheAble(cacheInMinute = 10, cacheName = "COMMON_RECALL_BATCH", returnClass = CommonRecallResult.class, excludeParams = { "uid", "order", "page", "viewNum" })
public CommonRecallResult doCommonRecallBatch(Map<String, String> paramMap) {
@SearchCacheAble(cacheInMinute = 10, cacheName = "COMMON_PAGE_RECALL_BATCH", returnClass = CommonRecallResult.class, excludeParams = { "uid", "order", "page" })
public CommonRecallResult doCommonPageRecallBatch(Map<String, String> paramMap) {
try {
int viewNum = this.getViewNum(paramMap);
// 1、构造SearchParam
List<SearchParam> searchParams = new ArrayList<SearchParam>();
searchParams.add(this.buildFirstSknSearchParam(paramMap));
searchParams.add(this.buildHeatValueDescSearchParam(paramMap));
searchParams.add(this.buildRandomSearchParam(paramMap));
searchParams.add(this.buildFirstSknSearchParam(paramMap, 1));
searchParams.add(this.buildHeatValueDescSearchParam(paramMap, viewNum * 2));
searchParams.add(this.buildRandomSearchParam(paramMap, viewNum * 2));
// 2、查询es
List<SearchResult> searchResults = searchCommonService.doMutiSearch(ISearchConstants.INDEX_NAME_PRODUCT_INDEX, searchParams);
// 3、获取值
return this.getCommonRecallResult(searchResults);
CommonRecallResult commonRecallResult = this.getCommonRecallResult(searchResults);
// 4、保留整数位-只有一页数据则直接返回
int recallMaxPage = (int) (commonRecallResult.listCount() / viewNum);
if (recallMaxPage == 0) {
return commonRecallResult;
}
List<CommonRecallSkn> recallSknList = commonRecallResult.getRecallSknList();
commonRecallResult.setRecallSknList(recallSknList.subList(0, recallMaxPage * viewNum));
return commonRecallResult;
} catch (Exception e) {
logger.error(e.getMessage(), e);
return new CommonRecallResult();
}
}
/**
* 支持FirstSkn的召回
*
... ... @@ -61,7 +83,7 @@ public class CommonRecallSceneService extends BaseRecallService {
* @return
* @throws Exception
*/
private SearchParam buildFirstSknSearchParam(Map<String, String> paramMap) throws Exception {
private SearchParam buildFirstSknSearchParam(Map<String, String> paramMap, int size) throws Exception {
// 1、构建filter
BoolQueryBuilder mustFilter = QueryBuilders.boolQuery();
String firstProductSkns = MapUtils.getString(paramMap, SearchRequestParams.PARAM_SEARCH_FIRST_PRODUCRSKN, "");
... ... @@ -75,7 +97,7 @@ public class CommonRecallSceneService extends BaseRecallService {
sortBuilders.add(SortBuilders.fieldSort(ProductIndexEsField.id).order(SortOrder.DESC));
// 3、构造SearchParam
return this.buildSearchParam(paramMap, mustFilter, sortBuilders, 1);
return this.buildSearchParam(paramMap, mustFilter, sortBuilders, size);
}
/**
... ... @@ -85,7 +107,7 @@ public class CommonRecallSceneService extends BaseRecallService {
* @return
* @throws Exception
*/
private SearchParam buildHeatValueDescSearchParam(Map<String, String> paramMap) throws Exception {
private SearchParam buildHeatValueDescSearchParam(Map<String, String> paramMap, int size) throws Exception {
// 1、构建filter
BoolQueryBuilder mustFilter = QueryBuilders.boolQuery();
mustFilter.must(QueryBuilders.rangeQuery(ProductIndexEsField.breakSizePercent).lte(50));// 非断码
... ... @@ -95,7 +117,7 @@ public class CommonRecallSceneService extends BaseRecallService {
sortBuilders.add(SortBuilders.fieldSort(ProductIndexEsField.heatValue).order(SortOrder.DESC));
// 3、构造SearchParam
return this.buildSearchParam(paramMap, mustFilter, sortBuilders, 100);
return this.buildSearchParam(paramMap, mustFilter, sortBuilders, size);
}
/**
... ... @@ -105,7 +127,7 @@ public class CommonRecallSceneService extends BaseRecallService {
* @return
* @throws Exception
*/
private SearchParam buildRandomSearchParam(Map<String, String> paramMap) throws Exception {
private SearchParam buildRandomSearchParam(Map<String, String> paramMap, int size) throws Exception {
// 1、构建filter
BoolQueryBuilder mustFilter = QueryBuilders.boolQuery();
... ... @@ -113,7 +135,9 @@ public class CommonRecallSceneService extends BaseRecallService {
List<SortBuilder<?>> sortBuilders = new ArrayList<SortBuilder<?>>();
sortBuilders.add(SortBuilders.scriptSort(new Script("Math.random()"), ScriptSortType.NUMBER).order(SortOrder.ASC));
return this.buildSearchParam(paramMap, mustFilter, sortBuilders, 100);
return this.buildSearchParam(paramMap, mustFilter, sortBuilders, size);
}
}
... ...
... ... @@ -12,22 +12,25 @@ import org.elasticsearch.search.sort.SortBuilders;
import org.elasticsearch.search.sort.SortOrder;
import com.yoho.search.base.utils.ProductIndexEsField;
import com.yoho.search.common.cache.aop.SearchCacheAbleParam;
import com.yoho.search.common.utils.HttpServletRequestUtils;
public class CommonRecallParam {
public class CommonRecallParam extends SearchCacheAbleParam {
private Map<String, String> paramMap;
private BoolQueryBuilder extendMustFilter;
private int size;
private List<SortBuilder<?>> sortBuilders;
private List<SortBuilder<?>> sortBuilders = new ArrayList<SortBuilder<?>>();
private int page;
private int viewNum;
public CommonRecallParam(Map<String, String> paramMap){
this.paramMap = paramMap;
}
public Map<String, String> getParamMap() {
return paramMap;
}
public void setParamMap(Map<String, String> paramMap) {
this.paramMap = paramMap;
}
public BoolQueryBuilder getExtendMustFilter() {
return extendMustFilter;
}
... ... @@ -36,14 +39,6 @@ public class CommonRecallParam {
this.extendMustFilter = extendMustFilter;
}
public int getSize() {
return size;
}
public void setSize(int size) {
this.size = size;
}
public List<SortBuilder<?>> getSortBuilders() {
return sortBuilders;
}
... ... @@ -52,6 +47,41 @@ public class CommonRecallParam {
this.sortBuilders = sortBuilders;
}
public int getPage() {
return page;
}
public void setPage(int page) {
this.page = page;
}
public int getViewNum() {
return viewNum;
}
public void setViewNum(int viewNum) {
this.viewNum = viewNum;
}
@Override
public String searchCacheKey() {
StringBuilder sb = new StringBuilder();
List<String> excludeParams = new ArrayList<String>();
excludeParams.add("uid");
if (page > 0) {
excludeParams.add("page");
}
if (viewNum > 0) {
excludeParams.add("viewNum");
}
sb.append("paramMap=").append(HttpServletRequestUtils.genParamStringWithExcludeParams(paramMap,excludeParams));
sb.append("&extendMustFilter=").append(extendMustFilter==null?"":extendMustFilter.toString());
sb.append("&sortBuilders=").append(sortBuilders==null||sortBuilders.isEmpty()?"":sortBuilders.toString());
sb.append("&page=").append(page);
sb.append("&viewNum=").append(viewNum);
return sb.toString();
}
public static void main(String[] args) {
List<SortBuilder<?>> sortBuilders = new ArrayList<SortBuilder<?>>();
sortBuilders.add(SortBuilders.fieldSort(ProductIndexEsField.heatValue).order(SortOrder.DESC));
... ...
... ... @@ -4,12 +4,15 @@ import java.io.Serializable;
import java.util.ArrayList;
import java.util.List;
public class CommonRecallResult implements Serializable {
import org.apache.commons.lang.StringUtils;
import com.yoho.search.common.cache.aop.SearchCacheAbleParam;
public class CommonRecallResult extends SearchCacheAbleParam implements Serializable {
private static final long serialVersionUID = -5811560665992908361L;
private long totalCount = 0;
private long listCount = 0;
private List<CommonRecallSkn> productList = new ArrayList<CommonRecallSkn>();
private List<CommonRecallSkn> recallSknList = new ArrayList<CommonRecallSkn>();
public long getTotalCount() {
return totalCount;
... ... @@ -19,20 +22,29 @@ public class CommonRecallResult implements Serializable {
this.totalCount = totalCount;
}
public long getListCount() {
return listCount;
public List<CommonRecallSkn> getRecallSknList() {
return recallSknList;
}
public void setListCount(long listCount) {
this.listCount = listCount;
public void setRecallSknList(List<CommonRecallSkn> recallSknList) {
this.recallSknList = recallSknList;
}
public List<CommonRecallSkn> getProductList() {
return productList;
public long listCount() {
return recallSknList.size();
}
public List<Integer> toSknList() {
List<Integer> productSkns = new ArrayList<Integer>();
for (CommonRecallSkn commonRecallSkn : recallSknList) {
productSkns.add(commonRecallSkn.getProductSkn());
}
return productSkns;
}
public void setProductList(List<CommonRecallSkn> productList) {
this.productList = productList;
@Override
public String searchCacheKey() {
return StringUtils.join(toSknList(),",");
}
}
... ...
package com.yoho.search.service.recall.model;
public class CommonRecallSkn {
import java.io.Serializable;
public class CommonRecallSkn implements Comparable<CommonRecallSkn>,Serializable {
private static final long serialVersionUID = -6820037942157341237L;
private int productSkn;
private int brandId;
private int smallSortId;
... ... @@ -47,4 +51,9 @@ public class CommonRecallSkn {
public void setScore(double score) {
this.score = score;
}
@Override
public int compareTo(CommonRecallSkn old) {
return score - old.score >= 0 ? -1 : 1;
}
}
... ...
... ... @@ -2,6 +2,8 @@ package com.yoho.search.service.scene;
import java.util.Map;
import org.apache.commons.collections.MapUtils;
import org.apache.commons.lang.StringUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.beans.factory.annotation.Autowired;
... ... @@ -10,6 +12,7 @@ import org.springframework.stereotype.Service;
import com.yoho.search.base.utils.SearchPageIdDefine;
import com.yoho.search.common.utils.SearchApiResultUtils;
import com.yoho.search.models.SearchApiResult;
import com.yoho.search.service.recall.CommonSceneProductListService;
import com.yoho.search.service.scene.common.AbstractSceneService;
import com.yoho.search.service.scene.common.SceneProductListService;
import com.yoho.search.service.scene.common.SceneSelectionsService;
... ... @@ -23,12 +26,14 @@ public class CommonSceneService extends AbstractSceneService {
private SceneProductListService sceneProductListService;
@Autowired
private SceneSelectionsService sceneSelectionsService;
@Autowired
private CommonSceneProductListService commonSceneProductListService;
@Override
public String pageId() {
return SearchPageIdDefine.PAGE_ID_DEFAULT;
}
@Override
public void addParamsToParamMap(Map<String, String> paramMap) {
super.addDefaultParamsToParamMap(paramMap);
... ... @@ -40,13 +45,20 @@ public class CommonSceneService extends AbstractSceneService {
// 1、添加默认参数
this.addParamsToParamMap(paramMap);
// 2、返回商品列表
return sceneProductListService.productList(paramMap);
int uid = MapUtils.getIntValue(paramMap, "uid", 0);
String order = MapUtils.getString(paramMap, "order", "");
if (uid > 0 && StringUtils.isEmpty(order)) {
return commonSceneProductListService.productListForPersional(paramMap);
} else {
return commonSceneProductListService.productListForDefault(paramMap);
}
// return sceneProductListService.productList(paramMap);
} catch (Exception e) {
logger.error("[func=CommonProductList][params=" + paramMap + "]", e);
return SearchApiResultUtils.errorSearchApiResult("CouponProductList", paramMap, e);
return SearchApiResultUtils.errorSearchApiResult("CommonSceneService", paramMap, e);
}
}
@Override
public SearchApiResult aggregations(Map<String, String> paramMap) {
try {
... ... @@ -56,8 +68,7 @@ public class CommonSceneService extends AbstractSceneService {
return sceneSelectionsService.aggregations(paramMap);
} catch (Exception e) {
logger.error("[func=Couponaggregations][params=" + paramMap + "]", e);
return SearchApiResultUtils.errorSearchApiResult("Couponaggregations", paramMap, e);
return SearchApiResultUtils.errorSearchApiResult("CommonAggregations", paramMap, e);
}
}
}
... ...
package com.yoho.search.service.scene.common;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Service;
import com.yoho.search.service.recall.CommonRecallSceneService;
@Service
public class CommonSceneProductListService {
@Autowired
private CommonRecallSceneService commonRecallSceneService;
// public SearchApiResult productList(Map<String, String> paramMap) {
//
// }
}