yoho-search / yoho-search-service · Commits

Go to a project

Authored by hugufei
Commit 7b75ad21549beb1f648acfcc4a19079890b4d012 1 parent ecc913c1

品类页召回日志优化

Inline Side-by-side
Showing 3 changed files with 88 additions and 49 deletions
... ... @@ -18,6 +18,7 @@ import org.elasticsearch.search.sort.SortOrder;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.core.NamedThreadLocal;
import com.alibaba.fastjson.JSON;
import com.alibaba.fastjson.JSONObject;
... ... @@ -29,6 +30,7 @@ import com.yoho.search.core.es.model.SearchParam;
import com.yoho.search.core.es.model.SearchResult;
import com.yoho.search.service.base.SearchCacheService;
import com.yoho.search.service.base.SearchCommonService;
import com.yoho.search.service.helper.SearchCommonHelper;
import com.yoho.search.service.helper.SearchServiceHelper;
import com.yoho.search.service.scene.recall.model.RecallProductSknList;
import com.yoho.search.service.scene.recall.model.RecallResult;
... ... @@ -39,7 +41,9 @@ import com.yoho.search.service.scene.recall.type.RecallType;
public abstract class AbstractRecallService {
private static final Logger logger = LoggerFactory.getLogger("RECALL");
protected static final Logger RECALL_LOGGER = LoggerFactory.getLogger("RECALL");
private static final ThreadLocal<Boolean> logEnableThreadLocal = new NamedThreadLocal<Boolean>("RECALL USER LOG ENABLE");
@Autowired
private SearchCommonService searchCommonService;
... ... @@ -49,6 +53,8 @@ public abstract class AbstractRecallService {
private SearchCacheService searchCacheService;
@Autowired
private SearchCacheFactory searchCacheFactory;
@Autowired
private SearchCommonHelper searchCommonHelper;
private SearchCache searchCache;
... ... @@ -57,6 +63,29 @@ public abstract class AbstractRecallService {
searchCache = searchCacheFactory.getRecallCache();
}
protected void setLogEnableThreadLocal(Map<String, String> paramMap) {
int uid = searchCommonHelper.getUid(paramMap);
if (uid == 13420925) {
logEnableThreadLocal.set(true);
} else {
logEnableThreadLocal.set(false);
}
}
protected void removelogEnableThreadLocal() {
logEnableThreadLocal.remove();
}
protected void doLogInfo(String format, Object... arguments) {
if (logEnableThreadLocal.get() != null && logEnableThreadLocal.get().booleanValue()) {
RECALL_LOGGER.info(format, arguments);
}
}
protected void doLogError(String msg, Throwable t) {
RECALL_LOGGER.error(msg, t);
}
private List<SearchParam> getSearchpaParams(List<RecallSearchParam> recallSearchParams) {
List<SearchParam> results = new ArrayList<SearchParam>();
for (RecallSearchParam recallSearchParam : recallSearchParams) {
... ... @@ -75,12 +104,12 @@ public abstract class AbstractRecallService {
long begin = System.currentTimeMillis();
// 1、获取召回策略
List<IRecallStrategy> recallStrategys = this.getRecallStrategys(paramMap);
logger.info("[func=getRecallStrategys][cost={}ms]", System.currentTimeMillis() - begin);
this.doLogInfo("[func=getRecallStrategys][cost={}ms]", System.currentTimeMillis() - begin);
// 2、获取召回参数
begin = System.currentTimeMillis();
List<RecallSearchParam> recallSearchParams = this.getRecallSearchParams(paramMap, recallStrategys);
logger.info("[func=getRecallSearchParams][cost={}ms]", System.currentTimeMillis() - begin);
this.doLogInfo("[func=getRecallSearchParams][cost={}ms]", System.currentTimeMillis() - begin);
// 3、缓存中获取
List<SearchParam> searchParams = this.getSearchpaParams(recallSearchParams);
... ... @@ -92,26 +121,26 @@ public abstract class AbstractRecallService {
// 4、查询
begin = System.currentTimeMillis();
RecallResult recallResult = this.queryRecallResult(recallSearchParams);
logger.info("[func=queryRecallResult][cost={}ms]", System.currentTimeMillis() - begin);
this.doLogInfo("[func=queryRecallResult][cost={}ms][info is {} ]", System.currentTimeMillis() - begin, recallResult.getRecallSearchResult());
// 5、从兜底策略中获取总数
long total = this.getTotalFromRecallResult(recallResult);
// 6、粗排
// 6、粗排-去重
begin = System.currentTimeMillis();
recallResult = this.doSketchyRank(paramMap, recallResult);
logger.info("[func=doSketchyRank][cost={}ms]", System.currentTimeMillis() - begin);
this.doLogInfo("[func=doSketchyRank][total={}][cost={}ms]", recallResult.getProductList().size(), System.currentTimeMillis() - begin);
// 7、精排
// 7、精排-品牌打散
begin = System.currentTimeMillis();
recallResult = this.doCarefulRank(paramMap, recallResult);
logger.info("[func=doCarefulRank][cost={}ms]", System.currentTimeMillis() - begin);
this.doLogInfo("[func=doCarefulRank][total={}][cost={}ms]", recallResult.getProductList().size(), System.currentTimeMillis() - begin);
// 8、重排
// 8、重排-处理firstSkn和直通车
begin = System.currentTimeMillis();
recallResult = this.doReRank(paramMap, recallResult);
logger.info("[func=doReRank][cost={}ms]", System.currentTimeMillis() - begin);
this.doLogInfo("[func=doReRank][total=][cost={}ms]", recallResult.getProductList().size(), System.currentTimeMillis() - begin);
// 9、构造返回结果
begin = System.currentTimeMillis();
List<Integer> productSknList = new ArrayList<Integer>();
... ... @@ -153,7 +182,7 @@ public abstract class AbstractRecallService {
// 3、返回结果
return recallSearchParams;
} catch (Exception e) {
logger.error(e.getMessage(), e);
this.doLogError(e.getMessage(), e);
}
return new ArrayList<RecallSearchParam>();
}
... ... @@ -171,7 +200,7 @@ public abstract class AbstractRecallService {
searchParams.add(recallSearchParam.getSearchParam());
}
List<SearchResult> searchResults = searchCommonService.doMutiSearch(ISearchConstants.INDEX_NAME_PRODUCT_INDEX, searchParams);
logger.info("doMutiSearch cost is:[{}]", System.currentTimeMillis() - begin);
this.doLogInfo("doMutiSearch cost is:[{}]", System.currentTimeMillis() - begin);
List<RecallSearchResult> recallSearchResults = new ArrayList<RecallSearchResult>();
for (int i = 0; i < recallSearchParams.size(); i++) {
recallSearchResults.add(new RecallSearchResult(recallSearchParams.get(i).getRecallType(), searchResults.get(i)));
... ...
... ... @@ -2,6 +2,7 @@ package com.yoho.search.service.scene.recall;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collections;
import java.util.HashMap;
import java.util.HashSet;
import java.util.Iterator;
... ... @@ -20,8 +21,6 @@ import org.elasticsearch.script.ScriptType;
import org.elasticsearch.search.sort.SortBuilder;
import org.elasticsearch.search.sort.SortBuilders;
import org.elasticsearch.search.sort.SortOrder;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Service;
... ... @@ -42,6 +41,7 @@ import com.yoho.search.service.base.SearchCommonService;
import com.yoho.search.service.base.SearchDynamicConfigService;
import com.yoho.search.service.base.SearchRequestParams;
import com.yoho.search.service.base.index.ProductIndexBaseService;
import com.yoho.search.service.helper.SearchCommonHelper;
import com.yoho.search.service.helper.SearchParamHelper;
import com.yoho.search.service.helper.SearchServiceHelper;
import com.yoho.search.service.helper.SearchSortHelper;
... ... @@ -64,8 +64,6 @@ import com.yoho.search.service.service.IProductIndexService;
@Service
public class SortRecallSceneService extends AbstractRecallService {
private static final Logger logger = LoggerFactory.getLogger("RECALL");
@Autowired
private SearchCacheService searchCacheService;
@Autowired
... ... @@ -86,6 +84,8 @@ public class SortRecallSceneService extends AbstractRecallService {
private BigDataRedisService bigDataRedisService;
@Autowired
private ProductListSortService productListSortService;
@Autowired
private SearchCommonHelper searchCommonHelper;
/**
* @品类页商品召回接口
... ... @@ -119,19 +119,22 @@ public class SortRecallSceneService extends AbstractRecallService {
*/
public SearchApiResult productList(Map<String, String> paramMap) {
try {
// 1）验证查询条数
// 0)日志打印检查
super.setLogEnableThreadLocal(paramMap);
// 1)验证查询条数
int page = this.getPage(paramMap);
int pageSize = this.getPageSize(paramMap);
// 2)获取商品列表
long begin = System.currentTimeMillis();
RecallProductInfoList recallProductInfoList = this.queryRecallProductInfoList(paramMap, page, pageSize);
logger.info("[func1=SortRecallSceneService.queryRealProductList][cost={}]", System.currentTimeMillis() - begin);
super.doLogInfo("[func1=SortRecallSceneService.queryRealProductList][cost={}]", System.currentTimeMillis() - begin);
// 3)填充变价计划，并做品牌打散
begin = System.currentTimeMillis();
List<Map<String, Object>> product_list = productIndexBaseService.getProductListWithPricePlan(recallProductInfoList.getProductInfoList());
logger.info("[func2=SortRecallSceneService.getProductListWithPricePlan][cost={}]", System.currentTimeMillis() - begin);
super.doLogInfo("[func2=SortRecallSceneService.getProductListWithPricePlan][cost={}]", System.currentTimeMillis() - begin);
// 4)构造返回结果
JSONObject dataMap = new JSONObject();
... ... @@ -142,14 +145,16 @@ public class SortRecallSceneService extends AbstractRecallService {
dataMap.put("product_list", product_list);
return new SearchApiResult().setData(dataMap);
} catch (Exception e) {
logger.error(e.getMessage(), e);
super.doLogError(e.getMessage(), e);
return new SearchApiResult().setData(new JSONObject()).setMessage("SortProductList Exception").setCode(500);
} finally {
super.removelogEnableThreadLocal();
}
}
private RecallProductInfoList queryRecallProductInfoList(Map<String, String> paramMap, int page, int pageSize) throws Exception {
// 1、获取召回的skn列表
RecallProductSknList recallProductList = this.queryRecallProductSknList(paramMap);
RecallProductSknList recallProductList = super.queryRecallProductSknList(paramMap);
// 2、获取SKN列表
List<Integer> productSknList = recallProductList.getProductSknList();
... ... @@ -157,10 +162,13 @@ public class SortRecallSceneService extends AbstractRecallService {
// 3、判断召回的productSkn是否包含当前页码
int maxPage = productSknList.size() / pageSize;
List<Map<String, Object>> productInfoList = null;
long begin = System.currentTimeMillis();
if (page <= maxPage) {
productInfoList = this.queryProductListWithSort(productSknList.subList((page - 1) * pageSize, page * pageSize));
super.doLogInfo("[func=queryProductListWithSort][resultsize is {}][cost={}]", productInfoList.size(), System.currentTimeMillis() - begin);
} else {
productInfoList = this.queryProductListWithDefault(productSknList, paramMap, page, pageSize);
super.doLogInfo("[func=queryProductListWithDefault][resultsize is {}][cost={}]", productInfoList.size(), System.currentTimeMillis() - begin);
}
// 4、返回结果
return new RecallProductInfoList(recallProductList.getTotal(), productInfoList);
... ... @@ -174,7 +182,6 @@ public class SortRecallSceneService extends AbstractRecallService {
* @return
*/
private List<Map<String, Object>> queryProductListWithSort(List<Integer> querySknList) {
long begin = System.currentTimeMillis();
// 1.构造搜索参数
SearchParam searchParam = new SearchParam();
searchParam.setFiter(QueryBuilders.boolQuery().must(QueryBuilders.termsQuery(ProductIndexEsField.productSkn, querySknList)));
... ... @@ -199,7 +206,6 @@ public class SortRecallSceneService extends AbstractRecallService {
}
realResults.add(productMap.get(produtSkn));
}
logger.info("[func=queryProductListWithSort][query by sknList][resultsize is {}][cost={}]", productList.size(), System.currentTimeMillis() - begin);
return realResults;
}
... ... @@ -212,7 +218,6 @@ public class SortRecallSceneService extends AbstractRecallService {
* @throws Exception
*/
private List<Map<String, Object>> queryProductListWithDefault(List<Integer> notProductSkns, Map<String, String> paramMap, int page, int pageSize) throws Exception {
long begin = System.currentTimeMillis();
// 1.构造filter
BoolQueryBuilder mustFilter = QueryBuilders.boolQuery();
int realPage = page;
... ... @@ -220,7 +225,6 @@ public class SortRecallSceneService extends AbstractRecallService {
mustFilter.mustNot(QueryBuilders.termsQuery(ProductIndexEsField.productSkn, notProductSkns));
realPage = realPage - notProductSkns.size() / pageSize;
}
// 2、构造参数
SearchParam searchParam = searchParamHelper.buildWithPersional(paramMap, false, mustFilter);
searchParam.setOffset((realPage - 1) * pageSize);
... ... @@ -234,7 +238,6 @@ public class SortRecallSceneService extends AbstractRecallService {
// 4、查询es
SearchResult searchResult = searchCommonService.doSearch(ISearchConstants.INDEX_NAME_PRODUCT_INDEX, searchParam);
logger.info("[func=queryProductListWithDefault][resultsize is {}][cost={}]", searchResult.getTotal(), System.currentTimeMillis() - begin);
return searchResult.getResultList();
}
... ... @@ -256,16 +259,16 @@ public class SortRecallSceneService extends AbstractRecallService {
String uid = MapUtils.getString(paramMap, "uid", "0");
String vectorFeatureVersion = searchDynamicConfigService.personalizedSearchVersion();
String userGlobalBrandIds = bigDataRedisService.getUserGlobalFaveriteBrand(uid, vectorFeatureVersion);
logger.info("uid is [{}],vectorFeatureVersion is [{}], userGlobalBrandIds is [{}]", uid, vectorFeatureVersion, userGlobalBrandIds);
super.doLogInfo("uid is [{}],vectorFeatureVersion is [{}], userGlobalBrandIds is [{}]", uid, vectorFeatureVersion, userGlobalBrandIds);
JSONArray brandJsonArray = JSON.parseArray(userGlobalBrandIds);
List<Integer> results = new ArrayList<Integer>();
for (int i = 0; i < brandJsonArray.size(); i++) {
results.add(Integer.valueOf(brandJsonArray.getString(i)));
}
logger.info("[getUserGlobalBrandIds,uid is[{}], brandIds is [{}] ]", uid, results);
super.doLogInfo("[getUserGlobalBrandIds,uid is[{}], brandIds is [{}] ]", uid, results);
return results;
} catch (Exception e) {
logger.error(e.getMessage(), e);
super.doLogError(e.getMessage(), e);
return new ArrayList<Integer>();
}
}
... ... @@ -328,10 +331,10 @@ public class SortRecallSceneService extends AbstractRecallService {
for (Map<String, Object> result : results) {
brandIds.add(MapUtils.getInteger(result, "id"));
}
logger.info("[getUserLikeBrandIds,uid is[{}], brandIds is [{}] ]", uid, brandIds);
super.doLogInfo("[getUserLikeBrandIds,uid is[{}], brandIds is [{}] ]", uid, brandIds);
return brandIds;
} catch (Exception e) {
logger.error(e.getMessage(), e);
super.doLogError(e.getMessage(), e);
return new ArrayList<Integer>();
}
}
... ... @@ -384,8 +387,8 @@ public class SortRecallSceneService extends AbstractRecallService {
@Override
protected RecallResult doSketchyRank(Map<String, String> paramMap, RecallResult recallResult) {
Set<String> existProductSkns = new HashSet<String>();
List<Map<String, Object>> productList = new ArrayList<Map<String, Object>>();
for (RecallSearchResult recallSearchResult : recallResult.getRecallSearchResult()) {
logger.info("[func=doSketchyRank][type={}][results={}]", recallSearchResult.getRecallType(), recallSearchResult.getResultList().size());
Iterator<Map<String, Object>> iterator = recallSearchResult.getResultList().iterator();
while (iterator.hasNext()) {
Map<String, Object> product = iterator.next();
... ... @@ -394,27 +397,24 @@ public class SortRecallSceneService extends AbstractRecallService {
iterator.remove();
} else {
existProductSkns.add(productSkn);
product.put("recallType", recallSearchResult.getRecallType());
productList.add(product);
}
}
}
recallResult.setProductList(productList);
return recallResult;
}
@Override
protected RecallResult doCarefulRank(Map<String, String> paramMap, RecallResult recallResult) {
// 1、数据组装
List<Map<String, Object>> productList = new ArrayList<Map<String, Object>>();
for (RecallSearchResult recallSearchResult : recallResult.getRecallSearchResult()) {
logger.info("[func=doCarefulRank][type={}][results={}]", recallSearchResult.getRecallType(), recallSearchResult.getResultList().size());
for (Map<String, Object> product : recallSearchResult.getResultList()) {
product.put("recallType", recallSearchResult.getRecallType());
productList.add(product);
}
}
// 1、获取productList
List<Map<String, Object>> productList = recallResult.getProductList();
// 2、精排
// Collections.shuffle(productList);
Collections.shuffle(productList);
// 3、TODO 品牌打散
// 4、设置productList
// 4、回设productList
recallResult.setProductList(productList);
return recallResult;
}
... ... @@ -464,7 +464,6 @@ public class SortRecallSceneService extends AbstractRecallService {
int count = newProductList.size();
int maxPage = count / pageSize;
recallResult.setProductList(newProductList.subList(0, maxPage * pageSize));
logger.info("[func=doReRank][total product size is ={}]", recallResult.getProductList().size());
return recallResult;
}
... ...
... ... @@ -4,15 +4,15 @@ import java.util.List;
import java.util.Map;
public class RecallResult {
private List<RecallSearchResult> recallSearchResult;
private List<Map<String, Object>> productList;
public RecallResult(List<RecallSearchResult> recallSearchResult) {
super();
this.recallSearchResult = recallSearchResult;
}
public List<RecallSearchResult> getRecallSearchResult() {
return recallSearchResult;
}
... ... @@ -20,9 +20,20 @@ public class RecallResult {
public List<Map<String, Object>> getProductList() {
return productList;
}
public void setProductList(List<Map<String, Object>> productList) {
this.productList = productList;
}
public String doGetRecallSearchResultInfo() {
if (recallSearchResult == null) {
return "";
}
StringBuilder logInfo = new StringBuilder();
for (RecallSearchResult recallSearchResult : recallSearchResult) {
logInfo.append(String.format("[type:%s,total:%s,size:%s]", recallSearchResult.getRecallType().name(),recallSearchResult.getTotal(),recallSearchResult.getResultList().size()));
}
return logInfo.toString();
}
}
... ...