Authored by hugufei

Merge branch 'master' into zj_skninfo

package com.yoho.search.service.index;
import com.yoho.search.base.utils.ISearchConstants;
import com.yoho.search.base.utils.ProductIndexEsField;
import com.yoho.search.common.SearchCommonService;
import com.yoho.search.core.es.model.SearchParam;
import com.yoho.search.core.es.model.SearchResult;
import com.yoho.search.dal.model.SimilarSkn;
import org.apache.commons.collections.MapUtils;
import org.apache.commons.lang.StringUtils;
import org.elasticsearch.index.query.BoolQueryBuilder;
import org.elasticsearch.index.query.QueryBuilders;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Service;
import org.springframework.util.CollectionUtils;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.List;
import java.util.Map;
/**
* @author wangnan
* @version 2018/5/23
*/
@Service
public class BigDataSimilarSknImgAliIndexBaseService {
private final Logger logger = LoggerFactory.getLogger(this.getClass());
@Autowired
private SearchCommonService searchCommonService;
public SimilarSkn querySimilarSkn(String productSkn) {
try {
SearchParam searchParam = new SearchParam();
BoolQueryBuilder boolFilter = QueryBuilders.boolQuery();
boolFilter.must(QueryBuilders.termsQuery(ProductIndexEsField.productSkn, productSkn));
searchParam.setFiter(boolFilter);
searchParam.setAggregationBuilders(null);
searchParam.setSize(1);
String productIndexName = ISearchConstants.INDEX_NAME_BIGDATA_SIMILAR_SKN_IMG_ALI_INDEX;
SearchResult searchResult = searchCommonService.doSearch(productIndexName, searchParam);
if (searchResult == null) {
return null;
}
List<Map<String, Object>> similarSknResults = searchResult.getResultList();
if (CollectionUtils.isEmpty(similarSknResults) || similarSknResults.get(0) == null) {
return null;
}
Map<String, Object> result = similarSknResults.get(0);
SimilarSkn similarSkn = new SimilarSkn();
similarSkn.setDataId(MapUtils.getInteger(result, "dateId", 0));
similarSkn.setSameShopImgSimilarSkns(MapUtils.getString(result, "sameShopImgSimilarSkns", ""));
similarSkn.setDiffShopImgSimilarSkns(MapUtils.getString(result, "diffShopImgSimilarSkns", ""));
return similarSkn;
} catch (Exception e) {
logger.error(e.getMessage(), e);
return null;
}
}
public List<String> getSimilarProductSknListInShop(String productSkn) {
SimilarSkn similarSkn = this.querySimilarSkn(productSkn);
List<String> similarProductSknList = new ArrayList<>();
if (similarSkn != null && StringUtils.isNotBlank(similarSkn.getSameShopImgSimilarSkns())) {
String[] productSknArray = similarSkn.getSameShopImgSimilarSkns().split(",");
if (productSknArray.length > 0) {
similarProductSknList = Arrays.asList(productSknArray);
}
}
return similarProductSknList;
}
}
... ...
... ... @@ -4,14 +4,15 @@ import com.alibaba.fastjson.JSONArray;
import com.alibaba.fastjson.JSONObject;
import com.yoho.search.base.utils.ISearchConstants;
import com.yoho.search.base.utils.ProductIndexEsField;
import com.yoho.search.common.SearchCommonService;
import com.yoho.search.core.es.model.SearchParam;
import com.yoho.search.core.es.model.SearchResult;
import com.yoho.search.models.SearchApiResult;
import com.yoho.search.models.SearchFieldBoost;
import com.yoho.search.common.SearchCommonService;
import com.yoho.search.service.index.ProductIndexBaseService;
import com.yoho.search.service.helper.FunctionScoreSearchHelper;
import com.yoho.search.service.helper.ProductListHelper;
import com.yoho.search.service.index.ProductIndexBaseService;
import org.apache.commons.collections.CollectionUtils;
import org.apache.commons.collections.MapUtils;
import org.apache.commons.lang.StringUtils;
import org.elasticsearch.index.query.BoolQueryBuilder;
... ... @@ -26,6 +27,7 @@ import org.springframework.stereotype.Component;
import javax.annotation.PostConstruct;
import java.util.*;
import java.util.stream.Collectors;
@Component
public class SearchLikeHelper {
... ... @@ -329,6 +331,39 @@ public class SearchLikeHelper {
}
/**
* 处理图片相似+文字相似skn列表,把图片相似skn排序
*/
public List<Map<String, Object>> queryProductListWithSimilarImg(List<SearchParam> searchParams, List<String> similarProductSknList) {
List<SearchResult> searchResults = searchCommonService.doMutiSearch(ISearchConstants.INDEX_NAME_PRODUCT_INDEX, searchParams);
List<Map<String, Object>> results = new ArrayList<>();
Set<String> existProductSkns = new HashSet<>();
if (CollectionUtils.isEmpty(searchResults) || searchResults.size() != 2) {
return results;
}
SearchResult searchResultFromImg = searchResults.get(0);
List<Map<String, Object>> searchResultFromImgList = searchResultFromImg.getResultList();
if (CollectionUtils.isNotEmpty(searchResultFromImgList)) {
Map<String, Map<String, Object>> searchResultFromImgMap = searchResultFromImgList.stream().collect(Collectors.toMap(p -> MapUtils.getString(p, ProductIndexEsField.productSkn), p -> p));
for (String skn : similarProductSknList) {
if (searchResultFromImgMap.containsKey(skn)) {
results.add(searchResultFromImgMap.get(skn));
existProductSkns.add(skn);
}
}
}
SearchResult searchResultFromWord = searchResults.get(1);
if (CollectionUtils.isNotEmpty(searchResultFromWord.getResultList())) {
for (Map<String, Object> product : searchResultFromWord.getResultList()) {
if (this.isProductSknExists(existProductSkns, product)) {
continue;
}
results.add(product);
}
}
return results;
}
/**
* 店铺外按文字找相似
*
* @param productInfoInEs
... ... @@ -387,7 +422,8 @@ public class SearchLikeHelper {
boolFilter.must(QueryBuilders.termQuery(ProductIndexEsField.middleSortId, middleSortId));
} else {
boolFilter.mustNot(QueryBuilders.termQuery(ProductIndexEsField.middleSortId, middleSortId));
boolFilter.mustNot(QueryBuilders.termQuery(ProductIndexEsField.smallSort_smallSort_keyword, productInfoInEs.getString(ProductIndexEsField.smallSort)));// 有些小分类同名,要排除这种情况。
// 有些小分类同名,要排除这种情况。
boolFilter.mustNot(QueryBuilders.termQuery(ProductIndexEsField.smallSort_smallSort_keyword, productInfoInEs.getString(ProductIndexEsField.smallSort)));
}
return boolFilter;
}
... ...
package com.yoho.search.service.scene.searchlike;
import com.alibaba.fastjson.JSONObject;
import com.yoho.search.aop.cache.SearchCacheAble;
import com.yoho.search.base.utils.CollectionUtils;
import com.yoho.search.base.utils.ProductIndexEsField;
import com.yoho.search.aop.cache.SearchCacheAble;
import com.yoho.search.common.SearchRequestParams;
import com.yoho.search.core.es.model.SearchParam;
import com.yoho.search.models.SearchApiResult;
import com.yoho.search.common.SearchRequestParams;
import com.yoho.search.service.index.ProductIndexBaseService;
import com.yoho.search.service.helper.ProductListHelper;
import com.yoho.search.service.index.BigDataSimilarSknImgAliIndexBaseService;
import com.yoho.search.service.index.ProductIndexBaseService;
import org.apache.commons.lang.StringUtils;
import org.elasticsearch.index.query.BoolQueryBuilder;
import org.elasticsearch.index.query.QueryBuilder;
... ... @@ -29,114 +29,152 @@ import java.util.Map;
/**
* 店铺内找相似
*
*
* @author gufei.hu
*/
@Service
public class SearchLikeInShopService {
private static final Logger logger = LoggerFactory.getLogger(SearchLikeInShopService.class);
private static final Logger logger = LoggerFactory.getLogger(SearchLikeInShopService.class);
@Autowired
private SearchLikeHelper searchLikeHelper;
@Autowired
private ProductIndexBaseService productIndexBaseService;
@Autowired
private ProductListHelper productListHelper;
@Autowired
private BigDataSimilarSknImgAliIndexBaseService bigDataSimilarSknImgAliIndexBaseService;
/**
* 店铺内推荐
*/
@SearchCacheAble(cacheInMinute = 600, cacheName = "SEARCH_LIKE_IN_SHOP_NEW", includeParams = {"product_skn"})
public SearchApiResult searchLikeInShop(Map<String, String> paramMap) {
try {
// 1、获取参数
String productSkn = paramMap.get(SearchRequestParams.PARAM_SEARCH_PRODUCT_SKN);
if (StringUtils.isBlank(productSkn)) {
return new SearchApiResult().setCode(400).setMessage("请输入SKN");
}
// 2、检测分页参数【默认30条】
int viewNum = 30;
// 3、获取当前查询的SKN的基本信息
JSONObject productInfoInEs = searchLikeHelper.getProductInfoInEs(productSkn);
if (productInfoInEs == null) {
return new SearchApiResult().setCode(400).setMessage("SKN不存在");
}
// 4、设置SearchParams
List<SearchParam> searchParams = new ArrayList<>();
//4.1图片相似skn
List<String> similarProductSknList = bigDataSimilarSknImgAliIndexBaseService.getSimilarProductSknListInShop(productSkn);
if (similarProductSknList != null && !similarProductSknList.isEmpty()) {
searchParams.add(this.builderSearchParamForSimilarImg(productInfoInEs, similarProductSknList, viewNum));
}
//4.2文字相似skn
searchParams.add(this.builderSearchParam(productInfoInEs, Arrays.asList(productSkn), viewNum));
// 5、获取搜索结果[截取条数]
List<Map<String, Object>> tempProductList = new ArrayList<>();
//只包含文字相似
if (searchParams.size() == 1) {
tempProductList = searchLikeHelper.queryProductList(searchParams);
} else {
//包含图片+文字相似
tempProductList = searchLikeHelper.queryProductListWithSimilarImg(searchParams, similarProductSknList);
}
if (tempProductList.size() > viewNum) {
tempProductList = CollectionUtils.safeSubList(tempProductList, 0, viewNum);
}
//6、保留偶数
if (tempProductList.size() % 2 > 0) {
tempProductList = CollectionUtils.safeSubList(tempProductList, 0, tempProductList.size() - 1);
}
// 7、构造返回结果
List<Map<String, Object>> productListResults = new ArrayList<>();
if (!tempProductList.isEmpty()) {
productListResults = productListHelper.buildReturnInfoByEsSourceList(tempProductList);
}
JSONObject result = new JSONObject();
result.put("product_info", searchLikeHelper.genProductInfoResult(productInfoInEs));
result.put("product_list", productListResults);
return new SearchApiResult().setData(result);
} catch (Exception e) {
logger.error(e.getMessage(), e);
return new SearchApiResult().setData(null).setMessage("searchLikeInShop Exception").setCode(500);
}
}
@Autowired
private SearchLikeHelper searchLikeHelper;
@Autowired
private ProductIndexBaseService productIndexBaseService;
@Autowired
private ProductListHelper productListHelper;
/**
* 店铺内推荐
*
* @param paramMap
* @return
*/
@SearchCacheAble(cacheInMinute = 600, cacheName = "SEARCH_LIKE_IN_SHOP_NEW", includeParams = { "product_skn"})
public SearchApiResult searchLikeInShop(Map<String, String> paramMap) {
try {
// 1、获取参数
String productSkn = paramMap.get(SearchRequestParams.PARAM_SEARCH_PRODUCT_SKN);
if (StringUtils.isBlank(productSkn)) {
return new SearchApiResult().setCode(400).setMessage("请输入SKN");
}
// 2、检测分页参数【默认30条】
int viewNum = 30;
// 3、获取当前查询的SKN的基本信息
JSONObject productInfoInEs = searchLikeHelper.getProductInfoInEs(productSkn);
if (productInfoInEs == null) {
return new SearchApiResult().setCode(400).setMessage("SKN不存在");
}
// 4、设置SearchParams
List<SearchParam> searchParams = new ArrayList<SearchParam>();
searchParams.add(this.builderSearchParam(productInfoInEs, Arrays.asList(productSkn), viewNum));
// 5、获取搜索结果[截取条数]
List<Map<String, Object>> tempProductList = searchLikeHelper.queryProductList(searchParams);
if (tempProductList.size() > viewNum) {
tempProductList = CollectionUtils.safeSubList(tempProductList,0, viewNum);
}
//6、保留偶数
if (tempProductList.size() % 2 > 0) {
tempProductList = CollectionUtils.safeSubList(tempProductList, 0, tempProductList.size() - 1);
}
// 7、构造返回结果
List<Map<String, Object>> productListResults = new ArrayList<Map<String, Object>>();
if (!tempProductList.isEmpty()) {
productListResults = productListHelper.buildReturnInfoByEsSourceList(tempProductList);
}
JSONObject result = new JSONObject();
result.put("product_info", searchLikeHelper.genProductInfoResult(productInfoInEs));
result.put("product_list", productListResults);
return new SearchApiResult().setData(result);
} catch (Exception e) {
logger.error(e.getMessage(), e);
return new SearchApiResult().setData(null).setMessage("searchLikeInShop Exception").setCode(500);
}
}
private SearchParam builderSearchParam(JSONObject productInfoInEs, List<String> productSkns, int pageSize) {
// 1、设置SearchParam
SearchParam searchParam = new SearchParam();
// 2)设置query和filter
searchParam.setQuery(this.builderQueryBuilder(productInfoInEs, "20%"));
searchParam.setFiter(this.builderFilterBuilder(productInfoInEs, productSkns));
// 3、设置排序规则[按打分排序]
List<SortBuilder<?>> sortBuilders = new ArrayList<SortBuilder<?>>();
sortBuilders.add(SortBuilders.scoreSort().order(SortOrder.DESC));
searchParam.setSortBuilders(sortBuilders);
// 4、设置分页参数
searchParam.setOffset(0);
searchParam.setSize(pageSize);
// 5)设置返回的参数【节省带宽】
List<String> includeFields = productIndexBaseService.getProductIndexIncludeFields();
searchParam.setIncludeFields(includeFields);
return searchParam;
}
private SearchParam builderSearchParam(JSONObject productInfoInEs, List<String> productSkns, int pageSize) {
// 1、设置SearchParam
SearchParam searchParam = new SearchParam();
// 2)设置query和filter
searchParam.setQuery(this.builderQueryBuilder(productInfoInEs, "20%"));
searchParam.setFiter(this.builderFilterBuilder(productInfoInEs, productSkns));
// 3、设置排序规则[按打分排序]
List<SortBuilder<?>> sortBuilders = new ArrayList<SortBuilder<?>>();
sortBuilders.add(SortBuilders.scoreSort().order(SortOrder.DESC));
searchParam.setSortBuilders(sortBuilders);
// 4、设置分页参数
searchParam.setOffset(0);
searchParam.setSize(pageSize);
// 5)设置返回的参数【节省带宽】
List<String> includeFields = productIndexBaseService.getProductIndexIncludeFields();
searchParam.setIncludeFields(includeFields);
return searchParam;
}
private QueryBuilder builderFilterBuilder(JSONObject productInfoInEs, List<String> notProductSkns) {
String isGlobalInEs = productInfoInEs.getString(ProductIndexEsField.isGlobal);
boolean isGlobal = "Y".equalsIgnoreCase(isGlobalInEs);
BoolQueryBuilder boolFilter = searchLikeHelper.genDefaultSearchLikeFilter(notProductSkns, isGlobal);
// 1)设置此SKN相关的性别过滤条件
String gender = productInfoInEs.getString(ProductIndexEsField.gender);
List<String> genderList = searchLikeHelper.getGenderInfo(gender);
if (genderList != null && !genderList.isEmpty()) {
boolFilter.must(QueryBuilders.termsQuery(ProductIndexEsField.gender, genderList));
}
// 2)设置品牌或店铺信息
Integer brandId = productInfoInEs.getInteger(ProductIndexEsField.brandId);
Integer shopId = productInfoInEs.getInteger(ProductIndexEsField.shopId);
if (searchLikeHelper.isLegalInteger(shopId)) {
boolFilter.must(QueryBuilders.termQuery(ProductIndexEsField.shopId, shopId));
} else if (searchLikeHelper.isLegalInteger(brandId)) {
boolFilter.must(QueryBuilders.termQuery(ProductIndexEsField.brandId, brandId));
}
return boolFilter;
}
private QueryBuilder builderFilterBuilder(JSONObject productInfoInEs, List<String> notProductSkns) {
String isGlobalInEs = productInfoInEs.getString(ProductIndexEsField.isGlobal);
boolean isGlobal = "Y".equalsIgnoreCase(isGlobalInEs);
BoolQueryBuilder boolFilter = searchLikeHelper.genDefaultSearchLikeFilter(notProductSkns, isGlobal);
// 1)设置此SKN相关的性别过滤条件
String gender = productInfoInEs.getString(ProductIndexEsField.gender);
List<String> genderList = searchLikeHelper.getGenderInfo(gender);
if (genderList != null && !genderList.isEmpty()) {
boolFilter.must(QueryBuilders.termsQuery(ProductIndexEsField.gender, genderList));
}
// 2)设置品牌或店铺信息
Integer brandId = productInfoInEs.getInteger(ProductIndexEsField.brandId);
Integer shopId = productInfoInEs.getInteger(ProductIndexEsField.shopId);
if (searchLikeHelper.isLegalInteger(shopId)) {
boolFilter.must(QueryBuilders.termQuery(ProductIndexEsField.shopId, shopId));
} else if (searchLikeHelper.isLegalInteger(brandId)) {
boolFilter.must(QueryBuilders.termQuery(ProductIndexEsField.brandId, brandId));
}
return boolFilter;
}
private QueryBuilder builderQueryBuilder(JSONObject productInfoInEs, String minimumShouldMatch) {
String queryString = searchLikeHelper.genYohoQueryStringWithBrandName(productInfoInEs);
String productFeatureFactor = productInfoInEs.getString(ProductIndexEsField.productFeatureFactor);
QueryBuilder queryBuilder = searchLikeHelper.genSearchLikeQueryBuilder(queryString, minimumShouldMatch, productFeatureFactor);
return queryBuilder;
}
private QueryBuilder builderQueryBuilder(JSONObject productInfoInEs, String minimumShouldMatch) {
String queryString = searchLikeHelper.genYohoQueryStringWithBrandName(productInfoInEs);
String productFeatureFactor = productInfoInEs.getString(ProductIndexEsField.productFeatureFactor);
QueryBuilder queryBuilder = searchLikeHelper.genSearchLikeQueryBuilder(queryString, minimumShouldMatch, productFeatureFactor);
return queryBuilder;
}
/**
* 根据阿里相似图片的skn构建SearchParam去查productindex
*/
private SearchParam builderSearchParamForSimilarImg(JSONObject productInfoInEs, List<String> productSkns, int pageSize) {
// 1、设置SearchParam
SearchParam searchParam = new SearchParam();
// 2)设置filter
String isGlobalInEs = productInfoInEs.getString(ProductIndexEsField.isGlobal);
boolean isGlobal = "Y".equalsIgnoreCase(isGlobalInEs);
BoolQueryBuilder boolFilter = searchLikeHelper.genDefaultSearchLikeFilter(null, isGlobal);
boolFilter.must(QueryBuilders.termsQuery(ProductIndexEsField.productSkn, productSkns));
String gender = productInfoInEs.getString(ProductIndexEsField.gender);
List<String> genderList = searchLikeHelper.getGenderInfo(gender);
if (genderList != null && !genderList.isEmpty()) {
boolFilter.must(QueryBuilders.termsQuery(ProductIndexEsField.gender, genderList));
}
searchParam.setFiter(boolFilter);
// 4、设置分页参数
searchParam.setOffset(0);
searchParam.setSize(pageSize);
// 5)设置返回的参数【节省带宽】
List<String> includeFields = productIndexBaseService.getProductIndexIncludeFields();
searchParam.setIncludeFields(includeFields);
return searchParam;
}
}
... ...