Authored by hugufei

模糊搜索支持资源位

... ... @@ -3,6 +3,7 @@ package com.yoho.search.service.helper;
import com.alibaba.fastjson.JSONObject;
import com.yoho.search.aop.productlist.ProductListWithSkn;
import com.yoho.search.base.utils.ProductIndexEsField;
import com.yoho.search.common.SearchRequestParams;
import com.yoho.search.core.es.model.SearchParam;
import com.yoho.search.core.es.model.SearchResult;
import com.yoho.search.service.index.*;
... ... @@ -10,7 +11,6 @@ import com.yoho.search.service.index.promotion.PromotionIndexBaseService;
import com.yoho.search.service.index.promotion.PromotionPriceService;
import com.yoho.search.service.recall.strategy.NotRecallTypeEnum;
import org.apache.commons.collections.MapUtils;
import org.apache.commons.lang.StringUtils;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Component;
... ... @@ -60,19 +60,19 @@ public class ProductListHelper {
private SearchParam innerBuildProductListSearchParam(Map<String, String> paramMap, boolean needPersional, boolean containPhrase) throws Exception {
// 1)验证查询条数
int pageSize = StringUtils.isBlank(paramMap.get("viewNum")) ? 10 : Integer.parseInt(paramMap.get("viewNum"));
int page = StringUtils.isBlank(paramMap.get("page")) ? 1 : Integer.parseInt(paramMap.get("page"));
if (page < 1 || pageSize < 0) {
int page = MapUtils.getIntValue(paramMap,SearchRequestParams.PARAM_SEARCH_PAGE,1);
int viewNum = MapUtils.getIntValue(paramMap, SearchRequestParams.PARAM_SEARCH_VIEWNUM,20);
if (page < 1 || viewNum < 0) {
throw new IllegalArgumentException("分页参数不合法");
}
if (pageSize > 100) {
pageSize = 100;
if (viewNum > 100) {
viewNum = 100;
}
// 2)构建基本查询参数
SearchParam searchParam = searchParamHelper.buildWithPersional(paramMap, needPersional);
searchParam.setAggregationBuilders(null);
searchParam.setSize(pageSize);
searchParam.setOffset((page - 1) * pageSize);
searchParam.setSize(viewNum);
searchParam.setOffset((page - 1) * viewNum);
// 3)设置排序字段
searchParam.setSortBuilders(searchSortHelper.buildSortList(paramMap));
// 4)设置返回的结果
... ...
package com.yoho.search.service.index;
import com.google.common.cache.CacheBuilder;
import com.google.common.cache.CacheLoader;
import com.google.common.cache.LoadingCache;
import com.yoho.search.base.utils.DateUtil;
import com.yoho.search.base.utils.ISearchConstants;
import com.yoho.search.common.SearchCommonService;
import com.yoho.search.common.utils.ImageUrlAssist;
import com.yoho.search.core.es.model.SearchParam;
import com.yoho.search.core.es.model.SearchResult;
import org.apache.commons.collections.CollectionUtils;
import org.apache.commons.collections.MapUtils;
import org.elasticsearch.index.query.BoolQueryBuilder;
import org.elasticsearch.index.query.QueryBuilders;
import org.elasticsearch.search.sort.SortBuilder;
import org.elasticsearch.search.sort.SortBuilders;
import org.elasticsearch.search.sort.SortOrder;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Service;
import java.util.*;
import java.util.concurrent.TimeUnit;
import java.util.stream.Collectors;
@Service
public class CsSearchResourceFuzzyIndexBaseService {
private final Logger logger = LoggerFactory.getLogger(this.getClass());
@Autowired
private SearchCommonService searchCommonService;
//Guava Cache
private LoadingCache<String, List<Map<String, Object>>> fuzzySearchResourceCache = CacheBuilder.newBuilder()
.maximumSize(100).expireAfterWrite(3, TimeUnit.MINUTES).build(new CacheLoader<String, List<Map<String, Object>>>() {
public List<Map<String, Object>> load(String key) {
return queryValidFuzzySearchResourceFromEs(1000);
}
});
private List<Map<String, Object>> queryValidFuzzySearchResourceFromEs(int limit) {
try {
SearchParam searchParam = new SearchParam();
//1、设置filter
BoolQueryBuilder filter = QueryBuilders.boolQuery();
long current = DateUtil.getCurrentTimeSecond();
filter.must(QueryBuilders.rangeQuery("beginTime").lte(current));
filter.must(QueryBuilders.rangeQuery("endTime").gte(current));
searchParam.setFiter(filter);
//2、设置分页参数
searchParam.setOffset(0);
searchParam.setSize(limit);
//3、设置排序字段
List<SortBuilder<?>> sortBuilder = new ArrayList<>();
sortBuilder.add(SortBuilders.fieldSort("orderBy").order(SortOrder.DESC));
sortBuilder.add(SortBuilders.fieldSort("id").order(SortOrder.DESC));
searchParam.setSortBuilders(sortBuilder);
//4、执行搜索
SearchResult searchResult = searchCommonService.doSearch(ISearchConstants.INDEX_NAME_CS_SEARCH_RESOURCE_FUZZY, searchParam);
//5、构造返回结果
List<Map<String, Object>> results = searchResult.getResultList();
if (CollectionUtils.isEmpty(results)) {
return new ArrayList<>();
}
List<Map<String, Object>> resultList = new ArrayList<>();
for (Map<String, Object> esMap : results) {
resultList.add(this.getResourceMap(esMap));
}
return resultList;
} catch (Exception e) {
logger.error(e.getMessage());
return new ArrayList<>();
}
}
private Map<String, Object> getResourceMap(Map<String, Object> esMap) {
Map<String, Object> map = new HashMap<>();
map.put("id", MapUtils.getIntValue(esMap, "id"));
map.put("keyword", MapUtils.getString(esMap, "keyword",""));
map.put("link_url", MapUtils.getString(esMap, "linkUrl", ""));
String image = ImageUrlAssist.getAllProductPicUrl(MapUtils.getString(esMap, "image", ""), "goodsimg", "center", "d2hpdGU=");
map.put("image", image);
return map;
}
public List<Map<String, Object>> queryFuzzySearchResourcesByKeyWord(String keyword){
try {
List<Map<String, Object>> cacheResult = fuzzySearchResourceCache.get("DEFAULT");
return cacheResult.stream().filter(resource->{
if(MapUtils.getString(resource,"keyword").equalsIgnoreCase(keyword)){
return true;
}else{
return false;
}
}).collect(Collectors.toList());
} catch (Exception e) {
logger.error(e.getMessage(), e);
return new ArrayList<>(0);
}
}
}
... ...
... ... @@ -4,8 +4,11 @@ import com.alibaba.fastjson.JSONObject;
import com.yoho.search.base.utils.ConvertUtils;
import com.yoho.search.base.utils.SearchCollectionUtils;
import com.yoho.search.common.SearchDynamicConfigService;
import com.yoho.search.common.SearchRequestParams;
import com.yoho.search.common.utils.ABUserPartitionUtils;
import com.yoho.search.core.es.model.SearchParam;
import com.yoho.search.service.helper.SearchCommonHelper;
import com.yoho.search.service.index.CsSearchResourceFuzzyIndexBaseService;
import com.yoho.search.service.index.CsSearchResourceIndexBaseService;
import org.apache.commons.collections.MapUtils;
import org.slf4j.Logger;
... ... @@ -15,7 +18,6 @@ import org.springframework.stereotype.Component;
import org.springframework.util.CollectionUtils;
import java.util.ArrayList;
import java.util.Collections;
import java.util.List;
import java.util.Map;
import java.util.stream.Collectors;
... ... @@ -28,6 +30,9 @@ public class CsSearchResourceService {
@Autowired
private CsSearchResourceIndexBaseService csSearchResourceIndexBaseService;
@Autowired
private CsSearchResourceFuzzyIndexBaseService searchResourceFuzzyIndexBaseService;
@Autowired
private SearchCommonHelper searchCommonHelper;
@Autowired
private ABUserPartitionUtils abUserPartitionUtils;
... ... @@ -119,8 +124,8 @@ public class CsSearchResourceService {
return new ArrayList<>();
}
int[] index = new int[countPerSize];
index[0] = viewNum / 4;
index[1] = viewNum * 3 / 4;
index[0] = viewNum * 1/4;
index[1] = viewNum * 3/4;
for (int i = 0; i < subResourceList.size(); i++) {
subResourceList.get(i).put("index", index[i]);
}
... ... @@ -129,4 +134,10 @@ public class CsSearchResourceService {
return subResourceList;
}
//获取全部的资源位直通车
public List<Map<String, Object>> queryAllFuzzySearchResource(Map<String, String> paramMap) {
String keyword = MapUtils.getString(paramMap, SearchRequestParams.PARAM_SEARCH_QUERY,"");
return searchResourceFuzzyIndexBaseService.queryFuzzySearchResourcesByKeyWord(keyword);
}
}
... ...
... ... @@ -2,6 +2,7 @@ package com.yoho.search.service.scene.pages;
import com.alibaba.fastjson.JSONArray;
import com.alibaba.fastjson.JSONObject;
import com.yoho.search.base.utils.JsonUtil;
import com.yoho.search.base.utils.SearchCollectionUtils;
import com.yoho.search.base.utils.SearchPageIdDefine;
import com.yoho.search.common.SearchDynamicConfigService;
... ... @@ -10,11 +11,12 @@ import com.yoho.search.common.utils.SearchApiResultUtils;
import com.yoho.search.models.SearchApiResult;
import com.yoho.search.service.helper.SearchCommonHelper;
import com.yoho.search.service.helper.SearchKeyWordHelper;
import com.yoho.search.service.scene.general.CsSearchResourceService;
import com.yoho.search.service.scene.pages.entrance.ProductListSwitchService;
import com.yoho.search.service.scene.pages.selections.PageAggregationHelper;
import com.yoho.search.service.scene.pages.selections.PageSelectionsBrandsService;
import com.yoho.search.service.scene.pages.selections.PageSelectionsService;
import com.yoho.search.service.scene.shopbrand.ShopListService;
import com.yoho.search.service.scene.shopbrand.ShopListFuzzyService;
import com.yoho.search.service.scene.suggest.RecommendWordsService;
import org.apache.commons.collections.MapUtils;
import org.apache.commons.lang.StringUtils;
... ... @@ -22,9 +24,11 @@ import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Service;
import org.springframework.util.CollectionUtils;
import java.util.*;
import java.util.ArrayList;
import java.util.Collections;
import java.util.List;
import java.util.Map;
import java.util.concurrent.CompletableFuture;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
... ... @@ -52,7 +56,10 @@ public class FuzzySceneService extends AbstractPageSceneService {
@Autowired
private ProductListSwitchService productListSwitchService;
@Autowired
private ShopListService shopListService;
private ShopListFuzzyService shopListFuzzyService;
@Autowired
private CsSearchResourceService csSearchResourceService;
private ExecutorService executor = Executors.newFixedThreadPool(20);
... ... @@ -82,6 +89,7 @@ public class FuzzySceneService extends AbstractPageSceneService {
if (StringUtils.isBlank(paramMap.get(SearchRequestParams.PARAM_SEARCH_QUERY))) {
return new SearchApiResult().setCode(400).setMessage("请传query参数");
}
// 2、添加默认参数
this.addParamsToParamMap(paramMap);
... ... @@ -94,8 +102,8 @@ public class FuzzySceneService extends AbstractPageSceneService {
// 5、获取促销专题
//CompletableFuture<SearchApiResult> promotionsFuture = CompletableFuture.supplyAsync(() -> pageAggregationHelper.sceneAggPromotion(this.newParamMap(paramMap)), executor);
// 6、获取店铺结果
CompletableFuture<SearchApiResult> shopsFuture = CompletableFuture.supplyAsync(() -> shopListService.searchShopList(convertParamMap(paramMap)), executor);
// 6、获取店铺查询结果
CompletableFuture<JSONObject> shopsFuture = CompletableFuture.supplyAsync(() -> shopListFuzzyService.queryShopListForFuzzyScene(this.newParamMap(paramMap)), executor);
// 7、加入建议词
SearchApiResult productListResult = productListuture.get();
... ... @@ -116,129 +124,87 @@ public class FuzzySceneService extends AbstractPageSceneService {
//SearchApiResult promotions = promotionsFuture.get();
//dataMap.put(RECOMMEND_PROMOTION_LIST, pageAggregationHelper.subRecommendPromotions(promotions.getData(), this.getPage(paramMap), 1));
// 11、处理店铺结果
SearchApiResult shops = shopsFuture.get();
dealShopListResult(shops, dataMap, paramMap);
// 11、获取全部的资源位直通车
List<Map<String, Object>> allFuzzySearchResource = csSearchResourceService.queryAllFuzzySearchResource(paramMap);
// 12、返回最终结果
// 12、处理资源位和店铺分页
dealPageData(paramMap, allFuzzySearchResource, shopsFuture, dataMap);
// 13、返回最终结果
return productListResult;
} catch (Exception e) {
return SearchApiResultUtils.errorSearchApiResult(logger, paramMap, e);
}
}
private Map<String, String> convertParamMap(Map<String, String> paramMap) {
String query = paramMap.get(SearchRequestParams.PARAM_SEARCH_QUERY);
Map<String, String> newMap = new HashMap<>(paramMap);
newMap.put(SearchRequestParams.PARAM_SEARCH_SHOPS_KEYWORD, query);
return newMap;
}
/**
* 处理shopList节点
*
* @param shops
* @param dataMap
* @param paramMap
*/
private void dealShopListResult(SearchApiResult shops, JSONObject dataMap, Map<String, String> paramMap) {
// 1、参数检测
dataMap.put("shop_list", Collections.emptyList());
dataMap.put("shop_list_page", Collections.emptyList());
if (shops == null) {
return;
}
JSONObject jsonObject = (JSONObject) shops.getData();
if (jsonObject == null) {
return;
}
JSONArray jsonArray = jsonObject.getJSONArray("shop_list");
if (CollectionUtils.isEmpty(jsonArray)) {
return;
}
// 2、第一个数据或ufo数据放shop_list节点,放在前面
List<JSONObject> tempFirstShopList = new ArrayList<>();
List<JSONObject> tempOtherShopList = new ArrayList<>();
for (int i = 0; i < jsonArray.size(); i++) {
JSONObject subJsonObject = jsonArray.getJSONObject(i);
if (i == 0 || MapUtils.getObject(subJsonObject, "ufo_brand") != null) {
tempFirstShopList.add(subJsonObject);
} else {
tempOtherShopList.add(subJsonObject);
}
}
// 3、处理realFirstShops: 将tempFirstShopList中有search_show_image的节点放前面,如果都有,则顺序不变
List<JSONObject> realFirstShops = new ArrayList<>();
Iterator<JSONObject> iterator = tempFirstShopList.iterator();
while (iterator.hasNext()) {
JSONObject shopInfo = iterator.next();
JSONObject tbl_brand = shopInfo.getJSONObject("tbl_brand");
JSONObject ufo_brand = shopInfo.getJSONObject("ufo_brand");
JSONObject yoho_shop = shopInfo.getJSONObject("yoho_shop");
if (checkSearchShowImage(tbl_brand) || checkSearchShowImage(ufo_brand) || checkSearchShowImage(yoho_shop)) {
realFirstShops.add(shopInfo);
iterator.remove();
// 12、处理资源位和店铺分页
private void dealPageData(Map<String, String> paramMap, List<Map<String, Object>> allFuzzySearchResource, CompletableFuture<JSONObject> shopsFuture, JSONObject dataMap) {
try {
// 1、源数据获取
JSONObject shopsResult = shopsFuture.get();
JSONArray shop_list_page = shopsResult.getJSONArray("shop_list_page");
JSONArray search_resource_list = JsonUtil.listToJsonArray(allFuzzySearchResource);
// 2、数据整合,优先资源位数据,再店铺数据
List<JSONObject> dataList = new ArrayList<>();
for (int i = 0; i < search_resource_list.size(); i++) {
JSONObject resource = search_resource_list.getJSONObject(i);
resource.put("dataJoinType", "resource");
dataList.add(resource);
}
}
realFirstShops.addAll(tempFirstShopList);
// 4、处理realFirstShops: 将realFirstShops中,除了第一个节点的search_show_image的节点,全部清除
for(int i=0;i<realFirstShops.size();i++){
if(i==0){
continue;
for (int i = 0; i < shop_list_page.size(); i++) {
JSONObject shop = shop_list_page.getJSONObject(i);
shop.put("dataJoinType", "shop");
dataList.add(shop);
}
JSONObject shopInfo = realFirstShops.get(i);
JSONObject tbl_brand = shopInfo.getJSONObject("tbl_brand");
JSONObject ufo_brand = shopInfo.getJSONObject("ufo_brand");
JSONObject yoho_shop = shopInfo.getJSONObject("yoho_shop");
cleanSearchShowImage(tbl_brand);
cleanSearchShowImage(ufo_brand);
cleanSearchShowImage(yoho_shop);
}
dataMap.put("shop_list", realFirstShops);
// 5、处理分页节点,一页放两个
if (CollectionUtils.isEmpty(tempOtherShopList)) {
return;
}
int page = MapUtils.getIntValue(paramMap,"page",1);
int pageSize = 2;
List<JSONObject> subShopList = SearchCollectionUtils.safeSubList(tempOtherShopList, (page - 1) * pageSize, page * pageSize);
if (CollectionUtils.isEmpty(subShopList)) {
return;
}
int viewNum = MapUtils.getIntValue(paramMap,"viewNum",10);
if (viewNum > 100) {
viewNum = 100;
}
int[] index = new int[pageSize];
index[0] = viewNum / 4;
index[1] = viewNum * 3 / 4;
for (int i = 0; i < subShopList.size(); i++) {
subShopList.get(i).put("index", index[i]);
}
List productList = (List) dataMap.get("product_list");
int productListSize = productList == null ? 0 : productList.size();
subShopList = subShopList.stream().filter(e -> e.getIntValue("index") < productListSize).collect(Collectors.toList());
dataMap.put("shop_list_page", subShopList);
}
// 3、得到分页数据
int countPerSize = 2;
int page = MapUtils.getIntValue(paramMap, "page", 1);
List<JSONObject> pageDataList = SearchCollectionUtils.safeSubList(dataList, (page - 1) * countPerSize, page * countPerSize);
private boolean checkSearchShowImage(JSONObject data) {
if (data != null && StringUtils.isNotBlank(MapUtils.getString(data, "search_show_image", ""))) {
return true;
}
return false;
}
// 4、处理数据索引
int viewNum = MapUtils.getIntValue(paramMap, "viewNum", 20);
if (viewNum > 100) {
viewNum = 100;
}
int[] index = new int[countPerSize];
index[0] = viewNum * 1 / 4;
index[1] = viewNum * 3 / 4;
for (int i = 0; i < pageDataList.size(); i++) {
pageDataList.get(i).put("index", index[i]);
}
List productList = (List) dataMap.get("product_list");
int productListSize = productList == null ? 0 : productList.size();
pageDataList = pageDataList.stream().filter(e -> e.getIntValue("index") < productListSize).collect(Collectors.toList());
//5、处理结果
List<JSONObject> search_resource_list1 = new ArrayList<>();
List<JSONObject> shop_list_page1 = new ArrayList<>();
pageDataList.forEach(data -> {
String dataJoinType = MapUtils.getString(data, "dataJoinType", "");
if (dataJoinType.equalsIgnoreCase("resource")) {
search_resource_list1.add(data);
}
if (dataJoinType.equalsIgnoreCase("shop")) {
shop_list_page1.add(data);
}
data.remove("dataJoinType");
});
//6、加入返回结果
dataMap.put("shop_list", shopsResult.getJSONArray("shop_list"));
dataMap.put("shop_list_page", shop_list_page1);
dataMap.put("search_resource_list", search_resource_list1);
private void cleanSearchShowImage(JSONObject data) {
if (data != null) {
data.put("search_show_image","");
} catch (Exception e) {
logger.error(e.getMessage(), e);
dataMap.put("shop_list", Collections.emptyList());
dataMap.put("shop_list_page", Collections.emptyList());
dataMap.put("search_resource_list", Collections.emptyList());
}
}
@Override
public SearchApiResult aggregations(Map<String, String> paramMap) {
try {
... ...
package com.yoho.search.service.scene.shopbrand;
import com.alibaba.fastjson.JSONArray;
import com.alibaba.fastjson.JSONObject;
import com.yoho.search.common.SearchRequestParams;
import com.yoho.search.models.SearchApiResult;
import org.apache.commons.collections.MapUtils;
import org.apache.commons.lang.StringUtils;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Service;
import org.springframework.util.CollectionUtils;
import java.util.*;
@Service
public class ShopListFuzzyService {
@Autowired
private ShopListService shopListService;
/**
* 模糊搜索列表查询店铺信息,并处理好头结点
*
* @param paramMap
* @return
*/
public JSONObject queryShopListForFuzzyScene(Map<String, String> paramMap) {
//1、数据转换,调shopListService接口查询
String query = MapUtils.getString(paramMap, SearchRequestParams.PARAM_SEARCH_QUERY, "");
paramMap.put(SearchRequestParams.PARAM_SEARCH_SHOPS_KEYWORD, query);
SearchApiResult shops = shopListService.searchShopList(paramMap);
//2、构造默认的返回结果
JSONObject result = new JSONObject();
result.put("shop_list", Collections.emptyList());
result.put("shop_list_page", Collections.emptyList());
//3、参数检测
if (shops == null) {
return result;
}
JSONObject jsonObject = (JSONObject) shops.getData();
if (jsonObject == null) {
return result;
}
JSONArray jsonArray = jsonObject.getJSONArray("shop_list");
if (CollectionUtils.isEmpty(jsonArray)) {
return result;
}
// 4、第一个数据或ufo数据放shop_list节点,即放在前面
List<JSONObject> topShopListTemp = new ArrayList<>();
List<JSONObject> pageShopList = new ArrayList<>();
for (int i = 0; i < jsonArray.size(); i++) {
JSONObject subJsonObject = jsonArray.getJSONObject(i);
if (i == 0 || MapUtils.getObject(subJsonObject, "ufo_brand") != null) {
topShopListTemp.add(subJsonObject);
} else {
pageShopList.add(subJsonObject);
}
}
// 5、处理realFirstShops: 将tempFirstShopList中有search_show_image的节点放前面,如果都有,则顺序不变
List<JSONObject> topShopList = new ArrayList<>();
Iterator<JSONObject> iterator = topShopListTemp.iterator();
while (iterator.hasNext()) {
JSONObject shopInfo = iterator.next();
JSONObject tbl_brand = shopInfo.getJSONObject("tbl_brand");
JSONObject ufo_brand = shopInfo.getJSONObject("ufo_brand");
JSONObject yoho_shop = shopInfo.getJSONObject("yoho_shop");
if (checkSearchShowImage(tbl_brand) || checkSearchShowImage(ufo_brand) || checkSearchShowImage(yoho_shop)) {
topShopList.add(shopInfo);
iterator.remove();
}
}
topShopList.addAll(topShopListTemp);
// 6、处理topShopList: 将topShopList中,除了第一个节点的search_show_image的节点,全部清除
for (int i = 0; i < topShopList.size(); i++) {
if (i == 0) {
continue;
}
JSONObject shopInfo = topShopList.get(i);
JSONObject tbl_brand = shopInfo.getJSONObject("tbl_brand");
JSONObject ufo_brand = shopInfo.getJSONObject("ufo_brand");
JSONObject yoho_shop = shopInfo.getJSONObject("yoho_shop");
cleanSearchShowImage(tbl_brand);
cleanSearchShowImage(ufo_brand);
cleanSearchShowImage(yoho_shop);
}
// 7、返回结果
result.put("shop_list", topShopList);
result.put("shop_list_page", pageShopList);
return result;
}
private boolean checkSearchShowImage(JSONObject data) {
if (data != null && StringUtils.isNotBlank(MapUtils.getString(data, "search_show_image", ""))) {
return true;
}
return false;
}
private void cleanSearchShowImage(JSONObject data) {
if (data != null) {
data.put("search_show_image", "");
}
}
}
... ...