Merge branch 'master' into zj_groupshop

@@ -21,7 +21,7 @@ import org.springframework.stereotype.Component;
 import org.springframework.util.Assert;
 import org.springframework.util.Assert;
 
 
 import com.yoho.search.base.utils.SearchPageIdDefine;
 import com.yoho.search.base.utils.SearchPageIdDefine;
-import com.yoho.search.core.personalized.PersonalizedSearch;
+import com.yoho.search.core.personalized.models.PersonalizedSearch;
 import com.yoho.search.models.FirstShelveTimeScore;
 import com.yoho.search.models.FirstShelveTimeScore;
 import com.yoho.search.service.scorer.YohoFilterFunctionBuilders;
 import com.yoho.search.service.scorer.YohoFilterFunctionBuilders;
 import com.yoho.search.common.SearchDynamicConfigService;
 import com.yoho.search.common.SearchDynamicConfigService;

@@ -2,7 +2,7 @@ package com.yoho.search.service.recall.beans.builder;
 
 
 import com.yoho.search.base.utils.CollectionUtils;
 import com.yoho.search.base.utils.CollectionUtils;
 import com.yoho.search.base.utils.Transfer;
 import com.yoho.search.base.utils.Transfer;
-import com.yoho.search.core.personalized.PersonalizedSearch;
+import com.yoho.search.core.personalized.models.PersonalizedSearch;
 import com.yoho.search.core.personalized.models.SortPriceAreas;
 import com.yoho.search.core.personalized.models.SortPriceAreas;
 import com.yoho.search.service.recall.beans.persional.ProductFeatureFactorComponent;
 import com.yoho.search.service.recall.beans.persional.ProductFeatureFactorComponent;
 import com.yoho.search.service.recall.config.RecallConfigConstants;
 import com.yoho.search.service.recall.config.RecallConfigConstants;

 package com.yoho.search.service.recall.beans.persional;
 package com.yoho.search.service.recall.beans.persional;
 
 
 import com.yoho.search.base.helper.Word2VectorCalculator;
 import com.yoho.search.base.helper.Word2VectorCalculator;
-import com.yoho.search.core.personalized.PersonalizedSearch;
+import com.yoho.search.core.personalized.models.PersonalizedSearch;
 import com.yoho.search.service.recall.models.personal.UserFeatureFactor;
 import com.yoho.search.service.recall.models.personal.UserFeatureFactor;
 import org.apache.commons.lang.StringUtils;
 import org.apache.commons.lang.StringUtils;
 import org.slf4j.Logger;
 import org.slf4j.Logger;

 package com.yoho.search.service.recall.beans.persional;
 package com.yoho.search.service.recall.beans.persional;
 
 
-import com.alibaba.fastjson.JSON;
-import com.alibaba.fastjson.JSONObject;
-import com.yoho.core.rest.client.ServiceCaller;
-import com.yoho.search.core.personalized.models.*;
+import com.yoho.search.core.personalized.models.UserPersonalFactorRspNew;
+import com.yoho.search.core.personalized.service.BidataServiceCaller;
 import org.slf4j.Logger;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 import org.slf4j.LoggerFactory;
 import org.springframework.beans.factory.annotation.Autowired;
 import org.springframework.beans.factory.annotation.Autowired;
@@ -16,11 +14,9 @@ import java.util.List;
 public class UserPersionalFactorComponent {
 public class UserPersionalFactorComponent {
 
 
     private static final Logger RECALL_NEW_LOGGER = LoggerFactory.getLogger("RECALL");
     private static final Logger RECALL_NEW_LOGGER = LoggerFactory.getLogger("RECALL");
-    private static final String SERVICE_NAME = "bigdata.searchPersonalFactors";
-    private static final int timeOut = 100;
 
 
     @Autowired
     @Autowired
-    private ServiceCaller serviceCaller;
+    private BidataServiceCaller bidataServiceCaller;
 
 
     /**
     /**
      * 获取用户的个性化因子
      * 获取用户的个性化因子
@@ -31,10 +27,7 @@ public class UserPersionalFactorComponent {
      */
      */
     public UserPersonalFactorRspNew queryUserPersionalFactor(int uid, String udid, List<Integer> misortIds) {
     public UserPersonalFactorRspNew queryUserPersionalFactor(int uid, String udid, List<Integer> misortIds) {
         try {
         try {
-            UserPersonalFactorReq userPersionalFactorReq = new UserPersonalFactorReq(uid, udid, misortIds);
-            JSONObject result = serviceCaller.call(SERVICE_NAME, userPersionalFactorReq, JSONObject.class, timeOut);
-            JSONObject userPersonalFactorRspJSon = result.getJSONObject("data");
-            UserPersonalFactorRspNew rsp = JSON.toJavaObject(userPersonalFactorRspJSon, UserPersonalFactorRspNew.class);
+            UserPersonalFactorRspNew rsp = bidataServiceCaller.queryUserPersionalFactor(uid, udid, misortIds);
             if (rsp == null) {
             if (rsp == null) {
                 rsp = new UserPersonalFactorRspNew();
                 rsp = new UserPersonalFactorRspNew();
             }
             }

 package com.yoho.search.service.recall.models.personal;
 package com.yoho.search.service.recall.models.personal;
 
 
-import com.yoho.search.core.personalized.PersonalizedSearch;
+import com.yoho.search.core.personalized.models.PersonalizedSearch;
 
 
 import java.util.ArrayList;
 import java.util.ArrayList;
 import java.util.List;
 import java.util.List;

@@ -3,7 +3,7 @@ package com.yoho.search.service.scorer.personal;
 import com.alibaba.fastjson.JSONObject;
 import com.alibaba.fastjson.JSONObject;
 import com.yoho.search.cache.CacheType;
 import com.yoho.search.cache.CacheType;
 import com.yoho.search.aop.cache.SearchCacheAble;
 import com.yoho.search.aop.cache.SearchCacheAble;
-import com.yoho.search.core.personalized.BigDataRedisService;
+import com.yoho.search.core.personalized.service.BidataServiceCaller;
 import org.slf4j.Logger;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 import org.slf4j.LoggerFactory;
 import org.springframework.beans.factory.annotation.Autowired;
 import org.springframework.beans.factory.annotation.Autowired;
@@ -17,14 +17,14 @@ public class PersonalGenderFeatureSearch {
     private static final Logger logger = LoggerFactory.getLogger(PersonalGenderFeatureSearch.class);
     private static final Logger logger = LoggerFactory.getLogger(PersonalGenderFeatureSearch.class);
 
 
     @Autowired
     @Autowired
-    private BigDataRedisService bigDataRedisService;
+    private BidataServiceCaller bidataServiceCaller;
 
 
     @SearchCacheAble(cacheInMinute = 30, cacheName = "USER_GENDER_FEATURE", returnClass = JSONObject.class, cacheType = CacheType.SEARCH_REDIS, includeParams = {"uid"})
     @SearchCacheAble(cacheInMinute = 30, cacheName = "USER_GENDER_FEATURE", returnClass = JSONObject.class, cacheType = CacheType.SEARCH_REDIS, includeParams = {"uid"})
     public JSONObject queryUserGenderFeature(Map<String, String> paramMap) {
     public JSONObject queryUserGenderFeature(Map<String, String> paramMap) {
         try {
         try {
             JSONObject jsonObject = new JSONObject();
             JSONObject jsonObject = new JSONObject();
-            Map<String, Float> userGenderFloat = bigDataRedisService.getUserGenderFeature(paramMap.getOrDefault("uid", "0"));
-            if(userGenderFloat!=null){
+            Map<String, Float> userGenderFloat = bidataServiceCaller.getUserGenderFeature(paramMap.getOrDefault("uid", "0"));
+            if (userGenderFloat != null) {
                 jsonObject.putAll(userGenderFloat);
                 jsonObject.putAll(userGenderFloat);
             }
             }
             return jsonObject;
             return jsonObject;

 package com.yoho.search.service.scorer.personal;
 package com.yoho.search.service.scorer.personal;
 
 
-import com.yoho.search.base.utils.DateUtil;
-import com.yoho.search.cache.CacheType;
 import com.yoho.search.aop.cache.SearchCacheAble;
 import com.yoho.search.aop.cache.SearchCacheAble;
-import com.yoho.search.core.personalized.BigDataRedisService;
-import com.yoho.search.core.personalized.PersonalizedSearch;
-import com.yoho.search.core.personalized.version.PersonalVersionManager;
+import com.yoho.search.cache.CacheType;
+import com.yoho.search.core.personalized.models.PersonalizedSearch;
+import com.yoho.search.core.personalized.service.BidataServiceCaller;
+import com.yoho.search.core.personalized.service.PersonalVersionManager;
 import org.apache.commons.lang3.StringUtils;
 import org.apache.commons.lang3.StringUtils;
 import org.slf4j.Logger;
 import org.slf4j.Logger;
 import org.slf4j.LoggerFactory;
 import org.slf4j.LoggerFactory;
 import org.springframework.beans.factory.annotation.Autowired;
 import org.springframework.beans.factory.annotation.Autowired;
 import org.springframework.stereotype.Service;
 import org.springframework.stereotype.Service;
 
 
-import java.util.Calendar;
-import java.util.HashMap;
 import java.util.Map;
 import java.util.Map;
 
 
 @Service
 @Service
@@ -21,13 +18,10 @@ public class PersonalVectorFeatureSearch {
 
 
 	private static final Logger logger = LoggerFactory.getLogger(PersonalVectorFeatureSearch.class);
 	private static final Logger logger = LoggerFactory.getLogger(PersonalVectorFeatureSearch.class);
 
 
-	private static final Double BASE_CONSTANT = 1.0D;
-	private static final Double FACTOR_CONSTANT = 0.8D;
-
 	@Autowired
 	@Autowired
 	private PersonalVersionManager personalVersionManager;
 	private PersonalVersionManager personalVersionManager;
 	@Autowired
 	@Autowired
-	private BigDataRedisService bigDataRedisService;
+	private BidataServiceCaller bidataServiceCaller;
 
 
 	@SearchCacheAble(cacheInMinute = 30, cacheName = "PERSIONAL_VECTOR", returnClass = PersonalizedSearch.class, cacheType = CacheType.SEARCH_REDIS, includeParams = { "uid" })
 	@SearchCacheAble(cacheInMinute = 30, cacheName = "PERSIONAL_VECTOR", returnClass = PersonalizedSearch.class, cacheType = CacheType.SEARCH_REDIS, includeParams = { "uid" })
 	public PersonalizedSearch getPersonalizedSearch(Map<String, String> paramMap) {
 	public PersonalizedSearch getPersonalizedSearch(Map<String, String> paramMap) {
@@ -43,7 +37,7 @@ public class PersonalVectorFeatureSearch {
 				return null;
 				return null;
 			}
 			}
 			// 3. 获取用户的特征向量
 			// 3. 获取用户的特征向量
-			String userVectorFeature = bigDataRedisService.getUserVectorFeature(uid, vectorFeatureVersion);
+			String userVectorFeature = bidataServiceCaller.getUserVectorFeature(uid, vectorFeatureVersion);
 			if (StringUtils.isEmpty(userVectorFeature)) {
 			if (StringUtils.isEmpty(userVectorFeature)) {
 				return null;
 				return null;
 			}
 			}
@@ -53,99 +47,5 @@ public class PersonalVectorFeatureSearch {
 			return null;
 			return null;
 		}
 		}
 	}
 	}
-	
-	/**
-	 * 根据UID和SKN列表获取特征相关性评分，该方法用于测试和问题定位
-	 *
-	 * @param uid
-	 *            用户标识
-	 * @param productVectorFeatureMap
-	 *            商品特征map
-	 * @return 相关性计算结果
-	 */
-	public Map<String, Object> calVectorFeature(String uid, String version, Map<String, String> productVectorFeatureMap) {
-		Map<String, Object> scoreMap = new HashMap<>();
-		if (productVectorFeatureMap == null || productVectorFeatureMap.isEmpty()) {
-			scoreMap.put("productVectorFeatureMap", "empty");
-			return scoreMap;
-		}
-
-		String vectorFeatureVersion = StringUtils.isNotEmpty(version) ? version : personalVersionManager.getCurrentVersionInZk();
-		scoreMap.put("vectorFeatureVersion", vectorFeatureVersion);
-		if (StringUtils.isEmpty(vectorFeatureVersion) || "-1".equals(vectorFeatureVersion)) {
-			return scoreMap;
-		}
-
-		// 2. 获取用户的特征向量
-		String userVectorFeature = bigDataRedisService.getUserVectorFeature(uid, vectorFeatureVersion);
-		scoreMap.put("userVectorFeature", userVectorFeature);
-		if (StringUtils.isEmpty(userVectorFeature)) {
-			return scoreMap;
-		}
-
-		String[] userFeatureFactorArr = userVectorFeature.split(",");
-		double tempUserFeatureVectorNorm = 0.0D;
-		int dimensionOfFactors = userFeatureFactorArr.length;
-		double[] userFeatureFactors = new double[dimensionOfFactors];
-		double temp;
-		for (int index = 0; index < dimensionOfFactors; index++) {
-			temp = Double.parseDouble(userFeatureFactorArr[index].trim());
-			userFeatureFactors[index] = temp;
-			tempUserFeatureVectorNorm += temp * temp;
-		}
-
-		final double userFeatureVectorNorm = tempUserFeatureVectorNorm;
-		scoreMap.put("userFeatureVectorNorm", userFeatureVectorNorm);
-		scoreMap.put("dimensionOfFactors", dimensionOfFactors);
-
-		// 3. 计算相关性得分
-		productVectorFeatureMap.forEach((skn, vector) -> {
-			Map<String, Object> content = new HashMap<String, Object>();
-			content.put("vector", vector);
-			content.put("score", calculateScore(vector, vectorFeatureVersion, userFeatureFactors, userFeatureVectorNorm));
-			scoreMap.put(skn, content);
-		});
-
-		return scoreMap;
-	}
-
-	public double calculateScore(String productFeatureFactor, String vectorFeatureVersion, double[] userFeatureFactors, double userFeatureVectorNorm) {
-		if (productFeatureFactor == null || productFeatureFactor.trim().isEmpty()) {
-			return BASE_CONSTANT;
-		}
-
-		String versionPrefix = vectorFeatureVersion + "|";
-		if (!productFeatureFactor.trim().startsWith(versionPrefix)) {
-			return BASE_CONSTANT;
-		}
-
-		String[] productFeatureFactorArr = productFeatureFactor.trim().substring(versionPrefix.length()).split(",");
-		if (productFeatureFactorArr == null || productFeatureFactorArr.length != userFeatureFactors.length) {
-			return BASE_CONSTANT;
-		}
-
-		double prodFeatureVectorNorm = 0.0D;
-		double productiveSum = 0.0D;
-		double tempProdFactor;
-		for (int i = 0; i < userFeatureFactors.length; i++) {
-			tempProdFactor = Double.parseDouble(productFeatureFactorArr[i].trim());
-			productiveSum += tempProdFactor * userFeatureFactors[i];
-			prodFeatureVectorNorm += tempProdFactor * tempProdFactor;
-		}
-
-		if (prodFeatureVectorNorm == 0) {
-			return BASE_CONSTANT;
-		}
-
-		double cosScore = productiveSum / (Math.sqrt(prodFeatureVectorNorm) * Math.sqrt(userFeatureVectorNorm));
-		double finalScore = BASE_CONSTANT + FACTOR_CONSTANT * cosScore;
-		return finalScore;
-	}
-
-	public static void main(String[] args) {
-		Calendar now = Calendar.getInstance();
-		now.add(Calendar.YEAR, -1);
-		System.out.println(DateUtil.getFirstTimeSecond(now.getTime()));
-	}
 
 
 }
 }

@@ -17,15 +17,3 @@ redis:
         readOnly:
         readOnly:
         - 192.168.102.216:6379
         - 192.168.102.216:6379
         primary: no
         primary: no
-
-    yohoSearchBigDataRedis:
-        cluster: false
-        sync: false
-        auth:
-        timeout: 1000
-        servers:
-        - 192.168.102.216:6379
-        readOnly:
-        - 192.168.102.216:6379
-        primary: no
-

@@ -18,14 +18,3 @@ redis:
         - ${search.redis.address}:${search.redis.port}
         - ${search.redis.address}:${search.redis.port}
         primary: no
         primary: no
 
 
-    yohoSearchBigDataRedis:
-        cluster: false
-        sync: false
-        auth: ${bigDataRedis-search.proxy.auth}
-        timeout: 1000
-        servers:
-        - ${bigDataRedis-search.proxy.address}:${bigDataRedis-search.proxy.port}
-        readOnly:
-        - ${bigDataRedis-search.proxy.address}:${bigDataRedis-search.proxy.port}
-        primary: no
-