yoho-search / yoho-search-consumer · Commits

Go to a project

Authored by wangnan9279
Commit 0483470a45de3d1f554fb1e77aa6beb5a01d2fb7 2 parents 7efe3d7e 3e509153

Merge branch 'master' into wn_image

Inline Side-by-side
Showing 20 changed files with 26 additions and 443 deletions
package com.yoho.search.dal;
import com.yoho.search.dal.model.BasePinRatio;
import org.apache.ibatis.annotations.Param;
import java.util.List;
/**
* Created by wangnan on 2016/6/30.
*/
public interface BasePinRatioMapper {
int selectCount();
Integer selectLatestDateId();
List<BasePinRatio> selectBasePinRatio(@Param(value = "sknList") List<Integer> sknList, @Param(value = "dateId") Integer dateId);
}
package com.yoho.search.dal;
import com.yoho.search.dal.model.BrandVectorFeature;
import org.apache.ibatis.annotations.Param;
import java.util.List;
public interface BrandVectorFeatureMapper {
Integer selectLatestDateId();
int selectCountByDateId(@Param(value = "dateId") String dateId);
List<BrandVectorFeature> selectListByDateId( @Param(value = "dateId") String dateId);
void deleteBatchByDateId(@Param(value = "dateId") String dateId);
}
\ No newline at end of file
package com.yoho.search.dal;
import com.yoho.search.dal.model.BrokenCode;
import org.apache.ibatis.annotations.Param;
import java.util.List;
/**
* Created by wangnan on 2016/6/30.
*/
public interface BrokenCodeProductMapper {
int selectCount();
Integer selectLatestDateId();
List<BrokenCode> selectBrokenCode(@Param(value = "sknList") List<Integer> sknList, @Param(value = "dateId") Integer dateId);
}
package com.yoho.search.dal;
import com.yoho.search.dal.model.DiscountProductSkn;
import java.util.List;
/**
* Created by wangnan on 2016/6/30.
*/
public interface DiscountProductSknMapper {
List<DiscountProductSkn> selectDiscountProductSkn(List<Integer> sknList);
}
package com.yoho.search.dal.model;
public class BrandVectorFeature {
private Integer brandId;
private Integer dateId;
private String brandFeature;
private Integer createTime;
public Integer getBrandId() {
return brandId;
}
public void setBrandId(Integer brandId) {
this.brandId = brandId;
}
public Integer getDateId() {
return dateId;
}
public void setDateId(Integer dateId) {
this.dateId = dateId;
}
public String getBrandFeature() {
return brandFeature;
}
public void setBrandFeature(String brandFeature) {
this.brandFeature = brandFeature;
}
public Integer getCreateTime() {
return createTime;
}
public void setCreateTime(Integer createTime) {
this.createTime = createTime;
}
}
\ No newline at end of file
<?xml version="1.0" encoding="UTF-8" ?>
<!DOCTYPE mapper PUBLIC "-//mybatis.org//DTD Mapper 3.0//EN" "http://mybatis.org/dtd/mybatis-3-mapper.dtd" >
<mapper namespace="com.yoho.search.dal.BasePinRatioMapper">
<resultMap id="BaseResultMap" type="com.yoho.search.dal.model.BasePinRatio">
<result column="product_skn" property="productSkn" jdbcType="INTEGER"/>
<result column="base_pin_ratio" property="basePinRatio" jdbcType="DECIMAL"/>
</resultMap>
<select id="selectBasePinRatio" resultMap="BaseResultMap" timeout="20000">
<![CDATA[
SELECT
`a`.`product_skn` AS `product_skn`,
`a`.`base_pin_ratio` AS `base_pin_ratio`
FROM
`free_size_product` `a`
WHERE
`a`.`date_id` = #{dateId,jdbcType=VARCHAR}
AND
]]>
product_skn
in
<foreach item="item" index="index" collection="sknList" open="(" separator="," close=")">
#{item}
</foreach>
</select>
<select id="selectLatestDateId" resultType="java.lang.Integer">
select max(date_id) from free_size_product
</select>
<select id="selectCount" resultType="java.lang.Integer" timeout="20000">
SELECT count(*) FROM free_size_product
</select>
</mapper>
\ No newline at end of file
<?xml version="1.0" encoding="UTF-8" ?>
<!DOCTYPE mapper PUBLIC "-//mybatis.org//DTD Mapper 3.0//EN" "http://mybatis.org/dtd/mybatis-3-mapper.dtd" >
<mapper namespace="com.yoho.search.dal.BrandVectorFeatureMapper">
<resultMap id="BaseResultMap" type="com.yoho.search.dal.model.BrandVectorFeature">
<id column="brand_id" property="brandId" jdbcType="INTEGER" />
<result column="date_id" property="dateId" jdbcType="INTEGER" />
<result column="brand_feature" property="brandFeature" jdbcType="LONGVARCHAR" />
<result column="create_time" property="createTime" jdbcType="INTEGER" />
</resultMap>
<sql id="Base_Column_List">
brand_id, date_id, brand_feature, create_time
</sql>
<select id="selectLatestDateId" resultType="java.lang.Integer">
select max(date_id) from brand_vector_feature
</select>
<select id="selectCountByDateId" resultType="java.lang.Integer" timeout="20000">
SELECT count(*) FROM brand_vector_feature where date_id = #{dateId}
</select>
<select id="selectListByDateId" resultMap="BaseResultMap" timeout="20000">
select * from brand_vector_feature p where date_id = #{dateId}
</select>
<delete id="deleteBatchByDateId">
DELETE FROM brand_vector_feature WHERE date_id = #{dateId,jdbcType=VARCHAR}
</delete>
</mapper>
\ No newline at end of file
<?xml version="1.0" encoding="UTF-8" ?>
<!DOCTYPE mapper PUBLIC "-//mybatis.org//DTD Mapper 3.0//EN" "http://mybatis.org/dtd/mybatis-3-mapper.dtd" >
<mapper namespace="com.yoho.search.dal.BrokenCodeProductMapper">
<resultMap id="BaseResultMap" type="com.yoho.search.dal.model.BrokenCode">
<result column="product_skn" property="productSkn" jdbcType="INTEGER"/>
<result column="breaking_rate" property="breakingRate" jdbcType="DOUBLE"/>
</resultMap>
<select id="selectBrokenCode" resultMap="BaseResultMap" timeout="20000">
<![CDATA[
SELECT
`a`.`product_skn` AS `product_skn`,
`a`.`breaking_rate` AS `breaking_rate`
FROM
`nonfree_size_product` `a`
WHERE
`a`.`date_id` = #{dateId,jdbcType=VARCHAR}
AND
]]>
product_skn
in
<foreach item="item" index="index" collection="sknList" open="(" separator="," close=")">
#{item}
</foreach>
</select>
<select id="selectLatestDateId" resultType="java.lang.Integer">
select max(date_id) from nonfree_size_product
</select>
<select id="selectCount" resultType="java.lang.Integer" timeout="20000">
SELECT count(*) FROM nonfree_size_product
</select>
</mapper>
\ No newline at end of file
<?xml version="1.0" encoding="UTF-8" ?>
<!DOCTYPE mapper PUBLIC "-//mybatis.org//DTD Mapper 3.0//EN" "http://mybatis.org/dtd/mybatis-3-mapper.dtd" >
<mapper namespace="com.yoho.search.dal.DiscountProductSknMapper">
<resultMap id="BaseResultMap" type="com.yoho.search.dal.model.DiscountProductSkn">
<result column="product_skn" property="productSkn" jdbcType="INTEGER"/>
<result column="pool_id" property="poolId" jdbcType="VARCHAR"/>
<result column="score" property="score" jdbcType="DOUBLE"/>
</resultMap>
<select id="selectDiscountProductSkn" resultMap="BaseResultMap" timeout="20000">
<![CDATA[
SELECT
`dp`.`product_skn` AS `product_skn`,
group_concat(`dp`.`pool_id` SEPARATOR ',') AS `pool_id`,
`dp`.`score` AS `score`
FROM
`discount_product` `dp`
where
product_skn
in]]>
<foreach item="item" index="index" collection="list"
open="(" separator="," close=")">
#{item}
</foreach>
GROUP BY
`dp`.`product_skn`
</select>
</mapper>
\ No newline at end of file
... ... @@ -57,7 +57,7 @@ public class BigdataMonitorService {
private ApiResponse ifNeedAlarm(List<BigdataMonitor> results, int yesterday, int beforYesterday, int threshold, String tableName) {
if (CollectionUtils.isEmpty(results)) {
return ApiResponse.createError("bigdata_similar_skn: 没有数据", null);
return ApiResponse.createError(tableName + ": 没有数据", null);
}
Map<String, BigdataMonitor> dataMap = results.stream().collect(Collectors.toMap(BigdataMonitor::getDateId, Function.identity()));
StringBuilder msg = new StringBuilder();
... ...
... ... @@ -3,14 +3,10 @@ package com.yoho.search.consumer.index.fullbuild;
import com.yoho.search.consumer.index.common.IIndexBuilder;
import com.yoho.search.consumer.service.bo.BrandBO;
import com.yoho.search.consumer.service.daoService.BrandService;
import com.yoho.search.consumer.service.logicService.personal.PersonalVectorVersionManager;
import com.yoho.search.dal.BrandVectorFeatureMapper;
import com.yoho.search.dal.ProductMapper;
import com.yoho.search.dal.model.Brand;
import com.yoho.search.dal.model.BrandSortIdsBO;
import com.yoho.search.dal.model.BrandVectorFeature;
import org.apache.commons.collections.CollectionUtils;
import org.apache.commons.lang.StringUtils;
import org.springframework.beans.BeanUtils;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Component;
... ...
... ... @@ -66,8 +66,6 @@ public class GeneralBuilder {
productIndexBO.setShelveDay(DateUtil.TimeStamp2DateWithFormat(Long.valueOf(productIndexBO.getShelveTime()), "yyyy-MM-dd"));
}
productIndexBO.setEditTime(null == tblProduct.getUpdateTime() ? null : tblProduct.getUpdateTime());
productIndexBO.setBreakingRate(0d);
productIndexBO.setBasePinRatio(0);
// 处理国家id和名称
productIndexBO.setTblCountryId(tblProduct.getCountryId());
productIndexBO.setTblCountryName(countryFieldMap.get(tblProduct.getCountryId()));
... ...
... ... @@ -34,8 +34,6 @@ public class CleanDataJob {
@Autowired
private ProductSalesInfoMapper productSalesInfoMapper;
@Autowired
private BrandVectorFeatureMapper brandVectorFeatureMapper;
@Autowired
private SimilarSknMapper similarSknMapper;
private String getLastThreeDayDate() {
... ... @@ -109,15 +107,6 @@ public class CleanDataJob {
}
logger.info("productHeatValues clean end----[clearDate={}][cost={}]", generateDateStr, System.currentTimeMillis() - begin);
}
@Scheduled(cron = "0 25 1 * * ?")
public void cleanBrandVectorFeature() {
String generateDateStr = getLastThreeDayDate();
long begin = System.currentTimeMillis();
logger.info("BrandVectorFeature clean start----[begin={}]", begin);
brandVectorFeatureMapper.deleteBatchByDateId(generateDateStr);
logger.info("BrandVectorFeature clean end----[clearDateStr={}][cost={}]", generateDateStr, System.currentTimeMillis() - begin);
}
@Scheduled(cron = "0 30 1 * * ?")
public void cleanSimilarSkn() {
... ...
... ... @@ -499,15 +499,6 @@
"tblCountryName": {
"type": "keyword"
},
"basePinRatio": {
"type": "double"
},
"breakingRate": {
"type": "double"
},
"discountScore": {
"type": "double"
},
"forbiddenPageIds": {
"type": "text",
"analyzer": "comma_spliter"
... ...
... ... @@ -63,13 +63,13 @@ public class ProductIndexBO extends ProductIBO implements Serializable {
private String yohoodId;
// from base_pin_ratio
private double basePinRatio;
//private double basePinRatio;
// from broken_code
private double breakingRate;
//private double breakingRate;
// from discount_product_skn
private double discountScore;
//private double discountScore;
private String poolIds;
private String hrShopIds;
... ... @@ -305,17 +305,17 @@ public class ProductIndexBO extends ProductIBO implements Serializable {
return yohoodId;
}
public double getBasePinRatio() {
return basePinRatio;
}
public double getBreakingRate() {
return breakingRate;
}
public double getDiscountScore() {
return discountScore;
}
// public double getBasePinRatio() {
// return basePinRatio;
// }
//
// public double getBreakingRate() {
// return breakingRate;
// }
//
// public double getDiscountScore() {
// return discountScore;
// }
public void setProductKeyword(String productKeyword) {
this.productKeyword = productKeyword;
... ... @@ -381,17 +381,17 @@ public class ProductIndexBO extends ProductIBO implements Serializable {
this.yohoodId = yohoodId;
}
public void setBasePinRatio(double basePinRatio) {
this.basePinRatio = basePinRatio;
}
public void setBreakingRate(double breakingRate) {
this.breakingRate = breakingRate;
}
public void setDiscountScore(double discountScore) {
this.discountScore = discountScore;
}
// public void setBasePinRatio(double basePinRatio) {
// this.basePinRatio = basePinRatio;
// }
//
// public void setBreakingRate(double breakingRate) {
// this.breakingRate = breakingRate;
// }
//
// public void setDiscountScore(double discountScore) {
// this.discountScore = discountScore;
// }
public String getPoolIds() {
return poolIds;
... ...
... ... @@ -116,10 +116,7 @@ public class ProductIndexBOToMapService {
map.put(ProductIndexEsField.maxSort, productIndexBO.getMaxSort());
map.put(ProductIndexEsField.middleSort, productIndexBO.getMiddleSort());
map.put(ProductIndexEsField.smallSort, productIndexBO.getSmallSort());
map.put(ProductIndexEsField.basePinRatio, productIndexBO.getBasePinRatio());
map.put(ProductIndexEsField.breakingRate, productIndexBO.getBreakingRate());
map.put(ProductIndexEsField.discountScore, productIndexBO.getDiscountScore());
map.put(ProductIndexEsField.poolIds, productIndexBO.getPoolIds());
map.put(ProductIndexEsField.hrShopIds, productIndexBO.getHrShopIds());
map.put(ProductIndexEsField.productKeyword, productIndexBO.getProductKeyword());
... ...
... ... @@ -3,7 +3,6 @@ package com.yoho.search.consumer.service.logicService.personal;
import com.yoho.search.base.utils.DateUtil;
import com.yoho.search.consumer.service.daoService.ProductVectorFeatureService;
import com.yoho.search.core.personalized.version.PersonalVersionManager;
import com.yoho.search.dal.BrandVectorFeatureMapper;
import org.apache.commons.lang.StringUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
... ... @@ -24,8 +23,6 @@ public class PersonalVectorVersionManager {
private PersonalVersionManager personalVersionManager;
@Autowired
private ProductVectorFeatureService productVectorFeatureService;
@Autowired
private BrandVectorFeatureMapper brandVectorFeatureMapper;
private volatile String generateDate = null;
... ... @@ -35,9 +32,6 @@ public class PersonalVectorVersionManager {
if (productVectorFeatureService.selectCount(dateId) > 0) {
return true;
}
if (brandVectorFeatureMapper.selectCountByDateId(dateId) > 0) {
return true;
}
return false;
}
... ...
package com.yoho.search.consumer.service.logicService.productIndex;
import com.yoho.search.consumer.service.logicService.productIndex.IndexFieldBuilder;
import com.yoho.search.consumer.service.bo.ProductIndexBO;
import com.yoho.search.dal.BasePinRatioMapper;
import com.yoho.search.dal.model.BasePinRatio;
import org.apache.commons.collections.CollectionUtils;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Component;
import java.util.List;
import java.util.Map;
import java.util.stream.Collectors;
/**
* Created by wangnan on 2017/1/9.
*/
@Component
public class BasePinRatioBuilder implements IndexFieldBuilder {
private Integer dateId = null;
@Autowired
private BasePinRatioMapper basePinRatioMapper;
@Override
public void build(List<ProductIndexBO> productIndexBOs, List<Integer> idList, List<Integer> sknList) {
if (dateId == null) {
dateId = basePinRatioMapper.selectLatestDateId();
}
if (dateId == null) {
return;
}
List<BasePinRatio> basePinRatios = basePinRatioMapper.selectBasePinRatio(sknList, dateId);
if (CollectionUtils.isEmpty(basePinRatios)) {
return;
}
Map<Integer, BasePinRatio> basePinRatioMap = basePinRatios.stream().collect(Collectors.toMap(BasePinRatio::getProductSkn, b -> b));
productIndexBOs.stream().forEach(productIndexBO -> {
BasePinRatio basePinRatio = basePinRatioMap.get(productIndexBO.getProductSkn());
if (basePinRatio != null && basePinRatio.getBasePinRatio() != null) {
productIndexBO.setBasePinRatio(basePinRatio.getBasePinRatio().doubleValue());
}
});
}
@Override
public void finish() {
dateId = null;
}
}
package com.yoho.search.consumer.service.logicService.productIndex;
import com.yoho.search.consumer.service.logicService.productIndex.IndexFieldBuilder;
import com.yoho.search.consumer.service.bo.ProductIndexBO;
import com.yoho.search.dal.BrokenCodeProductMapper;
import com.yoho.search.dal.model.BrokenCode;
import org.apache.commons.collections.CollectionUtils;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Component;
import java.util.List;
import java.util.Map;
import java.util.stream.Collectors;
/**
* Created by wangnan on 2017/1/9.
*/
@Component
public class BreakingRateBuilder implements IndexFieldBuilder {
private Integer dateId = null;
@Autowired
private BrokenCodeProductMapper brokenCodeProductMapper;
@Override
public void build(List<ProductIndexBO> productIndexBOs, List<Integer> idList, List<Integer> sknList) {
if (dateId == null) {
dateId = brokenCodeProductMapper.selectLatestDateId();
}
if (dateId == null) {
return;
}
List<BrokenCode> brokenCodes = brokenCodeProductMapper.selectBrokenCode(sknList, dateId);
if (CollectionUtils.isEmpty(brokenCodes)) {
return;
}
Map<Integer, BrokenCode> brokenCodesMap = brokenCodes.stream().collect(Collectors.toMap(BrokenCode::getProductSkn, b -> b));
productIndexBOs.stream().forEach(productIndexBO -> {
BrokenCode brokenCode = brokenCodesMap.get(productIndexBO.getProductSkn());
if (brokenCode != null && brokenCode.getBreakingRate() != null) {
productIndexBO.setBreakingRate(brokenCode.getBreakingRate().doubleValue());
}
});
}
@Override
public void finish() {
dateId = null;
}
}
package com.yoho.search.consumer.service.logicService.productIndex;
import com.yoho.search.consumer.service.logicService.productIndex.IndexFieldBuilder;
import com.yoho.search.consumer.service.bo.ProductIndexBO;
import com.yoho.search.dal.DiscountProductSknMapper;
import com.yoho.search.dal.model.DiscountProductSkn;
import org.apache.commons.collections.CollectionUtils;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Component;
import java.util.List;
import java.util.Map;
import java.util.stream.Collectors;
/**
* Created by wangnan on 2017/1/9.
*/
@Component
public class DiscountScoreBuilder implements IndexFieldBuilder {
@Autowired
private DiscountProductSknMapper discountProductSknMapper;
@Override
public void build(List<ProductIndexBO> productIndexBOs, List<Integer> idList, List<Integer> sknList) {
List<DiscountProductSkn> discountProductSknList = discountProductSknMapper.selectDiscountProductSkn(sknList);
if (CollectionUtils.isEmpty(discountProductSknList)) {
return;
}
Map<Integer, DiscountProductSkn> discountProductSknMap = discountProductSknList.stream().collect(Collectors.toMap(DiscountProductSkn::getProductSkn, (p) -> p));
productIndexBOs.stream().forEach(productIndexBO -> {
DiscountProductSkn discountProductSkn = discountProductSknMap.get(productIndexBO.getProductSkn());
if (discountProductSkn != null && discountProductSkn.getScore() != null) {
productIndexBO.setDiscountScore(discountProductSkn.getScore());
}
});
}
}