yoho-search / yoho-search-consumer · Commits

Go to a project

Authored by hugufei
Commit ace795408502c6b54b1beddf1f05af10347c5c4f 2 parents 72bc16d4 bc9800b1

Merge branch 'master' into gray0520

Inline Side-by-side
Showing 9 changed files with 206 additions and 3 deletions
package com.yoho.search.dal;
import com.yoho.search.dal.model.BigdataUvScore;
import org.apache.ibatis.annotations.Param;
import java.util.List;
public interface BigdataUvScoreMapper {
Integer selectLatestDateId();
List<BigdataUvScore> selectList(@Param("sknList") List<Integer> sknList, @Param("dateId") Integer dateId);
List<Integer> selectSknListByDateId(@Param(value = "dateId") Integer dateId, @Param(value = "pageSize") Integer pageSize);
void deleteBatch(@Param(value = "sknList") List<Integer> sknList, @Param(value = "dateId") Integer dateId);
}
\ No newline at end of file
... ...
package com.yoho.search.dal.model;
import java.math.BigDecimal;
public class BigdataUvScore {
private Integer productSkn;
private Integer dateId;
private BigDecimal uvScore;
private Integer createTime;
public Integer getProductSkn() {
return productSkn;
}
public void setProductSkn(Integer productSkn) {
this.productSkn = productSkn;
}
public Integer getDateId() {
return dateId;
}
public void setDateId(Integer dateId) {
this.dateId = dateId;
}
public BigDecimal getUvScore() {
return uvScore;
}
public void setUvScore(BigDecimal uvScore) {
this.uvScore = uvScore;
}
public Integer getCreateTime() {
return createTime;
}
public void setCreateTime(Integer createTime) {
this.createTime = createTime;
}
}
\ No newline at end of file
... ...
<?xml version="1.0" encoding="UTF-8" ?>
<!DOCTYPE mapper PUBLIC "-//mybatis.org//DTD Mapper 3.0//EN" "http://mybatis.org/dtd/mybatis-3-mapper.dtd" >
<mapper namespace="com.yoho.search.dal.BigdataUvScoreMapper">
<resultMap id="BaseResultMap" type="com.yoho.search.dal.model.BigdataUvScore">
<id column="product_skn" property="productSkn" jdbcType="INTEGER"/>
<id column="date_id" property="dateId" jdbcType="INTEGER"/>
<result column="uv_score" property="uvScore" jdbcType="DECIMAL"/>
<result column="create_time" property="createTime" jdbcType="INTEGER"/>
</resultMap>
<sql id="Base_Column_List">
product_skn, date_id, uv_score, create_time
</sql>
<select id="selectLatestDateId" resultType="java.lang.Integer">
select max(date_id) from bigdata_uvscore limit 1
</select>
<select id="selectList" resultMap="BaseResultMap" timeout="20000">
select
<include refid="Base_Column_List"/>
from bigdata_uvscore
WHERE
product_skn in
<foreach item="item" index="index" collection="sknList" open="(" separator="," close=")">
#{item}
</foreach>
AND
date_id = #{dateId,jdbcType=INTEGER}
</select>
<select id="selectSknListByDateId" resultType="java.lang.Integer" timeout="20000">
select product_skn from bigdata_uvscore WHERE date_id = #{dateId,jdbcType=INTEGER}
limit #{pageSize}
</select>
<delete id="deleteBatch">
DELETE FROM bigdata_uvscore
WHERE
product_skn in
<foreach item="item" index="index" collection="sknList" open="(" separator="," close=")">
#{item}
</foreach>
AND
date_id = #{dateId,jdbcType=INTEGER}
</delete>
</mapper>
\ No newline at end of file
... ...
... ... @@ -95,7 +95,9 @@ public class CleanDataJob {
//15.bigdata_ufo_sales_money [ufo_search]
this.cleanBigdataUfoSalesMoney(jobIndex++);
this.safeSleep(10);
//16.bigdata_uvscore
this.cleanBigdataUvScore(jobIndex++);
this.safeSleep(10);
}
@Autowired
... ... @@ -511,6 +513,34 @@ public class CleanDataJob {
}
}
@Autowired
private BigdataUvScoreMapper bigdataUvScoreMapper;
private void cleanBigdataUvScore(int jobIndex) {
try {
//1、记录开始日志
String jobName = "cleanBigdataUvScore";
long begin = System.currentTimeMillis();
String generateDateStr = getLastFiveDayDate();
this.addBeginLog(jobIndex, jobName, generateDateStr, begin);
Integer dateId = Integer.valueOf(generateDateStr);
// 2、执行清理
int delCount = 0;
List<Integer> idList = bigdataUvScoreMapper.selectSknListByDateId(dateId, CLEAN_BATCH_COUNT);
while (CollectionUtils.isNotEmpty(idList)) {
delCount = delCount + idList.size();
bigdataUvScoreMapper.deleteBatch(idList, dateId);
idList = bigdataUvScoreMapper.selectSknListByDateId(dateId, CLEAN_BATCH_COUNT);
}
//3、记录结束日志
this.addFinishLog(jobIndex, jobName, generateDateStr, delCount, begin);
} catch (Exception e) {
logger.error(e.getMessage(), e);
}
}
... ...
... ... @@ -286,7 +286,8 @@
},
"csBrandKeyword": {
"type": "text",
"analyzer": "comma_spliter"
"analyzer": "comma_spliter",
"search_analyzer": "ik_complex_search"
},
"countryId": {
"type": "integer"
... ... @@ -622,6 +623,9 @@
"recallValue": {
"type": "double"
},
"uvScore": {
"type": "double"
},
"priceArea": {
"type": "integer"
},
... ...
... ... @@ -45,6 +45,9 @@ public class ProductIndexBO extends ProductIBO implements Serializable {
//from bigdata_sales_num
private Integer totalSalesNum;
//from bigdata_uvscore
private BigDecimal uvScore;
// from product_goods
private String goodsList;
... ... @@ -723,4 +726,11 @@ public class ProductIndexBO extends ProductIBO implements Serializable {
this.totalSalesNum = totalSalesNum;
}
public BigDecimal getUvScore() {
return uvScore;
}
public void setUvScore(BigDecimal uvScore) {
this.uvScore = uvScore;
}
}
\ No newline at end of file
... ...
... ... @@ -190,6 +190,7 @@ public class ProductIndexBOToMapService {
map.put(ProductIndexEsField.heatValue, productIndexBO.getHeatValue());
map.put(ProductIndexEsField.ctrValue, productIndexBO.getCtrValue());
map.put(ProductIndexEsField.recallValue, productIndexBO.getRecallValue());
map.put(ProductIndexEsField.uvScore, productIndexBO.getUvScore());
map.put(ProductIndexEsField.skuIds, productIndexBO.getSkuIds());
... ...
package com.yoho.search.consumer.service.logicService.productIndex;
import com.yoho.search.consumer.service.bo.ProductIndexBO;
import com.yoho.search.dal.BigdataUvScoreMapper;
import com.yoho.search.dal.model.BigdataUvScore;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Component;
import java.math.BigDecimal;
import java.util.ArrayList;
import java.util.List;
import java.util.Map;
import java.util.stream.Collectors;
@Component
public class UvScoreBuilder implements IndexFieldBuilder {
@Autowired
private BigdataUvScoreMapper bigdataUvScoreMapper;
private Integer dateId = null;
@Override
public void init() {
dateId = bigdataUvScoreMapper.selectLatestDateId();
}
@Override
public void build(List<ProductIndexBO> productIndexBOs, List<Integer> idList, List<Integer> sknList) {
List<BigdataUvScore> bigdataUvScoreList;
if (dateId == null) {
bigdataUvScoreList = new ArrayList<>();
} else {
bigdataUvScoreList = bigdataUvScoreMapper.selectList(sknList, dateId);
}
Map<Integer, BigDecimal> sknUvscoreMap = bigdataUvScoreList.stream().collect(Collectors.toMap(BigdataUvScore::getProductSkn, BigdataUvScore::getUvScore));
productIndexBOs.stream().forEach(productIndexBO -> {
BigDecimal uvScore = sknUvscoreMap.getOrDefault(productIndexBO.getProductSkn(),BigDecimal.ZERO);
productIndexBO.setUvScore(uvScore);
});
}
@Override
public void finish() {
dateId = null;
}
}
... ...
... ... @@ -27,7 +27,7 @@ public class UfoToYohoGeneralService {
productIndexBO.setProductKeyword(product.getKeyWords());
//状态
productIndexBO.setStatus(1);
if (product.getDelStatus() != 0 || product.getShelveStatus() == 0) {
if (product.getDelStatus() != 0 || product.getShelveStatus() != 1){
productIndexBO.setStatus(0);
}
productIndexBO.setIsSales(productIndexBO.getStatus() == 1 ? "Y" : "N");
... ...