Merge branch 'dev_log_day'

@@ -71,6 +71,9 @@
                             <value>/UnionRest/queryUnionOrders</value>
                             <value>/UnionRest/queryUnionOrders</value>
                             <value>/getProdcutDetail</value>
                             <value>/getProdcutDetail</value>
                             <value>/pushArticle</value>
                             <value>/pushArticle</value>
+                            <value>/UnionClickCountDayRest/delete</value>
+                			<value>/UnionClickCountDayRest/count</value>
+                            
                      </list>
                      </list>
               </property>
               </property>
               <property name="excludeMethods">
               <property name="excludeMethods">

@@ -4,6 +4,7 @@ package com.yoho.unions.dal;
 import com.yoho.service.model.union.request.ActivateDeviceIdReqBO;
 import com.yoho.service.model.union.request.ActivateDeviceIdReqBO;
 import com.yoho.service.model.union.request.UnionLogsReqBO;
 import com.yoho.service.model.union.request.UnionLogsReqBO;
 import com.yoho.service.model.union.response.ActivateDeviceIdRspBO;
 import com.yoho.service.model.union.response.ActivateDeviceIdRspBO;
+import com.yoho.unions.dal.model.UnionClickCount;
 import com.yoho.unions.dal.model.UnionClickLogs;
 import com.yoho.unions.dal.model.UnionClickLogs;
 import org.apache.ibatis.annotations.Param;
 import org.apache.ibatis.annotations.Param;
 
 
@@ -37,4 +38,6 @@ public interface IUnionClickLogsDAO {
     int queryUnionClickLogsCountByParam(UnionLogsReqBO unionLogsReqBO);
     int queryUnionClickLogsCountByParam(UnionLogsReqBO unionLogsReqBO);
 
 
     List<UnionClickLogs> queryUnionClickLogsByParam(UnionLogsReqBO unionLogsReqBO);
     List<UnionClickLogs> queryUnionClickLogsByParam(UnionLogsReqBO unionLogsReqBO);
+    
+    List<UnionClickCount> queryCountInfoByTime(@Param("beginTime") int beginTime,@Param("endTime") int endTime);
 }
 }

+package com.yoho.unions.dal;
+
+import java.util.List;
+
+import org.apache.ibatis.annotations.Param;
+
+import com.yoho.unions.dal.model.UnionClickCountDay;
+
+public interface UnionClickCountDayMapper {
+    int deleteByPrimaryKey(Integer id);
+
+    int insert(UnionClickCountDay record);
+
+    UnionClickCountDay selectByPrimaryKey(Integer id);
+
+    List<UnionClickCountDay> selectAll();
+
+    int updateByPrimaryKey(UnionClickCountDay record);
+
+	int deleteCountInfoByTime(@Param("beginTime") int beginTime,@Param("endTime") int endTime);
+
+}

+package com.yoho.unions.dal.model;
+import lombok.Data;
+import lombok.ToString;
+
+@Data
+@ToString
+public class UnionClickCount {
+    private String unionType;
+    private String clientType;
+    private Integer num; 
+}

+package com.yoho.unions.dal.model;
+
+public class UnionClickCountDay {
+    private Integer id;
+
+    private String dayId;
+
+    private String unionType;
+
+    private Integer clientType;
+
+    private Integer num;
+
+    private String comb;
+
+    private Integer createTime;
+
+    public Integer getId() {
+        return id;
+    }
+
+    public void setId(Integer id) {
+        this.id = id;
+    }
+
+    public String getDayId() {
+        return dayId;
+    }
+
+    public void setDayId(String dayId) {
+        this.dayId = dayId;
+    }
+
+    public String getUnionType() {
+        return unionType;
+    }
+
+    public void setUnionType(String unionType) {
+        this.unionType = unionType;
+    }
+
+    public Integer getClientType() {
+        return clientType;
+    }
+
+    public void setClientType(Integer clientType) {
+        this.clientType = clientType;
+    }
+
+    public Integer getNum() {
+        return num;
+    }
+
+    public void setNum(Integer num) {
+        this.num = num;
+    }
+
+    public String getComb() {
+        return comb;
+    }
+
+    public void setComb(String comb) {
+        this.comb = comb;
+    }
+
+    public Integer getCreateTime() {
+        return createTime;
+    }
+
+    public void setCreateTime(Integer createTime) {
+        this.createTime = createTime;
+    }
+}

+<?xml version="1.0" encoding="UTF-8"?>
+<!DOCTYPE mapper PUBLIC "-//mybatis.org//DTD Mapper 3.0//EN" "http://mybatis.org/dtd/mybatis-3-mapper.dtd">
+<mapper namespace="com.yoho.unions.dal.UnionClickCountDayMapper">
+  <resultMap id="BaseResultMap" type="com.yoho.unions.dal.model.UnionClickCountDay">
+    <id column="id" jdbcType="INTEGER" property="id" />
+    <result column="day_id" jdbcType="VARCHAR" property="dayId" />
+    <result column="union_type" jdbcType="VARCHAR" property="unionType" />
+    <result column="client_type" jdbcType="INTEGER" property="clientType" />
+    <result column="num" jdbcType="INTEGER" property="num" />
+    <result column="comb" jdbcType="VARCHAR" property="comb" />
+    <result column="create_time" jdbcType="INTEGER" property="createTime" />
+  </resultMap>
+  <delete id="deleteByPrimaryKey" parameterType="java.lang.Integer">
+    delete from union_click_count_day
+    where id = #{id,jdbcType=INTEGER}
+  </delete>
+  <insert id="insert" parameterType="com.yoho.unions.dal.model.UnionClickCountDay">
+    insert into union_click_count_day (id, day_id, union_type, 
+      client_type, num, comb, 
+      create_time)
+    values (#{id,jdbcType=INTEGER}, #{dayId,jdbcType=VARCHAR}, #{unionType,jdbcType=VARCHAR}, 
+      #{clientType,jdbcType=INTEGER}, #{num,jdbcType=INTEGER}, #{comb,jdbcType=VARCHAR}, 
+      #{createTime,jdbcType=INTEGER})
+  </insert>
+  <update id="updateByPrimaryKey" parameterType="com.yoho.unions.dal.model.UnionClickCountDay">
+    update union_click_count_day
+    set day_id = #{dayId,jdbcType=VARCHAR},
+      union_type = #{unionType,jdbcType=VARCHAR},
+      client_type = #{clientType,jdbcType=INTEGER},
+      num = #{num,jdbcType=INTEGER},
+      comb = #{comb,jdbcType=VARCHAR},
+      create_time = #{createTime,jdbcType=INTEGER}
+    where id = #{id,jdbcType=INTEGER}
+  </update>
+  <select id="selectByPrimaryKey" parameterType="java.lang.Integer" resultMap="BaseResultMap">
+    select id, day_id, union_type, client_type, num, comb, create_time
+    from union_click_count_day
+    where id = #{id,jdbcType=INTEGER}
+  </select>
+  <select id="selectAll" resultMap="BaseResultMap">
+    select id, day_id, union_type, client_type, num, comb, create_time
+    from union_click_count_day
+  </select>
+  <delete id="deleteCountInfoByTime">
+        delete
+    		from union_click_count_day
+        	where 
+        		create_time &gt;= #{beginTime,jdbcType=INTEGER} 
+        	and create_time &lt; #{endTime,jdbcType=INTEGER}
+    </delete>
+</mapper>

@@ -271,4 +271,12 @@
             and imei = #{imei}
             and imei = #{imei}
         </if>
         </if>
     </select>
     </select>
+    <select id="queryCountInfoByTime" resultType="com.yoho.unions.dal.model.UnionClickCount" >
+        select union_type as unionType,client_type as clientType,count(*) as num from union_click_logs 
+        	where 
+        		create_time &gt;= #{beginTime,jdbcType=INTEGER} 
+        	and create_time &lt; #{endTime,jdbcType=INTEGER}
+        	group by 
+        		union_type,client_type;
+    </select>
 </mapper>
 </mapper>

+package com.yoho.unions.server.restapi;
+
+import javax.annotation.Resource;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import org.springframework.stereotype.Controller;
+import org.springframework.web.bind.annotation.RequestMapping;
+import org.springframework.web.bind.annotation.ResponseBody;
+
+import com.yoho.unions.server.task.UnionClickCountDayTask;
+
+@Controller
+@RequestMapping("/UnionClickCountDayRest")
+public class UnionClickCountDayRest {
+
+	private static Logger log = LoggerFactory.getLogger(UnionClickCountDayRest.class);
+	
+	@Resource
+	private UnionClickCountDayTask task;
+	
+	@RequestMapping(value = "/count")
+	@ResponseBody
+	public String count(Integer start, Integer end) {
+		log.info("enter UnionClickCountDayRest.count with start is {}, end is {}", start, end);
+		try {
+			return task.exeTask(start, end);
+		} catch (Exception e) {
+			return "error : " + e.getMessage();
+		}
+	}
+	
+	@RequestMapping(value = "/delete")
+	@ResponseBody 
+	public String delete(Integer start, Integer end) {
+		log.info("enter UnionClickCountDayRest.delete with start is {}, end is {}", start, end);
+		try {
+			return task.deleteData(start, end);
+		} catch (Exception e) {
+			return "error : " + e.getMessage();
+		}
+	}
+}

+package com.yoho.unions.server.task;
+
+import java.text.SimpleDateFormat;
+import java.util.Calendar;
+import java.util.Date;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.concurrent.locks.Lock;
+import java.util.concurrent.locks.ReentrantLock;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.dao.QueryTimeoutException;
+import org.springframework.scheduling.annotation.Scheduled;
+import org.springframework.stereotype.Component;
+
+import com.yoho.unions.dal.IUnionClickLogsDAO;
+import com.yoho.unions.dal.UnionClickCountDayMapper;
+import com.yoho.unions.dal.model.UnionClickCount;
+import com.yoho.unions.dal.model.UnionClickCountDay;
+
+@Component
+public class UnionClickCountDayTask {
+
+    private Logger logger = LoggerFactory.getLogger(UnionClickCountDayTask.class);
+
+    @Autowired
+    private UnionClickCountDayMapper unionClickCountDayMapper;
+
+    @Autowired
+    private IUnionClickLogsDAO unionClickLogsDAO;
+    
+    private SimpleDateFormat format = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss");
+    private SimpleDateFormat formatDay = new SimpleDateFormat("yyyyMMdd");
+    
+    // 防止交错执行
+    private Lock lock = new ReentrantLock();
+    
+    // 每天00:05分执行
+    @Scheduled(cron = "* 05 0 * * ?")
+    //@Scheduled(cron = "* 0/2 * * * ?")
+	public void run() {
+    	
+    	Calendar time = Calendar.getInstance();
+		time.set(Calendar.HOUR_OF_DAY, 0);
+		time.set(Calendar.MINUTE, 0);
+		time.set(Calendar.SECOND, 0);
+		time.set(Calendar.MILLISECOND, 0);
+		int dayStart = (int) (time.getTimeInMillis() / 1000);
+		int begin = dayStart - 24 * 60 * 60;
+		int end = dayStart;
+    	
+		exeTask(begin, end);
+	}
+
+	public String exeTask(int begin, int end) {
+		
+		if (!lock.tryLock()) {
+			logger.warn("UnionClickCountDayTask run time={} a job has running", format.format(new Date()));
+			return "a job has running";
+		}
+
+		try {
+
+			if(end - begin > 24 * 60 * 60) {
+				logger.warn("UnionClickCountDayTask run time={} begin to end over a day", format.format(new Date()));
+				return "begin to end over a day";
+			}
+			
+			if(format.format(new Date(begin * 1000L)).equals(format.format(new Date(end * 1000L - 1)))) {
+				logger.warn("UnionClickCountDayTask run time={} begin and end not in same day", format.format(new Date()));
+				return "begin and end not in same day";
+			}
+			
+			int queryAcross = 5 * 60;
+			int timeoutRetryCount = 3;
+			logger.info("UnionClickCountDayTask run start={} , end={} , across={} , retry={}",
+					format.format(new Date(begin * 1000L)), format.format(new Date(end * 1000L - 1)), queryAcross,
+					timeoutRetryCount);
+			Map<String, UnionClickCount> map = new HashMap<>();
+			
+			long s = System.currentTimeMillis();
+			int count = 0;
+			// 当小于结束时间一直执行
+			while (begin < end) {
+				List<UnionClickCount> data = null;
+				for (int i = 0; i < timeoutRetryCount; i++) {
+					try {
+
+						logger.info("UnionClickCountDayTask run start={} , end={} , across={} , now_retry={}",
+								format.format(new Date(begin * 1000L)), format.format(new Date(end * 1000L - 1)),
+								queryAcross, i);
+
+						long exeStart = System.currentTimeMillis();
+						data = queryCountInfoByTime(begin, Math.min(begin + queryAcross, end));
+						begin += queryAcross;
+						long exeEnd = System.currentTimeMillis();
+						long use = exeEnd - exeStart;
+
+						logger.info("UnionClickCountDayTask run start={} , end={} , across={} , use={}",
+								format.format(new Date(begin * 1000L)), format.format(new Date(end * 1000L - 1)),
+								queryAcross, use);
+						if (use < 700) {
+							logger.info("UnionClickCountDayTask change across from {} to {}", queryAcross,
+									queryAcross + 60);
+							queryAcross += 60;
+						}
+						if (use > 800) {
+							logger.info("UnionClickCountDayTask change across from {} to {}", queryAcross,
+									queryAcross - 60);
+							queryAcross -= 60;
+						}
+						// 执行成功，则不再重试
+						break;
+					} catch (QueryTimeoutException e) {
+						logger.warn("Exe queryCountInfoByTime start={}, end={} timeout!", begin,
+								Math.min(begin + queryAcross, end));
+						if (queryAcross <= 2) {
+							logger.warn("Exe queryCountInfoByTime queryAcross={} return", queryAcross);
+							return "queryCountInfoByTime queryAcross < 2";
+						}
+						queryAcross >>>= 2;
+					}
+				}
+
+				if (data == null) {
+					logger.warn("Exe queryCountInfoByTime sleep start={}, end={} timeout!", begin,
+							Math.min(begin + queryAcross, end));
+					try {
+						Thread.sleep(1000);
+					} catch (InterruptedException e) {
+					}
+				} else {
+					logger.info("Exe queryCountInfoByTime data.size={} ", data.size());
+					addDataToMap(data, map);
+				}
+			}
+			// 把统计结果放到表里
+			logger.info("UnionClickCountDayTask run start to insert map.size={}", map.size());
+			String dayId = formatDay.format(new Date(begin * 1000L));
+			int successCount = 0;
+			int failedCount = 0;
+			for (Map.Entry<String, UnionClickCount> entry : map.entrySet()) {
+
+				try {
+					UnionClickCount info = entry.getValue();
+					UnionClickCountDay bean = new UnionClickCountDay();
+					bean.setDayId(dayId);
+					bean.setUnionType(info.getUnionType());
+					bean.setClientType(getClientType(info.getClientType()));
+					bean.setNum(info.getNum());
+					bean.setCreateTime((int) (System.currentTimeMillis() / 1000));
+					bean.setComb(dayId + "_" + bean.getUnionType() + "_" + bean.getClientType());
+
+					logger.info("UnionClickCountDayTask insert bean={}", bean);
+					unionClickCountDayMapper.insert(bean);
+					successCount++;
+					count += info.getNum();
+				} catch (Exception e) {
+					logger.warn("UnionClickCountDayTask insert error={}", e.getMessage());
+					failedCount++;
+				}
+			}
+			long e = System.currentTimeMillis();
+			long u = e - s;
+			logger.info("UnionClickCountDayTask run over data count={}, insert over! success={}, error={}, timeUse={}", count, successCount, failedCount, u);
+			return "UnionClickCountDayTask run over data count=" + count + ", insert success=" + successCount + ", error=" + failedCount+", timseUse="+u;
+		} finally {
+			lock.unlock();
+		}
+	}
+	
+	public String deleteData(int begin, int end) {
+
+		if (!lock.tryLock()) {
+			logger.warn("UnionClickCountDayTask deleteData time={} a job has running", format.format(new Date()));
+			return "a job has running";
+		}
+
+		try {
+
+			if (end - begin > 24 * 60 * 60) {
+				logger.warn("UnionClickCountDayTask deleteData time={} begin to end over a day",
+						format.format(new Date()));
+				return "begin to end over a day";
+			}
+
+			if (format.format(new Date(begin * 1000L)).equals(format.format(new Date(end * 1000L - 1)))) {
+				logger.warn("UnionClickCountDayTask deleteData time={} begin and end not in same day",
+						format.format(new Date()));
+				return "begin and end not in same day";
+			}
+
+			int queryAcross = 5 * 60;
+			int timeoutRetryCount = 3;
+			logger.info("UnionClickCountDayTask deleteData start={} , end={} , across={} , retry={}",
+					format.format(new Date(begin * 1000L)), format.format(new Date(end * 1000L - 1)), queryAcross,
+					timeoutRetryCount);
+			Integer count = null;
+			try {
+				count = deleteCountInfoByTime(begin, end);
+			} catch (Exception e) {
+				logger.info("UnionClickCountDayTask deleteData error! msg={}", e.getMessage());
+				return "Error : " + e.getMessage();
+			}
+			logger.info("UnionClickCountDayTask deleteData over! deleteCount={}", count);
+			return "deleteData over, deleteCount=" + count;
+		} finally {
+			lock.unlock();
+		}
+	}
+    
+	private void addDataToMap(List<UnionClickCount> data, Map<String, UnionClickCount> map) {
+		for (UnionClickCount d : data) {
+			String key = d.getUnionType() + "_" + d.getClientType();
+			UnionClickCount store = map.get(key);
+			if (store == null) {
+				map.put(key, d);
+			} else {
+				store.setNum(store.getNum() + d.getNum());
+			}
+		}
+	}
+
+	private List<UnionClickCount> queryCountInfoByTime(int start, int end) {
+		return unionClickLogsDAO.queryCountInfoByTime(start, end);
+	}
+	
+	private Integer deleteCountInfoByTime(int start, int end) {
+		return unionClickCountDayMapper.deleteCountInfoByTime(start, end);
+	}
+	
+	private Integer getClientType(String clientType) {
+		if (clientType == null) {
+			return 0;
+		}
+		if (clientType.equals("android")) {
+			return 1;
+		}
+		if (clientType.equals("ios")) {
+			return 2;
+		}
+		return 0;
+	}
+}

@@ -68,6 +68,7 @@ datasources:
         - com.yoho.unions.dal.IChannelReportFormDAO
         - com.yoho.unions.dal.IChannelReportFormDAO
         - com.yoho.unions.dal.IMktReportFormDAO
         - com.yoho.unions.dal.IMktReportFormDAO
         - com.yoho.unions.dal.MktUnionCodeMapper
         - com.yoho.unions.dal.MktUnionCodeMapper
+        - com.yoho.unions.dal.UnionClickCountDayMapper
 
 
 
 
    bigdata_yh_unions:
    bigdata_yh_unions:

@@ -59,6 +59,7 @@ datasources:
         - com.yoho.unions.dal.IChannelReportFormDAO
         - com.yoho.unions.dal.IChannelReportFormDAO
         - com.yoho.unions.dal.IMktReportFormDAO
         - com.yoho.unions.dal.IMktReportFormDAO
         - com.yoho.unions.dal.MktUnionCodeMapper
         - com.yoho.unions.dal.MktUnionCodeMapper
+        - com.yoho.unions.dal.UnionClickCountDayMapper
 
 
 
 
    bigdata_yh_unions:
    bigdata_yh_unions: