datasync / yoho-datasync-fullsync · Commits

Go to a project

Authored by chaogeng
Commit 2b423d3ea843e21ef49b301a217951f6fd385468 2 parents 1adb073e a1c130df

Merge branch 'test6.8.9'

Inline Side-by-side
Showing 7 changed files with 101 additions and 19 deletions
@@ -13,5 +13,5 @@ import java.util.List; @@ -13,5 +13,5 @@ import java.util.List;
13 */ 13 */
14 public interface IPostsBlockRepository extends JpaRepository<PostsBlock, Integer> { 14 public interface IPostsBlockRepository extends JpaRepository<PostsBlock, Integer> {
15 @Query(nativeQuery = true,value = " select * from posts_block where posts_id in :postIds order by order_by") 15 @Query(nativeQuery = true,value = " select * from posts_block where posts_id in :postIds order by order_by")
16 - List<PostsBlock> selectTotal(@Param("postIds") List<Integer> postIds); 16 + List<PostsBlock> selectByPostIds(@Param("postIds") List<Integer> postIds);
17 } 17 }
@@ -30,8 +30,10 @@ @@ -30,8 +30,10 @@
30 <artifactId>commons-lang3</artifactId> 30 <artifactId>commons-lang3</artifactId>
31 <version>3.4</version> 31 <version>3.4</version>
32 </dependency> 32 </dependency>
  33 + <dependency>
  34 + <groupId>com.yoho.quartz</groupId>
  35 + <artifactId>yoho-quartz-client</artifactId>
  36 + <version>1.3.8-SNAPSHOT</version>
  37 + </dependency>
33 </dependencies> 38 </dependencies>
34 -  
35 -  
36 -  
37 </project> 39 </project>
1 package com.yoho.datasync.fullsync.comm; 1 package com.yoho.datasync.fullsync.comm;
2 2
3 -import java.io.File;  
4 -import java.util.*;  
5 -  
6 import org.apache.commons.lang3.StringUtils; 3 import org.apache.commons.lang3.StringUtils;
7 import org.slf4j.Logger; 4 import org.slf4j.Logger;
8 import org.slf4j.LoggerFactory; 5 import org.slf4j.LoggerFactory;
9 6
  7 +import java.io.File;
  8 +import java.util.*;
  9 +
10 10
11 /** 11 /**
12 * 七牛云图片上传辅助工具类 12 * 七牛云图片上传辅助工具类
@@ -116,7 +116,7 @@ public class ImagesHelper { @@ -116,7 +116,7 @@ public class ImagesHelper {
116 //随机从两个域名中获取 116 //随机从两个域名中获取
117 String[] domains = DOMAIN_LIST.get(fileMode); 117 String[] domains = DOMAIN_LIST.get(fileMode);
118 int domainMode = new Random().nextInt(2); 118 int domainMode = new Random().nextInt(2);
119 - return "http://" + domains[domainMode] + File.separator + bucket + fileName; 119 + return "http://" + domains[domainMode] + "/" + bucket + fileName;
120 } 120 }
121 121
122 122
@@ -16,13 +16,13 @@ public class YohoNowDataSynController { @@ -16,13 +16,13 @@ public class YohoNowDataSynController {
16 // private ITblAttentionUserRepository tblAttentionUserRepository; 16 // private ITblAttentionUserRepository tblAttentionUserRepository;
17 @Resource 17 @Resource
18 private YohoNowDataSynService yohoNowDataSynService; 18 private YohoNowDataSynService yohoNowDataSynService;
19 -// @RequestMapping("/test")  
20 -// public TblAttentionUser test(){  
21 -// TblAttentionUser user = tblAttentionUserRepository.getOne(99);  
22 -// TblAttentionUser t = new TblAttentionUser();  
23 -// BeanUtils.copyProperties(user,t);  
24 -// return t;  
25 -// } 19 +
  20 + @RequestMapping("/synYohoNow/switch")
  21 + public String setStop(@RequestParam("isStop") boolean isStop){
  22 + yohoNowDataSynService.setStop(isStop);
  23 + return "set switch is"+yohoNowDataSynService.getStop();
  24 + }
  25 +
26 @RequestMapping("/synYohoNow/article") 26 @RequestMapping("/synYohoNow/article")
27 public String synArticle(@RequestParam("begTime") long begTime,@RequestParam("endTime") long endTime){ 27 public String synArticle(@RequestParam("begTime") long begTime,@RequestParam("endTime") long endTime){
28 return yohoNowDataSynService.synArticle(begTime,endTime); 28 return yohoNowDataSynService.synArticle(begTime,endTime);
  1 +package com.yoho.datasync.fullsync.quartz;
  2 +
  3 +import com.yoho.datasync.fullsync.service.YohoNowDataSynService;
  4 +import org.slf4j.Logger;
  5 +import org.slf4j.LoggerFactory;
  6 +import org.springframework.beans.factory.annotation.Autowired;
  7 +import org.springframework.beans.factory.annotation.Configurable;
  8 +import org.springframework.scheduling.annotation.EnableScheduling;
  9 +import org.springframework.scheduling.annotation.Scheduled;
  10 +import org.springframework.stereotype.Component;
  11 +
  12 +import java.util.Calendar;
  13 +
  14 +/**
  15 + * Created by shengguo.cai on 2019/3/28.
  16 + */
  17 +@Component
  18 +@Configurable
  19 +@EnableScheduling
  20 +public class SynNowDataTask {
  21 + private final Logger logger = LoggerFactory.getLogger(SynNowDataTask.class);
  22 +
  23 + @Autowired
  24 + private YohoNowDataSynService yohoNowDataSynService;
  25 +
  26 + @Scheduled(cron = "0 0 2 * * ? ")
  27 + public void reportCurrentByCron(){
  28 + logger.info("enter SynNowDataScheduler.....");
  29 + boolean isStop = yohoNowDataSynService.getStop();
  30 + logger.info("current switch is {}",isStop);
  31 + if(isStop){
  32 + logger.info("SynNowDataScheduler clearSwitch is off ");
  33 + return ;
  34 + }
  35 + logger.info("SynNowDataScheduler is runinng ...");
  36 + try {
  37 + //同步昨天一天的数据
  38 + Calendar calendar = Calendar.getInstance();
  39 + calendar.add(Calendar.DATE,-1);
  40 + calendar.set(Calendar.HOUR,0);
  41 + calendar.set(Calendar.MINUTE,0);
  42 + calendar.set(Calendar.SECOND,0);
  43 + calendar.set(Calendar.MILLISECOND,0);
  44 + long begTime = calendar.getTimeInMillis();
  45 + calendar.set(Calendar.HOUR,23);
  46 + calendar.set(Calendar.MINUTE,59);
  47 + calendar.set(Calendar.SECOND,59);
  48 + calendar.set(Calendar.MILLISECOND,999);
  49 + long endTime = calendar.getTimeInMillis();
  50 + //先同步文章
  51 + yohoNowDataSynService.synArticle(begTime,endTime);
  52 + yohoNowDataSynService.synArticleAttach(begTime,endTime,0,false);
  53 + yohoNowDataSynService.synArticleAttach(begTime,endTime,1,false);
  54 + yohoNowDataSynService.synArticleAttach(begTime,endTime,2,false);
  55 + } catch (Exception e) {
  56 + logger.error("SynNowDataScheduler error {}", e);
  57 + }
  58 + }
  59 +}
@@ -12,6 +12,8 @@ import com.yoho.datasync.fullsync.dal.repository.yhcms.model.TblAttentionUser; @@ -12,6 +12,8 @@ import com.yoho.datasync.fullsync.dal.repository.yhcms.model.TblAttentionUser;
12 import com.yoho.datasync.fullsync.dal.repository.yhcms.model.TblCommentInfo; 12 import com.yoho.datasync.fullsync.dal.repository.yhcms.model.TblCommentInfo;
13 import com.yoho.datasync.fullsync.dal.repository.yhsocial.*; 13 import com.yoho.datasync.fullsync.dal.repository.yhsocial.*;
14 import com.yoho.datasync.fullsync.dal.repository.yhsocial.model.*; 14 import com.yoho.datasync.fullsync.dal.repository.yhsocial.model.*;
  15 +import org.slf4j.Logger;
  16 +import org.slf4j.LoggerFactory;
15 import org.springframework.stereotype.Service; 17 import org.springframework.stereotype.Service;
16 import org.springframework.util.CollectionUtils; 18 import org.springframework.util.CollectionUtils;
17 import org.springframework.util.StringUtils; 19 import org.springframework.util.StringUtils;
@@ -61,9 +63,11 @@ public class YohoNowDataSynService { @@ -61,9 +63,11 @@ public class YohoNowDataSynService {
61 private PublicUserAttentionRepository publicUserAttentionRepository; 63 private PublicUserAttentionRepository publicUserAttentionRepository;
62 @Resource 64 @Resource
63 private ITblAttentionUserRepository tblAttentionUserRepository; 65 private ITblAttentionUserRepository tblAttentionUserRepository;
  66 + private final Logger logger = LoggerFactory.getLogger(YohoNowDataSynService.class);
64 @Resource(name = "entityManagerPrimary") 67 @Resource(name = "entityManagerPrimary")
65 private EntityManager publicEntityManager; 68 private EntityManager publicEntityManager;
66 public final int size=50; 69 public final int size=50;
  70 + private Boolean isStop =false;
67 71
68 public String synArticle(long begTime, long endTime) { 72 public String synArticle(long begTime, long endTime) {
69 int total = postsRepository.selectTotal(begTime,endTime); 73 int total = postsRepository.selectTotal(begTime,endTime);
@@ -74,10 +78,11 @@ public class YohoNowDataSynService { @@ -74,10 +78,11 @@ public class YohoNowDataSynService {
74 } 78 }
75 List<Integer> postIds = postsList.stream().map(Posts::getId).collect(Collectors.toList()); 79 List<Integer> postIds = postsList.stream().map(Posts::getId).collect(Collectors.toList());
76 List<PostsPraiseTotal> praiseTotals = postsPraiseRepository.selectTotal(postIds); 80 List<PostsPraiseTotal> praiseTotals = postsPraiseRepository.selectTotal(postIds);
77 - List<PostsBlock> postsBlocks = postsBlockRepository.selectTotal(postIds); 81 + List<PostsBlock> postsBlocks = postsBlockRepository.selectByPostIds(postIds);
78 Map<Integer,List<PostsBlock>> postsBlockMap = BeanConvertUtils.listToListMap(postsBlocks,Integer.class,"postsId"); 82 Map<Integer,List<PostsBlock>> postsBlockMap = BeanConvertUtils.listToListMap(postsBlocks,Integer.class,"postsId");
79 Map<Integer,PostsPraiseTotal> postsPraiseTotalMap = BeanConvertUtils.listToMap(praiseTotals,Integer.class,"postsId"); 83 Map<Integer,PostsPraiseTotal> postsPraiseTotalMap = BeanConvertUtils.listToMap(praiseTotals,Integer.class,"postsId");
80 List<PublicArticle> pArticles = buildPublicArticles(postsList,postsBlockMap,postsPraiseTotalMap); 84 List<PublicArticle> pArticles = buildPublicArticles(postsList,postsBlockMap,postsPraiseTotalMap);
  85 + logger.info("before synArticle.size is {}",pArticles.size());
81 publicArticleRepository.saveAll(pArticles); 86 publicArticleRepository.saveAll(pArticles);
82 //文章审核表同步 87 //文章审核表同步
83 synArticleAudit(pArticles,postsList); 88 synArticleAudit(pArticles,postsList);
@@ -122,6 +127,7 @@ public class YohoNowDataSynService { @@ -122,6 +127,7 @@ public class YohoNowDataSynService {
122 audit.setAuditStatus(2);//审核都是通过的 127 audit.setAuditStatus(2);//审核都是通过的
123 auditList.add(audit); 128 auditList.add(audit);
124 } 129 }
  130 + logger.info("before synArticleAudit.size is {}",auditList.size());
125 if(!auditList.isEmpty()){ 131 if(!auditList.isEmpty()){
126 publicArticleAuditRepository.saveAll(auditList); 132 publicArticleAuditRepository.saveAll(auditList);
127 } 133 }
@@ -173,6 +179,7 @@ public class YohoNowDataSynService { @@ -173,6 +179,7 @@ public class YohoNowDataSynService {
173 needSyncList.add(articleLabel); 179 needSyncList.add(articleLabel);
174 }); 180 });
175 } 181 }
  182 + logger.info("before synArticleLabel.size is {}",needSyncList.size());
176 if(!CollectionUtils.isEmpty(needSyncList)){ 183 if(!CollectionUtils.isEmpty(needSyncList)){
177 publicArticleLabelRepository.saveAll(needSyncList); 184 publicArticleLabelRepository.saveAll(needSyncList);
178 } 185 }
@@ -219,6 +226,7 @@ public class YohoNowDataSynService { @@ -219,6 +226,7 @@ public class YohoNowDataSynService {
219 } 226 }
220 } 227 }
221 } 228 }
  229 + logger.info("before synArticleBlock.size is {}",pBlocks.size());
222 if(!pBlocks.isEmpty()){ 230 if(!pBlocks.isEmpty()){
223 publicArticleBlockRepository.saveAll(pBlocks); 231 publicArticleBlockRepository.saveAll(pBlocks);
224 } 232 }
@@ -276,7 +284,7 @@ public class YohoNowDataSynService { @@ -276,7 +284,7 @@ public class YohoNowDataSynService {
276 // article.setFavoriteNum();//favoriteCount TODO 284 // article.setFavoriteNum();//favoriteCount TODO
277 article.setStatus(1);//auth_status 285 article.setStatus(1);//auth_status
278 article.setAuditStatus(2);//auth_status 与审核相关 286 article.setAuditStatus(2);//auth_status 与审核相关
279 - article.setPublishTime(posts.getCreateTime());//create_time 287 + article.setPublishTime(posts.getRevieweTime());//create_time
280 article.setCreateTime(posts.getCreateTime());//create_time 288 article.setCreateTime(posts.getCreateTime());//create_time
281 article.setUpdateTime(posts.getUpdateTime());//update_time 289 article.setUpdateTime(posts.getUpdateTime());//update_time
282 //设置封面 290 //设置封面
@@ -335,6 +343,7 @@ public class YohoNowDataSynService { @@ -335,6 +343,7 @@ public class YohoNowDataSynService {
335 List<Integer> fanUids = fanUsersMap.keySet().stream().collect(Collectors.toList()); 343 List<Integer> fanUids = fanUsersMap.keySet().stream().collect(Collectors.toList());
336 List<Integer> filtedFanUids = communityUserRepository.selectRealUids(fanUids); 344 List<Integer> filtedFanUids = communityUserRepository.selectRealUids(fanUids);
337 List<PublicUserAttention> userAttentions = buildSynAttentionAuthor(fanUsersMap,filtedFanUids); 345 List<PublicUserAttention> userAttentions = buildSynAttentionAuthor(fanUsersMap,filtedFanUids);
  346 + logger.info("before synAttentionAuthorByArticle.size is {}",userAttentions.size());
338 if(!CollectionUtils.isEmpty(userAttentions)){ 347 if(!CollectionUtils.isEmpty(userAttentions)){
339 publicUserAttentionRepository.saveAll(userAttentions); 348 publicUserAttentionRepository.saveAll(userAttentions);
340 try{ 349 try{
@@ -358,6 +367,7 @@ public class YohoNowDataSynService { @@ -358,6 +367,7 @@ public class YohoNowDataSynService {
358 List<Integer> fanUids = fanUsersMap.keySet().stream().collect(Collectors.toList()); 367 List<Integer> fanUids = fanUsersMap.keySet().stream().collect(Collectors.toList());
359 List<Integer> filtedFanUids = communityUserRepository.selectRealUids(fanUids); 368 List<Integer> filtedFanUids = communityUserRepository.selectRealUids(fanUids);
360 List<PublicUserAttention> userAttentions = buildSynAttentionAuthor(fanUsersMap,filtedFanUids); 369 List<PublicUserAttention> userAttentions = buildSynAttentionAuthor(fanUsersMap,filtedFanUids);
  370 + logger.info("before synAttentionAuthorByTime.size is {}",userAttentions.size());
361 if(!CollectionUtils.isEmpty(userAttentions)){ 371 if(!CollectionUtils.isEmpty(userAttentions)){
362 publicUserAttentionRepository.saveAll(userAttentions); 372 publicUserAttentionRepository.saveAll(userAttentions);
363 try{ 373 try{
@@ -416,6 +426,7 @@ public class YohoNowDataSynService { @@ -416,6 +426,7 @@ public class YohoNowDataSynService {
416 List<PostsPraise> praiseList = postsPraiseRepository.selectByPostIds(relateIds,i,size); 426 List<PostsPraise> praiseList = postsPraiseRepository.selectByPostIds(relateIds,i,size);
417 //将点赞数据同步到表grass_article_praise 427 //将点赞数据同步到表grass_article_praise
418 List<PublicUserPraise> articlePraises = buildSynArticlePraises(articleMap,praiseList); 428 List<PublicUserPraise> articlePraises = buildSynArticlePraises(articleMap,praiseList);
  429 + logger.info("before synPraiseDataByArticle.size is {}",articlePraises.size());
419 if(!articlePraises.isEmpty()){ 430 if(!articlePraises.isEmpty()){
420 publicUserPraiseRepository.saveAll(articlePraises); 431 publicUserPraiseRepository.saveAll(articlePraises);
421 try{ 432 try{
@@ -442,6 +453,7 @@ public class YohoNowDataSynService { @@ -442,6 +453,7 @@ public class YohoNowDataSynService {
442 453
443 //将点赞数据同步到表grass_article_praise 454 //将点赞数据同步到表grass_article_praise
444 List<PublicUserPraise> articlePraises = buildSynArticlePraises(articleMap,praiseList); 455 List<PublicUserPraise> articlePraises = buildSynArticlePraises(articleMap,praiseList);
  456 + logger.info("before synPraiseDataByTime.size is {}",articlePraises.size());
445 if(!articlePraises.isEmpty()){ 457 if(!articlePraises.isEmpty()){
446 publicUserPraiseRepository.saveAll(articlePraises); 458 publicUserPraiseRepository.saveAll(articlePraises);
447 try{ 459 try{
@@ -482,6 +494,7 @@ public class YohoNowDataSynService { @@ -482,6 +494,7 @@ public class YohoNowDataSynService {
482 if(CollectionUtils.isEmpty(rootComments)){ 494 if(CollectionUtils.isEmpty(rootComments)){
483 break; 495 break;
484 } 496 }
  497 + logger.info("before synCommentDataByArticle.size is {}",rootComments.size());
485 List<PublicUserComment> parentArticleComments = batchSaveNewComment(articleMap,rootComments,null); 498 List<PublicUserComment> parentArticleComments = batchSaveNewComment(articleMap,rootComments,null);
486 if(!CollectionUtils.isEmpty(parentArticleComments)){ 499 if(!CollectionUtils.isEmpty(parentArticleComments)){
487 List<Integer> parentIds = parentArticleComments.stream().map(comm->comm.getRelateId()).collect(Collectors.toList()); 500 List<Integer> parentIds = parentArticleComments.stream().map(comm->comm.getRelateId()).collect(Collectors.toList());
@@ -510,7 +523,7 @@ public class YohoNowDataSynService { @@ -510,7 +523,7 @@ public class YohoNowDataSynService {
510 List<Integer> articleRelateIds = rootComments.stream().map(TblCommentInfo::getContentId).collect(Collectors.toList()); 523 List<Integer> articleRelateIds = rootComments.stream().map(TblCommentInfo::getContentId).collect(Collectors.toList());
511 List<PublicArticle> articleList = publicArticleRepository.selectByRelateIds(articleRelateIds); 524 List<PublicArticle> articleList = publicArticleRepository.selectByRelateIds(articleRelateIds);
512 Map<Integer,PublicArticle> articleMap = BeanConvertUtils.listToMap(articleList,Integer.class,"relateId"); 525 Map<Integer,PublicArticle> articleMap = BeanConvertUtils.listToMap(articleList,Integer.class,"relateId");
513 - 526 + logger.info("before synCommentDataByTime.size is {}",rootComments.size());
514 List<PublicUserComment> parentArticleComments = batchSaveNewComment(articleMap,rootComments,null); 527 List<PublicUserComment> parentArticleComments = batchSaveNewComment(articleMap,rootComments,null);
515 if(!CollectionUtils.isEmpty(parentArticleComments)){ 528 if(!CollectionUtils.isEmpty(parentArticleComments)){
516 List<Integer> parentIds = parentArticleComments.stream().map(comm->comm.getRelateId()).collect(Collectors.toList()); 529 List<Integer> parentIds = parentArticleComments.stream().map(comm->comm.getRelateId()).collect(Collectors.toList());
@@ -615,4 +628,12 @@ public class YohoNowDataSynService { @@ -615,4 +628,12 @@ public class YohoNowDataSynService {
615 } 628 }
616 } 629 }
617 } 630 }
  631 +
  632 + public Boolean getStop() {
  633 + return isStop;
  634 + }
  635 +
  636 + public void setStop(Boolean stop) {
  637 + isStop = stop;
  638 + }
618 } 639 }
@@ -3,7 +3,7 @@ package com.yoho.datasync.fullsync.starter; @@ -3,7 +3,7 @@ package com.yoho.datasync.fullsync.starter;
3 import org.springframework.boot.SpringApplication; 3 import org.springframework.boot.SpringApplication;
4 import org.springframework.boot.autoconfigure.SpringBootApplication; 4 import org.springframework.boot.autoconfigure.SpringBootApplication;
5 5
6 -@SpringBootApplication(scanBasePackages = "com.yoho.datasync.fullsync") 6 +@SpringBootApplication(scanBasePackages = {"com.yoho.datasync.fullsync"})
7 public class YohoFullsyncApplication { 7 public class YohoFullsyncApplication {
8 8
9 public static void main(String[] args) { 9 public static void main(String[] args) {