datasync / yoho-datasync-fullsync · Commits

Go to a project

Authored by chaogeng
Commit 2b423d3ea843e21ef49b301a217951f6fd385468 2 parents 1adb073e a1c130df

Merge branch 'test6.8.9'

Inline Side-by-side
Showing 7 changed files with 101 additions and 19 deletions
... ... @@ -13,5 +13,5 @@ import java.util.List;
*/
public interface IPostsBlockRepository extends JpaRepository<PostsBlock, Integer> {
@Query(nativeQuery = true,value = " select * from posts_block where posts_id in :postIds order by order_by")
List<PostsBlock> selectTotal(@Param("postIds") List<Integer> postIds);
List<PostsBlock> selectByPostIds(@Param("postIds") List<Integer> postIds);
}
... ...
... ... @@ -30,8 +30,10 @@
<artifactId>commons-lang3</artifactId>
<version>3.4</version>
</dependency>
<dependency>
<groupId>com.yoho.quartz</groupId>
<artifactId>yoho-quartz-client</artifactId>
<version>1.3.8-SNAPSHOT</version>
</dependency>
</dependencies>
</project>
\ No newline at end of file
... ...
package com.yoho.datasync.fullsync.comm;
import java.io.File;
import java.util.*;
import org.apache.commons.lang3.StringUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.io.File;
import java.util.*;
/**
* 七牛云图片上传辅助工具类
... ... @@ -116,7 +116,7 @@ public class ImagesHelper {
//随机从两个域名中获取
String[] domains = DOMAIN_LIST.get(fileMode);
int domainMode = new Random().nextInt(2);
return "http://" + domains[domainMode] + File.separator + bucket + fileName;
return "http://" + domains[domainMode] + "/" + bucket + fileName;
}
... ...
... ... @@ -16,13 +16,13 @@ public class YohoNowDataSynController {
// private ITblAttentionUserRepository tblAttentionUserRepository;
@Resource
private YohoNowDataSynService yohoNowDataSynService;
// @RequestMapping("/test")
// public TblAttentionUser test(){
// TblAttentionUser user = tblAttentionUserRepository.getOne(99);
// TblAttentionUser t = new TblAttentionUser();
// BeanUtils.copyProperties(user,t);
// return t;
// }
@RequestMapping("/synYohoNow/switch")
public String setStop(@RequestParam("isStop") boolean isStop){
yohoNowDataSynService.setStop(isStop);
return "set switch is"+yohoNowDataSynService.getStop();
}
@RequestMapping("/synYohoNow/article")
public String synArticle(@RequestParam("begTime") long begTime,@RequestParam("endTime") long endTime){
return yohoNowDataSynService.synArticle(begTime,endTime);
... ...
package com.yoho.datasync.fullsync.quartz;
import com.yoho.datasync.fullsync.service.YohoNowDataSynService;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.beans.factory.annotation.Configurable;
import org.springframework.scheduling.annotation.EnableScheduling;
import org.springframework.scheduling.annotation.Scheduled;
import org.springframework.stereotype.Component;
import java.util.Calendar;
/**
* Created by shengguo.cai on 2019/3/28.
*/
@Component
@Configurable
@EnableScheduling
public class SynNowDataTask {
private final Logger logger = LoggerFactory.getLogger(SynNowDataTask.class);
@Autowired
private YohoNowDataSynService yohoNowDataSynService;
@Scheduled(cron = "0 0 2 * * ? ")
public void reportCurrentByCron(){
logger.info("enter SynNowDataScheduler.....");
boolean isStop = yohoNowDataSynService.getStop();
logger.info("current switch is {}",isStop);
if(isStop){
logger.info("SynNowDataScheduler clearSwitch is off ");
return ;
}
logger.info("SynNowDataScheduler is runinng ...");
try {
//同步昨天一天的数据
Calendar calendar = Calendar.getInstance();
calendar.add(Calendar.DATE,-1);
calendar.set(Calendar.HOUR,0);
calendar.set(Calendar.MINUTE,0);
calendar.set(Calendar.SECOND,0);
calendar.set(Calendar.MILLISECOND,0);
long begTime = calendar.getTimeInMillis();
calendar.set(Calendar.HOUR,23);
calendar.set(Calendar.MINUTE,59);
calendar.set(Calendar.SECOND,59);
calendar.set(Calendar.MILLISECOND,999);
long endTime = calendar.getTimeInMillis();
//先同步文章
yohoNowDataSynService.synArticle(begTime,endTime);
yohoNowDataSynService.synArticleAttach(begTime,endTime,0,false);
yohoNowDataSynService.synArticleAttach(begTime,endTime,1,false);
yohoNowDataSynService.synArticleAttach(begTime,endTime,2,false);
} catch (Exception e) {
logger.error("SynNowDataScheduler error {}", e);
}
}
}
... ...
... ... @@ -12,6 +12,8 @@ import com.yoho.datasync.fullsync.dal.repository.yhcms.model.TblAttentionUser;
import com.yoho.datasync.fullsync.dal.repository.yhcms.model.TblCommentInfo;
import com.yoho.datasync.fullsync.dal.repository.yhsocial.*;
import com.yoho.datasync.fullsync.dal.repository.yhsocial.model.*;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.stereotype.Service;
import org.springframework.util.CollectionUtils;
import org.springframework.util.StringUtils;
... ... @@ -61,9 +63,11 @@ public class YohoNowDataSynService {
private PublicUserAttentionRepository publicUserAttentionRepository;
@Resource
private ITblAttentionUserRepository tblAttentionUserRepository;
private final Logger logger = LoggerFactory.getLogger(YohoNowDataSynService.class);
@Resource(name = "entityManagerPrimary")
private EntityManager publicEntityManager;
public final int size=50;
private Boolean isStop =false;
public String synArticle(long begTime, long endTime) {
int total = postsRepository.selectTotal(begTime,endTime);
... ... @@ -74,10 +78,11 @@ public class YohoNowDataSynService {
}
List<Integer> postIds = postsList.stream().map(Posts::getId).collect(Collectors.toList());
List<PostsPraiseTotal> praiseTotals = postsPraiseRepository.selectTotal(postIds);
List<PostsBlock> postsBlocks = postsBlockRepository.selectTotal(postIds);
List<PostsBlock> postsBlocks = postsBlockRepository.selectByPostIds(postIds);
Map<Integer,List<PostsBlock>> postsBlockMap = BeanConvertUtils.listToListMap(postsBlocks,Integer.class,"postsId");
Map<Integer,PostsPraiseTotal> postsPraiseTotalMap = BeanConvertUtils.listToMap(praiseTotals,Integer.class,"postsId");
List<PublicArticle> pArticles = buildPublicArticles(postsList,postsBlockMap,postsPraiseTotalMap);
logger.info("before synArticle.size is {}",pArticles.size());
publicArticleRepository.saveAll(pArticles);
//文章审核表同步
synArticleAudit(pArticles,postsList);
... ... @@ -122,6 +127,7 @@ public class YohoNowDataSynService {
audit.setAuditStatus(2);//审核都是通过的
auditList.add(audit);
}
logger.info("before synArticleAudit.size is {}",auditList.size());
if(!auditList.isEmpty()){
publicArticleAuditRepository.saveAll(auditList);
}
... ... @@ -173,6 +179,7 @@ public class YohoNowDataSynService {
needSyncList.add(articleLabel);
});
}
logger.info("before synArticleLabel.size is {}",needSyncList.size());
if(!CollectionUtils.isEmpty(needSyncList)){
publicArticleLabelRepository.saveAll(needSyncList);
}
... ... @@ -219,6 +226,7 @@ public class YohoNowDataSynService {
}
}
}
logger.info("before synArticleBlock.size is {}",pBlocks.size());
if(!pBlocks.isEmpty()){
publicArticleBlockRepository.saveAll(pBlocks);
}
... ... @@ -276,7 +284,7 @@ public class YohoNowDataSynService {
// article.setFavoriteNum();//favoriteCount TODO
article.setStatus(1);//auth_status
article.setAuditStatus(2);//auth_status 与审核相关
article.setPublishTime(posts.getCreateTime());//create_time
article.setPublishTime(posts.getRevieweTime());//create_time
article.setCreateTime(posts.getCreateTime());//create_time
article.setUpdateTime(posts.getUpdateTime());//update_time
//设置封面
... ... @@ -335,6 +343,7 @@ public class YohoNowDataSynService {
List<Integer> fanUids = fanUsersMap.keySet().stream().collect(Collectors.toList());
List<Integer> filtedFanUids = communityUserRepository.selectRealUids(fanUids);
List<PublicUserAttention> userAttentions = buildSynAttentionAuthor(fanUsersMap,filtedFanUids);
logger.info("before synAttentionAuthorByArticle.size is {}",userAttentions.size());
if(!CollectionUtils.isEmpty(userAttentions)){
publicUserAttentionRepository.saveAll(userAttentions);
try{
... ... @@ -358,6 +367,7 @@ public class YohoNowDataSynService {
List<Integer> fanUids = fanUsersMap.keySet().stream().collect(Collectors.toList());
List<Integer> filtedFanUids = communityUserRepository.selectRealUids(fanUids);
List<PublicUserAttention> userAttentions = buildSynAttentionAuthor(fanUsersMap,filtedFanUids);
logger.info("before synAttentionAuthorByTime.size is {}",userAttentions.size());
if(!CollectionUtils.isEmpty(userAttentions)){
publicUserAttentionRepository.saveAll(userAttentions);
try{
... ... @@ -416,6 +426,7 @@ public class YohoNowDataSynService {
List<PostsPraise> praiseList = postsPraiseRepository.selectByPostIds(relateIds,i,size);
//将点赞数据同步到表grass_article_praise
List<PublicUserPraise> articlePraises = buildSynArticlePraises(articleMap,praiseList);
logger.info("before synPraiseDataByArticle.size is {}",articlePraises.size());
if(!articlePraises.isEmpty()){
publicUserPraiseRepository.saveAll(articlePraises);
try{
... ... @@ -442,6 +453,7 @@ public class YohoNowDataSynService {
//将点赞数据同步到表grass_article_praise
List<PublicUserPraise> articlePraises = buildSynArticlePraises(articleMap,praiseList);
logger.info("before synPraiseDataByTime.size is {}",articlePraises.size());
if(!articlePraises.isEmpty()){
publicUserPraiseRepository.saveAll(articlePraises);
try{
... ... @@ -482,6 +494,7 @@ public class YohoNowDataSynService {
if(CollectionUtils.isEmpty(rootComments)){
break;
}
logger.info("before synCommentDataByArticle.size is {}",rootComments.size());
List<PublicUserComment> parentArticleComments = batchSaveNewComment(articleMap,rootComments,null);
if(!CollectionUtils.isEmpty(parentArticleComments)){
List<Integer> parentIds = parentArticleComments.stream().map(comm->comm.getRelateId()).collect(Collectors.toList());
... ... @@ -510,7 +523,7 @@ public class YohoNowDataSynService {
List<Integer> articleRelateIds = rootComments.stream().map(TblCommentInfo::getContentId).collect(Collectors.toList());
List<PublicArticle> articleList = publicArticleRepository.selectByRelateIds(articleRelateIds);
Map<Integer,PublicArticle> articleMap = BeanConvertUtils.listToMap(articleList,Integer.class,"relateId");
logger.info("before synCommentDataByTime.size is {}",rootComments.size());
List<PublicUserComment> parentArticleComments = batchSaveNewComment(articleMap,rootComments,null);
if(!CollectionUtils.isEmpty(parentArticleComments)){
List<Integer> parentIds = parentArticleComments.stream().map(comm->comm.getRelateId()).collect(Collectors.toList());
... ... @@ -615,4 +628,12 @@ public class YohoNowDataSynService {
}
}
}
public Boolean getStop() {
return isStop;
}
public void setStop(Boolean stop) {
isStop = stop;
}
}
... ...
... ... @@ -3,7 +3,7 @@ package com.yoho.datasync.fullsync.starter;
import org.springframework.boot.SpringApplication;
import org.springframework.boot.autoconfigure.SpringBootApplication;
@SpringBootApplication(scanBasePackages = "com.yoho.datasync.fullsync")
@SpringBootApplication(scanBasePackages = {"com.yoho.datasync.fullsync"})
public class YohoFullsyncApplication {
public static void main(String[] args) {
... ...