OPENTECH / yoho-node-ci · Commits

Go to a project

Authored by 郝肖肖
Commit 56549661d009998759059dc7148f37d7e2845c09 2 parents c5b26395 1e37d758

Merge branch 'feature/seo' into 'master' 

随机关联词和显示小分类及大分类



See merge request !4
Inline Side-by-side
Showing 3 changed files with 150 additions and 47 deletions
@@ -99,6 +99,35 @@ const mgets = async(ctx, item) => { @@ -99,6 +99,35 @@ const mgets = async(ctx, item) => {
99 }); 99 });
100 }; 100 };
101 101
  102 +//通过小分类同步中分类和大分类
  103 +const getMsort = async(ctx) => {
  104 + let redis = ctx.redis;
  105 +
  106 + return redis.getAsync(`global:yoho:sorts`).then((sorts) => {
  107 + let sData = {};
  108 + sorts = JSON.parse(sorts) || [];
  109 +
  110 + _.forEach(_.get(sorts, 'data.sort'), (msort) => {
  111 + _.forEach(_.get(msort, 'sub'), (misort) => {
  112 + _.forEach(_.get(misort, 'sub'), (sort) => {
  113 + if (!sData[sort.sort_id]) {
  114 + sData[sort.sort_id] = {
  115 + msort: msort.sort_id,
  116 + msort_name: msort.sort_name,
  117 + misort: misort.sort_id,
  118 + misort_name: misort.sort_name,
  119 + sort_id: sort.sort_id,
  120 + sort_name: sort.sort_name,
  121 + };
  122 + }
  123 + })
  124 + })
  125 + })
  126 +
  127 + return sData;
  128 + });
  129 +};
  130 +
102 r.get('/', async(ctx) => { 131 r.get('/', async(ctx) => {
103 let resData = {}; 132 let resData = {};
104 let q = ctx.request.query, 133 let q = ctx.request.query,
@@ -301,10 +330,6 @@ r.get('/expand', async(ctx) => { @@ -301,10 +330,6 @@ r.get('/expand', async(ctx) => {
301 resData.pager = pager(Math.floor((total - 1) / limit) + 1, ctx.query); 330 resData.pager = pager(Math.floor((total - 1) / limit) + 1, ctx.query);
302 331
303 _.each(d, (val) => { 332 _.each(d, (val) => {
304 - if (val.sort_id) {  
305 - sortIds.push(`${singleSortKeyPre}${val.sort_id}`);  
306 - }  
307 -  
308 if (val.brand_id) { 333 if (val.brand_id) {
309 brandIds.push(`${singleBrandKeyPre}${val.brand_id}`); 334 brandIds.push(`${singleBrandKeyPre}${val.brand_id}`);
310 } 335 }
@@ -315,14 +340,16 @@ r.get('/expand', async(ctx) => { @@ -315,14 +340,16 @@ r.get('/expand', async(ctx) => {
315 }); 340 });
316 341
317 brandIds = await mgets(ctx, brandIds); 342 brandIds = await mgets(ctx, brandIds);
318 - sortIds = await mgets(ctx, sortIds); 343 + sortIds = await getMsort(ctx);
319 rootIds = await getRootKeyword(rootIds, mysql); 344 rootIds = await getRootKeyword(rootIds, mysql);
320 345
321 resData.tabs = _.map(d, (elem) => { 346 resData.tabs = _.map(d, (elem) => {
322 return Object.assign({}, elem, { 347 return Object.assign({}, elem, {
323 root: rootIds[elem.root_id], 348 root: rootIds[elem.root_id],
324 brand: brandIds[`${singleBrandKeyPre}${elem.brand_id}`], 349 brand: brandIds[`${singleBrandKeyPre}${elem.brand_id}`],
325 - sort: sortIds[`${singleSortKeyPre}${elem.sort_id}`], 350 + sort: sortIds[elem.sort_id] && sortIds[elem.sort_id].sort_name,
  351 + misort: sortIds[elem.sort_id] && sortIds[elem.sort_id].misort_name,
  352 + msort: sortIds[elem.sort_id] && sortIds[elem.sort_id].msort_name,
326 is_push: elem.is_push ? '是' : '否', 353 is_push: elem.is_push ? '是' : '否',
327 add_time: elem.add_time && moment(elem.add_time * 1000).format('YYYY-MM-DD HH:mm'), 354 add_time: elem.add_time && moment(elem.add_time * 1000).format('YYYY-MM-DD HH:mm'),
328 }); 355 });
@@ -372,7 +399,7 @@ r.post('/expand/del', async(ctx) => { @@ -372,7 +399,7 @@ r.post('/expand/del', async(ctx) => {
372 * @return {[type]} [description] 399 * @return {[type]} [description]
373 */ 400 */
374 r.post('/expand/randWords', async(ctx) => { 401 r.post('/expand/randWords', async(ctx) => {
375 - let key = `golobal:yoho:seo:keywords:isRun`; 402 + let key = `global:yoho:seo:keywords:isRun`;
376 403
377 return ctx.redis.getAsync(key).then((d) => { 404 return ctx.redis.getAsync(key).then((d) => {
378 405
@@ -6,10 +6,10 @@ const MysqlPromise = require('../../../lib/mysql-promise'); @@ -6,10 +6,10 @@ const MysqlPromise = require('../../../lib/mysql-promise');
6 const Promise = require('bluebird'); 6 const Promise = require('bluebird');
7 7
8 const keywordsRedis = { 8 const keywordsRedis = {
9 - 'keywordsAllIds': `golobal:yoho:seo:keywords:allIds`,// 关键词表符合条件所有id  
10 - 'keywordsSortId': `golobal:yoho:seo:keywords:sortId`,// 分类下的所有关键词  
11 - 'keywordsId': `golobal:yoho:seo:keywords:id`,// 每条关键词关联其它12条关键词  
12 - 'keywordsIsRun': `golobal:yoho:seo:keywords:isRun`,// 判断是否正在执行中 9 + 'keywordsAllIds': `global:yoho:seo:keywords:allIds`,// 关键词表符合条件所有id
  10 + 'keywordsSortId': `global:yoho:seo:keywords:sortId`,// 分类下的所有关键词
  11 + 'keywordsId': `global:yoho:seo:keywords:id`,// 每条关键词关联其它12条关键词
  12 + 'keywordsIsRun': `global:yoho:seo:keywords:isRun`,// 判断是否正在执行中
13 }; 13 };
14 14
15 class expandModel extends model { 15 class expandModel extends model {
@@ -22,59 +22,131 @@ class expandModel extends model { @@ -22,59 +22,131 @@ class expandModel extends model {
22 } 22 }
23 23
24 getRanData() { 24 getRanData() {
25 - this.dataIds = [];  
26 return this.redis.setAsync(`${keywordsRedis.keywordsIsRun}`, 1).then(() => {// 设置正在运行的key 25 return this.redis.setAsync(`${keywordsRedis.keywordsIsRun}`, 1).then(() => {// 设置正在运行的key
27 - return this.redis.expireAsync(`${keywordsRedis.keywordsIsRun}`, 5 * 60);// 设置正在运行的key有效期 26 + return this.redis.expireAsync(`${keywordsRedis.keywordsIsRun}`, 10800);// 设置正在运行的key有效期
28 }).then(() => { 27 }).then(() => {
29 - return this.mysql.query(`SELECT sort_id FROM seo_keywords WHERE status = 1 GROUP BY sort_id;`); 28 + return this.sortIdInit();
30 }).then(rdata => { 29 }).then(rdata => {
31 - return Promise.each(rdata, (item) => this._setSortToRedis(item.sort_id));  
32 - }).then(d => {  
33 - return this.redis.setAsync(`${keywordsRedis.keywordsAllIds}`, JSON.stringify(this.dataIds));  
34 - }).then(d => {  
35 -  
36 - this.redis.delAsync(`${keywordsRedis.keywordsIsRun}`); 30 + return this.msortInit();
  31 + }).then(() => {
  32 + return this.allIdsExec(1);
  33 + }).then(() => {
  34 + return this.redis.delAsync(`${keywordsRedis.keywordsIsRun}`);
  35 + }).then(() => {
  36 + return true;
  37 + });
  38 + }
37 39
38 - this.dataIds = [];  
39 - return d; 40 + //处理小分类function
  41 + sortIdInit() {
  42 + return this.mysql.query(`SELECT sort_id FROM seo_keywords WHERE status = 1 AND sort_id > 0 GROUP BY sort_id;`).then(rdata => {
  43 + return Promise.each(rdata, (item) => this.sortIdExec(item.sort_id, 1));
40 }); 44 });
41 } 45 }
  46 + sortIdExec(sortId, page) {
  47 + let pageSize = 1000;
  48 + let pageStart = (page - 1) * pageSize;
  49 +
  50 + return this.sleep(5000).then(() => {
  51 + return this.mysql.query(`SELECT id, keyword, root_id, brand_id, sort_id FROM seo_keywords WHERE status = 1 AND yoho_goods_num > 3 AND sort_id=${sortId} LIMIT ${pageStart}, ${pageSize};`);
  52 + }).then(dsort => {
  53 + if (dsort.length <= 0) {
  54 + return false;
  55 + }
42 56
43 - _setSortToRedis(sortId) {  
44 - return this.mysql.query(`SELECT id, keyword, root_id, brand_id, sort_id FROM seo_keywords WHERE status = 1 AND yoho_goods_num > 3 AND sort_id=${sortId};`).then(dsort => {  
45 - let len = dsort.length;  
46 - let key;  
47 - let tdata; 57 + return this.execRandData(dsort).then(() => {
  58 + return this.redis.setAsync(`${keywordsRedis.keywordsSortId}:${sortId}:page:${page}`, JSON.stringify(dsort));
  59 + }).then(() => {
  60 + dsort = [];
  61 + return this.sortIdExec(sortId, ++page);
  62 + });
  63 + });
  64 + }
48 65
49 - if (len <= 0) {  
50 - return Promise.resolve(true); 66 + //处理小分类为0且大分类>=0的function
  67 + msortInit() {
  68 + return this.mysql.query(`SELECT msort FROM seo_keywords WHERE status = 1 AND sort_id = 0 GROUP BY msort;`).then(rdata => {
  69 + return Promise.each(rdata, (item) => this.msortExec(item.msort, 1));
  70 + });
  71 + }
  72 + msortExec(sortId, page) {
  73 + let pageSize = 1000;
  74 + let pageStart = (page - 1) * pageSize;
  75 + return this.sleep(5000).then(() => {
  76 + return this.mysql.query(`SELECT id, keyword, root_id, brand_id, sort_id FROM seo_keywords WHERE status = 1 AND sort_id = 0 AND yoho_goods_num > 3 AND msort=${sortId} LIMIT ${pageStart}, ${pageSize};`);
  77 + }).then(dsort => {
  78 + if (dsort.length <= 0) {
  79 + return false;
51 } 80 }
52 81
53 - return this.redis.setAsync(`${keywordsRedis.keywordsSortId}:${sortId}`, JSON.stringify(dsort)).then(() => {  
54 - return Promise.each(dsort, (el, index) => {  
55 - key = `${keywordsRedis.keywordsId}:${el.id}`; 82 + /**
  83 + * 1、小分类为0且大分类大于0，要随机关联12词
  84 + * 2、小分类为0且大分类小于0，不要随机关联12个词。即大分类和小分类都为0的情况，data: [];
  85 + */
  86 + return this.execRandData(dsort).then(() => {
  87 + dsort = [];
  88 + return this.msortExec(sortId, ++page);
  89 + });
  90 + });
  91 + }
  92 +
  93 + //随机关联12个关键词
  94 + execRandData(dsort) {
  95 + let len = dsort.length;
  96 + let key;
  97 + let tdata;
  98 +
  99 + if (len <= 0) {
  100 + return Promise.resolve(false);
  101 + }
  102 +
  103 + return Promise.each(dsort, (el, index) => {
  104 + key = `${keywordsRedis.keywordsId}:${el.id}`;
  105 +
  106 + tdata = {
  107 + name: el.keyword,
  108 + data: _.compact(_.map(this.getRandom(dsort, index), (mval) => {
  109 + return dsort[mval];
  110 + })) || []
  111 + };
  112 +
  113 + console.log(`execRandData, index: ${index}, len: ${len}, id: ${el.id}, keywords: ${el.keyword}, key: ${key}`);
  114 + return this.redis.setAsync(key, JSON.stringify(tdata));
  115 + });
  116 + }
  117 +
  118 + getRandom(dsort, index) {
  119 + return _.shuffle(_.difference(_.keys(dsort), [`${index}`])).slice(0, 12);
  120 + }
  121 +
  122 + //符合条件和关键词设置redis中
  123 + allIdsExec(page) {
  124 + let pageSize = 40000;
  125 + let pageStart = (page - 1) * pageSize;
56 126
57 - this.dataIds.push(el.id); 127 + return this.mysql.query(`SELECT id FROM seo_keywords WHERE status = 1 AND yoho_goods_num > 3 LIMIT ${pageStart}, ${pageSize};`).then(dsort => {
  128 + if (dsort.length <= 0) {
  129 + return false;
  130 + }
58 131
59 - tdata = {  
60 - name: el.keyword,  
61 - data: _.compact(_.map(this._getRandom(dsort, index), (mval) => {  
62 - return dsort[mval];  
63 - }))  
64 - }; 132 + let ids = _.map(dsort, (sort) => {
  133 + return sort.id;
  134 + });
65 135
66 - console.log(`setSortToRedis, index: ${index}, len: ${len}, id: ${el.id}, keywords: ${el.keyword}, key: ${key}`);  
67 - return this.redis.setAsync(key, JSON.stringify(tdata));  
68 - });  
69 - }).then((d) => { 136 + return this.redis.setAsync(`${keywordsRedis.keywordsAllIds}:page:${page}`, JSON.stringify(ids)).then(() => {
70 dsort = []; 137 dsort = [];
71 - return true; 138 + ids = [];
  139 + return this.allIdsExec(++page);
72 }); 140 });
73 }); 141 });
74 } 142 }
75 143
76 - _getRandom(dsort, index) {  
77 - return _.shuffle(_.difference(_.keys(dsort), [`${index}`])).slice(0, 12); 144 + sleep(time) {
  145 + return new Promise((resolve, reject) => {
  146 + setTimeout(() => {
  147 + resolve();
  148 + }, time);
  149 + });
78 } 150 }
79 } 151 }
80 152
@@ -69,7 +69,8 @@ @@ -69,7 +69,8 @@
69 <th>关键词</th> 69 <th>关键词</th>
70 <th>词根</th> 70 <th>词根</th>
71 <th>品牌</th> 71 <th>品牌</th>
72 - <th>品类</th> 72 + <th>小品类</th>
  73 + <th>大品类</th>
73 <th>商品数</th> 74 <th>商品数</th>
74 <th>是否推送</th> 75 <th>是否推送</th>
75 <th>添加时间</th> 76 <th>添加时间</th>
@@ -94,6 +95,9 @@ @@ -94,6 +95,9 @@
94 {{sort}} 95 {{sort}}
95 </td> 96 </td>
96 <td> 97 <td>
  98 + {{msort}}
  99 + </td>
  100 + <td>
97 {{yoho_goods_num}} 101 {{yoho_goods_num}}
98 </td> 102 </td>
99 <td> 103 <td>