fe / yoho-seo · Commits

Go to a project

Authored by 周少峰
Commit fe9dbe057f0b8c5cd9b92b74b36a07461d0162bd 1 parent cd3791b3

keywords page

Inline Side-by-side
Showing 2 changed files with 162 additions and 164 deletions
1 'use strict'; 1 'use strict';
2 2
3 -const api = global.yoho.API;  
4 -const rp = require('request-promise');  
5 -const serviceApi = global.yoho.ServiceAPI;  
6 -const Promise = require('bluebird');  
7 -const co = Promise.coroutine;  
8 -const _ = require('lodash');  
9 -const logger = global.yoho.logger;  
10 -const helper = global.yoho.helpers;  
11 -const config = require('../config/config');  
12 const schedule = require('node-schedule'); 3 const schedule = require('node-schedule');
13 -const qs = require('querystring');  
14 const seoModel = require('./seoModel'); 4 const seoModel = require('./seoModel');
15 5
16 -const baiduUrls = {  
17 - urls: 'http://data.zz.baidu.com/urls',  
18 - update: 'http://data.zz.baidu.com/update',  
19 - del: 'http://data.zz.baidu.com/del'  
20 -};  
21 -  
22 -const siteUrls = {  
23 - pcProduct: {  
24 - site: 'https://item.yohobuy.com',  
25 - url: []  
26 - },  
27 - pcGuang: {  
28 - site: 'https://guang.yohobuy.com',  
29 - url: []  
30 - },  
31 - mProduct: {  
32 - site: 'https://m.yohobuy.com',  
33 - url: []  
34 - },  
35 - mGuang: {  
36 - site: 'https://guang.m.yohobuy.com',  
37 - url: [],  
38 - type: 'mip'  
39 - }  
40 -  
41 -};  
42 -  
43 -  
44 /** 6 /**
45 - * 获取最新1000条商品详情链接和逛详情链接 7 + * 同步建议词（把接口拓展的建议词同步到灰度redis）
46 */ 8 */
47 -const getUrls = () => {  
48 - let apiArr = [api.get('', {method: 'web.product.bdPromotion'}),  
49 - serviceApi.get('/guang/api/v2/article/getLastArticleList', {limit: 100})];  
50 -  
51 - return api.all(apiArr).spread((productData, articleData) => {  
52 -  
53 - _.forEach(_.get(productData, 'data', {}), value => {  
54 - siteUrls.pcProduct.url.push('https:' + helper.urlFormat(`/${value.id}.html`, null, 'item'));  
55 - siteUrls.mProduct.url.push('https:' + helper.urlFormat(`/product/${value.id}.html`, null, 'm'));  
56 - });  
57 -  
58 - _.forEach(_.get(articleData, 'data.artList', {}), value => {  
59 - siteUrls.pcGuang.url.push('https:' + helper.urlFormat(`/${value.articleId}.html`, null, 'guang'));  
60 - siteUrls.mGuang.url.push('https:' + helper.urlFormat(`/mip/guang/info/${value.articleId}.html`  
61 -, null, 'guang.m'));  
62 - });  
63 -  
64 - return siteUrls;  
65 - }); 9 +const synchronousKeywords = (req, res) => {
  10 + seoModel.synchronousKeywords();
  11 + res.end();
66 }; 12 };
67 13
68 /** 14 /**
69 - * 将链接推送到百度站长  
70 - * @param params object {site: 'https://www.yohobuy.com', type: 'mip'} 默认不需要type  
71 - * @param urls 15 + * 定时缓慢爬取关键词页面生成缓存，防止蜘蛛爬取
72 */ 16 */
73 -const sendUrlsToBaidu = (params, urls) => {  
74 - let paramsDef = {  
75 - token: config.baiduToken  
76 - };  
77 -  
78 - // 过滤无效的参数  
79 - _.forEach(params, (val, key) => {  
80 - if (!val) {  
81 - delete params[key];  
82 - }  
83 - });  
84 -  
85 - qs.escape = (str) => {  
86 - return str;  
87 - };  
88 17
89 - let options = {  
90 - url: `${baiduUrls.urls}?${qs.stringify(Object.assign(paramsDef, params), null, null, {})}`,  
91 - headers: {  
92 - 'Content-Type': 'text/plain'  
93 - },  
94 - method: 'post',  
95 - form: urls.join('\n'),  
96 - json: true,  
97 - timeout: 10000,  
98 - gzip: true  
99 - };  
100 -  
101 - return rp(options).then(result => {  
102 - logger.info(Object.assign(params, result, {length: urls.length}));  
103 - });  
104 -};  
105 18
106 /** 19 /**
107 - * 获取最新商品详情1000条和逛详情100条推送到相应的站点域名（pc和wap） 20 + * 向百度推送页面关键词静态页面
108 */ 21 */
109 -const sendUrls = () => {  
110 -  
111 - co(function*() {  
112 - // 获取pc/wap的商品详情和逛的链接  
113 - let sendArr = [],  
114 - urls = yield getUrls();  
115 -  
116 - _.forEach(urls, value => {  
117 - sendArr.push(sendUrlsToBaidu({site: value.site, type: value.type}, value.url));  
118 - });  
119 -  
120 - // 推送url  
121 - api.all(sendArr);  
122 - })();  
123 -};  
124 -  
125 -// 同步建议词（把接口拓展的建议词同步到灰度redis）  
126 -const synchronousKeywords = (req, res) => {  
127 - seoModel.synchronousKeywords();  
128 - res.end();  
129 -};  
130 -  
131 -// 定时缓慢爬取关键词页面生成缓存，防止蜘蛛爬取  
132 -  
133 -  
134 -// 向百度推送页面新的页面  
135 const sendKeywordsUrls = (req, res) => { 22 const sendKeywordsUrls = (req, res) => {
136 seoModel.sendKeywordsUrls(); 23 seoModel.sendKeywordsUrls();
137 res.end(); 24 res.end();
138 -  
139 }; 25 };
140 26
141 /** 27 /**
142 - * 定时每天1点推送最新商品和文章，更新站点sitemap 28 + * 商品详情，逛推送
  29 + */
  30 +const sendUrls = () => {
  31 + seoModel.sendUrls();
  32 +}
  33 +
  34 +/**
  35 + * 定时任务
143 */ 36 */
144 const start = () => { 37 const start = () => {
145 schedule.scheduleJob('0 0 1 * * *', function() { 38 schedule.scheduleJob('0 0 1 * * *', function() {
  39 +
  40 + // 推送最新的商品详情和逛文章
146 sendUrls(); 41 sendUrls();
  42 +
  43 + // 同步关键词
  44 + synchronousKeywords();
  45 + });
  46 +
  47 + schedule.scheduleJob('0 0 2 * * *', function() {
  48 +
  49 + // 推送关键词页面
  50 + sendKeywordsUrls();
147 }); 51 });
148 }; 52 };
149 53
150 module.exports = { 54 module.exports = {
151 start, 55 start,
152 - sendUrls,  
153 synchronousKeywords, 56 synchronousKeywords,
154 sendKeywordsUrls 57 sendKeywordsUrls
155 }; 58 };
1 'use strict'; 1 'use strict';
2 const api = global.yoho.API; 2 const api = global.yoho.API;
3 const redis = global.yoho.redis; 3 const redis = global.yoho.redis;
  4 +const rp = require('request-promise');
  5 +const serviceApi = global.yoho.ServiceAPI;
  6 +const Promise = require('bluebird');
  7 +const co = Promise.coroutine;
4 const _ = require('lodash'); 8 const _ = require('lodash');
  9 +const logger = global.yoho.logger;
  10 +const helper = global.yoho.helpers;
  11 +const config = require('../config/config');
  12 +const qs = require('querystring');
  13 +
5 14
6 /** 15 /**
7 * redis multi command 16 * redis multi command
@@ -12,44 +21,126 @@ const multiAsync = (multi)=>{ @@ -12,44 +21,126 @@ const multiAsync = (multi)=>{
12 }); 21 });
13 }; 22 };
14 23
  24 +const baiduUrls = {
  25 + urls: 'http://data.zz.baidu.com/urls',
  26 + update: 'http://data.zz.baidu.com/update',
  27 + del: 'http://data.zz.baidu.com/del'
  28 +};
  29 +
  30 +const siteUrls = {
  31 + pcProduct: {
  32 + site: 'https://item.yohobuy.com',
  33 + url: []
  34 + },
  35 + pcGuang: {
  36 + site: 'https://guang.yohobuy.com',
  37 + url: []
  38 + },
  39 + mProduct: {
  40 + site: 'https://m.yohobuy.com',
  41 + url: []
  42 + },
  43 + mGuang: {
  44 + site: 'https://guang.m.yohobuy.com',
  45 + url: [],
  46 + type: 'mip'
  47 + }
  48 +
  49 +};
  50 +
  51 +
  52 +
  53 +// 配置
  54 +const redisKey = {
  55 + keywordsList: 'keywords_mana_list', // 关键词列表
  56 +}
  57 +
15 /** 58 /**
16 * 将链接推送到百度站长 59 * 将链接推送到百度站长
17 - * @param site string 站点  
18 - * @param urls object {site: 'https://www.yohobuy.com', type: 'mip'} 默认不需要type 60 + * @param params object {site: 'https://www.yohobuy.com', type: 'mip'} 默认不需要type
  61 + * @param urls
19 */ 62 */
20 -// const sendUrlsToBaidu = (params, urls) => {  
21 -// let paramsDef = {  
22 -// token: config.baiduToken  
23 -// };  
24 -//  
25 -// // 过滤无效的参数  
26 -// _.forEach(params, (val, key) => {  
27 -// if (!val) {  
28 -// delete params[key];  
29 -// }  
30 -// });  
31 -//  
32 -// qs.escape = (str) => {  
33 -// return str;  
34 -// };  
35 -//  
36 -// let options = {  
37 -// url: `${baiduUrls.urls}?${qs.stringify(Object.assign(paramsDef, params), null, null, {})}`,  
38 -// headers: {  
39 -// 'Content-Type': 'text/plain'  
40 -// },  
41 -// method: 'post',  
42 -// form: urls.join('\n'),  
43 -// json: true,  
44 -// timeout: 10000,  
45 -// gzip: true  
46 -// };  
47 -//  
48 -// return rp(options).then(result => {  
49 -// logger.info(result);  
50 -// });  
51 -// }; 63 +const sendUrlsToBaidu = (params, urls) => {
  64 + let paramsDef = {
  65 + token: config.baiduToken
  66 + };
  67 +
  68 + // 过滤无效的参数
  69 + _.forEach(params, (val, key) => {
  70 + if (!val) {
  71 + delete params[key];
  72 + }
  73 + });
  74 +
  75 + qs.escape = (str) => {
  76 + return str;
  77 + };
  78 +
  79 + let options = {
  80 + url: `${baiduUrls.urls}?${qs.stringify(Object.assign(paramsDef, params), null, null, {})}`,
  81 + headers: {
  82 + 'Content-Type': 'text/plain'
  83 + },
  84 + method: 'post',
  85 + form: urls.join('\n'),
  86 + json: true,
  87 + timeout: 10000,
  88 + gzip: true
  89 + };
52 90
  91 + return rp(options).then(result => {
  92 + logger.info(Object.assign(params, result, {length: urls.length}));
  93 + });
  94 +};
  95 +
  96 +/**
  97 + * 获取最新1000条商品详情链接和逛详情链接
  98 + */
  99 +const getUrls = () => {
  100 + let apiArr = [api.get('', {method: 'web.product.bdPromotion'}),
  101 + serviceApi.get('/guang/api/v2/article/getLastArticleList', {limit: 100})];
  102 +
  103 + return api.all(apiArr).spread((productData, articleData) => {
  104 +
  105 + _.forEach(_.get(productData, 'data', {}), value => {
  106 + siteUrls.pcProduct.url.push('https:' + helper.urlFormat(`/${value.id}.html`, null, 'item'));
  107 + siteUrls.mProduct.url.push('https:' + helper.urlFormat(`/product/${value.id}.html`, null, 'm'));
  108 + });
  109 +
  110 + _.forEach(_.get(articleData, 'data.artList', {}), value => {
  111 + siteUrls.pcGuang.url.push('https:' + helper.urlFormat(`/${value.articleId}.html`, null, 'guang'));
  112 + siteUrls.mGuang.url.push('https:' + helper.urlFormat(`/mip/guang/info/${value.articleId}.html`
  113 + , null, 'guang.m'));
  114 + });
  115 +
  116 + return siteUrls;
  117 + });
  118 +};
  119 +
  120 +/**
  121 + * 发送最新商品详情1000条和逛详情100条推送到相应的站点域名（pc和wap）
  122 + */
  123 +const sendUrls = () => {
  124 +
  125 + co(function*() {
  126 + // 获取pc/wap的商品详情和逛的链接
  127 + let sendArr = [],
  128 + urls = yield getUrls();
  129 +
  130 + _.forEach(urls, value => {
  131 + sendArr.push(sendUrlsToBaidu({site: value.site, type: value.type}, value.url));
  132 + });
  133 +
  134 + // 推送url
  135 + api.all(sendArr);
  136 + })();
  137 +};
  138 +
  139 +
  140 +
  141 +/**
  142 + * 调用接口建议词
  143 + */
53 const getKeywordsApi = (page, limit) => { 144 const getKeywordsApi = (page, limit) => {
54 let params = { 145 let params = {
55 page: page || 1, 146 page: page || 1,
@@ -68,6 +159,7 @@ const synchronousKeywords = () => { @@ -68,6 +159,7 @@ const synchronousKeywords = () => {
68 return getKeywordsApi(1, 1).then(res => { 159 return getKeywordsApi(1, 1).then(res => {
69 let start = 0, 160 let start = 0,
70 page = 1, 161 page = 1,
  162 + intervalTime = 1000, // 循环调用的时间间隔
71 limit = 1000, // 每次请求接口关键词数量 163 limit = 1000, // 每次请求接口关键词数量
72 total = _.get(res, 'data.total', 0); 164 total = _.get(res, 'data.total', 0);
73 165
@@ -102,7 +194,7 @@ const synchronousKeywords = () => { @@ -102,7 +194,7 @@ const synchronousKeywords = () => {
102 clearInterval(interval); 194 clearInterval(interval);
103 }); 195 });
104 196
105 - }, 1000); 197 + }, intervalTime);
106 198
107 }); 199 });
108 }; 200 };
@@ -113,13 +205,14 @@ const synchronousKeywords = () => { @@ -113,13 +205,14 @@ const synchronousKeywords = () => {
113 */ 205 */
114 206
115 const getRedisKeywords = (start, end) => { 207 const getRedisKeywords = (start, end) => {
116 - return redis.lrangeAsync('keywords_mana_list', start, end).then(res => {  
117 - let urls = []; 208 + return redis.lrangeAsync(redisKey.keywordsList, start, end).then(res => {
  209 + let urls = {pc:[], wap: []};
118 210
119 _.forEach(res, keyword => { 211 _.forEach(res, keyword => {
120 let buff = new Buffer(keyword).toString('hex').toUpperCase(); 212 let buff = new Buffer(keyword).toString('hex').toUpperCase();
121 213
122 - urls.push(`https://www.yohobuy.com/so/${buff}.html`); 214 + urls.pc.push(`https://www.yohobuy.com/so/${buff}.html`);
  215 + urls.wap.push(`https://m.yohobuy.com/so/${buff}.html`);
123 }); 216 });
124 217
125 return urls; 218 return urls;
@@ -130,13 +223,14 @@ const getRedisKeywords = (start, end) => { @@ -130,13 +223,14 @@ const getRedisKeywords = (start, end) => {
130 * 推送url 223 * 推送url
131 */ 224 */
132 const sendKeywordsUrls = () => { 225 const sendKeywordsUrls = () => {
133 - return redis.llenAsync('keywords_mana_list').then(total => { 226 + return redis.llenAsync(redisKey.keywordsList).then(total => {
134 console.log(total); 227 console.log(total);
135 if (total <= 0) { 228 if (total <= 0) {
136 return; 229 return;
137 } 230 }
138 231
139 let start = 0, 232 let start = 0,
  233 + intervalTime = 1000, // 循环调用的时间间隔
140 count = 1000; 234 count = 1000;
141 235
142 let interval = setInterval(() => { 236 let interval = setInterval(() => {
@@ -144,19 +238,19 @@ const sendKeywordsUrls = () => { @@ -144,19 +238,19 @@ const sendKeywordsUrls = () => {
144 clearInterval(interval); 238 clearInterval(interval);
145 } 239 }
146 240
147 - console.log(start);  
148 getRedisKeywords(start, start + count).then(urls => { 241 getRedisKeywords(start, start + count).then(urls => {
149 console.log(urls); 242 console.log(urls);
150 243
151 // 发送到百度 244 // 发送到百度
152 - // sendUrlsToBaidu({site: 'https://www.yohobuy.com'}, urls); 245 + sendUrlsToBaidu({site: 'https://www.yohobuy.com'}, urls.pc);
  246 + sendUrlsToBaidu({site: 'https://m.yohobuy.com'}, urls.wap);
153 }).catch(() => { 247 }).catch(() => {
154 clearInterval(interval); 248 clearInterval(interval);
155 }); 249 });
156 250
157 start += count; 251 start += count;
158 252
159 - }, 1000); 253 + }, intervalTime);
160 254
161 return []; 255 return [];
162 256
@@ -164,6 +258,7 @@ const sendKeywordsUrls = () => { @@ -164,6 +258,7 @@ const sendKeywordsUrls = () => {
164 }; 258 };
165 259
166 module.exports = { 260 module.exports = {
  261 + sendUrls,
167 synchronousKeywords, 262 synchronousKeywords,
168 sendKeywordsUrls 263 sendKeywordsUrls
169 }; 264 };