Merge branch 'feature/keywordsPage'

@@ -6,7 +6,7 @@ const yohoLib = require('yoho-node-lib');
 
 // 全局注册library
 yohoLib.global(config);
-
+global.yoho.redis = require('./libs/redis');
 const logger = global.yoho.logger;
 const app = express();
 const seo = require('./apps/seo');
@@ -14,10 +14,10 @@ const seo = require('./apps/seo');
 // 定时任务 主动推送和生成xml
 seo.start();
 
-// 提供sitemap给搜索百度访问
-app.use(express.static(config.sitemapPath));
+app.get('/synchronousKeywords', seo.synchronousKeywords);
+app.get('/sendKeywordsUrls', seo.sendKeywordsUrls);
 
 app.listen(config.port, function() {
-    logger.info('yoho seo start');
+    logger.info(`yoho seo start : ${config.port}`);
 });
 

 'use strict';
 
-const api = global.yoho.API;
-const rp = require('request-promise');
-const serviceApi = global.yoho.ServiceAPI;
-const Promise = require('bluebird');
-const co = Promise.coroutine;
-const _ = require('lodash');
-const logger = global.yoho.logger;
-const helper = global.yoho.helpers;
-const config = require('../config/config');
 const schedule = require('node-schedule');
-const qs = require('querystring');
-const baiduUrls = {
-    urls: 'http://data.zz.baidu.com/urls',
-    update: 'http://data.zz.baidu.com/update',
-    del: 'http://data.zz.baidu.com/del'
-};
-
-const siteUrls = {
-    pcProduct: {
-        site: 'https://item.yohobuy.com',
-        url: []
-    },
-    pcGuang: {
-        site: 'https://guang.yohobuy.com',
-        url: []
-    },
-    mProduct: {
-        site: 'https://m.yohobuy.com',
-        url: []
-    },
-    mGuang: {
-        site: 'https://guang.m.yohobuy.com',
-        url: [],
-        type: 'mip'
-    }
-
-};
-
+const seoModel = require('./seoModel');
 
 /**
- * 获取最新1000条商品详情链接和逛详情链接
+ * 同步建议词（把接口拓展的建议词同步到灰度redis）
  */
-const getUrls = () => {
-    let apiArr = [api.get('', {method: 'web.product.bdPromotion'}),
-        serviceApi.get('/guang/api/v2/article/getLastArticleList', {limit: 100})];
-
-    return api.all(apiArr).spread((productData, articleData) => {
-
-        _.forEach(_.get(productData, 'data', {}), value => {
-            siteUrls.pcProduct.url.push('https:' + helper.urlFormat(`/${value.id}.html`, null, 'item'));
-            siteUrls.mProduct.url.push('https:' + helper.urlFormat(`/product/${value.id}.html`, null, 'm'));
-        });
-
-        _.forEach(_.get(articleData, 'data.artList', {}), value => {
-            siteUrls.pcGuang.url.push('https:' + helper.urlFormat(`/${value.articleId}.html`, null, 'guang'));
-            siteUrls.mGuang.url.push('https:' + helper.urlFormat(`/mip/guang/info/${value.articleId}.html`
-, null, 'guang.m'));
-        });
-
-        return siteUrls;
-    });
+const synchronousKeywords = (req, res) => {
+    seoModel.synchronousKeywords();
+    res.end();
 };
 
 /**
- * 将链接推送到百度站长
- * @param params object {site: 'https://www.yohobuy.com', type: 'mip'} 默认不需要type
- * @param urls
+ * 定时缓慢爬取关键词页面生成缓存，防止蜘蛛爬取
  */
-const sendUrlsToBaidu = (params, urls) => {
-    let paramsDef = {
-        token: config.baiduToken
-    };
-
-    // 过滤无效的参数
-    _.forEach(params, (val, key) => {
-        if (!val) {
-            delete params[key];
-        }
-    });
 
-    qs.escape = (str) => {
-        return str;
-    };
 
-    let options = {
-        url: `${baiduUrls.urls}?${qs.stringify(Object.assign(paramsDef, params), null, null, {})}`,
-        headers: {
-            'Content-Type': 'text/plain'
-        },
-        method: 'post',
-        form: urls.join('\n'),
-        json: true,
-        timeout: 10000,
-        gzip: true
-    };
-
-    return rp(options).then(result => {
-        logger.info(Object.assign(params, result, {length: urls.length}));
-    });
+/**
+ * 向百度推送页面关键词静态页面
+ */
+const sendKeywordsUrls = (req, res) => {
+    seoModel.sendKeywordsUrls();
+    res.end();
 };
 
 /**
- * 获取最新商品详情1000条和逛详情100条推送到相应的站点域名（pc和wap）
+ * 商品详情，逛推送
  */
 const sendUrls = () => {
-
-    co(function*() {
-        // 获取pc/wap的商品详情和逛的链接
-        let sendArr = [],
-            urls = yield getUrls();
-
-        _.forEach(urls, value => {
-            sendArr.push(sendUrlsToBaidu({site: value.site, type: value.type}, value.url));
-        });
-
-        // 推送url
-        api.all(sendArr);
-    })();
+    seoModel.sendUrls();
 };
 
 /**
- * 定时每天1点推送最新商品和文章，更新站点sitemap
+ * 定时任务
  */
 const start = () => {
     schedule.scheduleJob('0 0 1 * * *', function() {
+
+        // 推送最新的商品详情和逛文章
         sendUrls();
+
+        // 同步关键词
+        synchronousKeywords();
+    });
+
+    schedule.scheduleJob('0 0 2 * * *', function() {
+
+        // 推送关键词页面
+        sendKeywordsUrls();
     });
 };
 
 module.exports = {
     start,
-    sendUrls
+    synchronousKeywords,
+    sendKeywordsUrls
 };

+'use strict';
+const api = global.yoho.API;
+const redis = global.yoho.redis;
+const rp = require('request-promise');
+const serviceApi = global.yoho.ServiceAPI;
+const Promise = require('bluebird');
+const co = Promise.coroutine;
+const _ = require('lodash');
+const logger = global.yoho.logger;
+const helper = global.yoho.helpers;
+const config = require('../config/config');
+const qs = require('querystring');
+
+
+/**
+ * redis multi command
+ */
+const multiAsync = (multi)=>{
+    return multi.execAsync().then(function(res) {
+        return res;
+    });
+};
+
+const baiduUrls = {
+    urls: 'http://data.zz.baidu.com/urls',
+    update: 'http://data.zz.baidu.com/update',
+    del: 'http://data.zz.baidu.com/del'
+};
+
+const siteUrls = {
+    pcProduct: {
+        site: 'https://item.yohobuy.com',
+        url: []
+    },
+    pcGuang: {
+        site: 'https://guang.yohobuy.com',
+        url: []
+    },
+    mProduct: {
+        site: 'https://m.yohobuy.com',
+        url: []
+    },
+    mGuang: {
+        site: 'https://guang.m.yohobuy.com',
+        url: [],
+        type: 'mip'
+    }
+
+};
+
+
+
+// 配置
+const redisKey = {
+    keywordsList: 'keywords_mana_list' // 关键词列表
+};
+
+/**
+ * 将链接推送到百度站长
+ * @param params object {site: 'https://www.yohobuy.com', type: 'mip'} 默认不需要type
+ * @param urls
+ */
+const sendUrlsToBaidu = (params, urls) => {
+    let paramsDef = {
+        token: config.baiduToken
+    };
+
+    // 过滤无效的参数
+    _.forEach(params, (val, key) => {
+        if (!val) {
+            delete params[key];
+        }
+    });
+
+    qs.escape = (str) => {
+        return str;
+    };
+
+    let options = {
+        url: `${baiduUrls.urls}?${qs.stringify(Object.assign(paramsDef, params), null, null, {})}`,
+        headers: {
+            'Content-Type': 'text/plain'
+        },
+        method: 'post',
+        form: urls.join('\n'),
+        json: true,
+        timeout: 10000,
+        gzip: true
+    };
+
+    return rp(options).then(result => {
+        logger.info(Object.assign(params, result, {length: urls.length}));
+    });
+};
+
+/**
+ * 获取最新1000条商品详情链接和逛详情链接
+ */
+const getUrls = () => {
+    let apiArr = [api.get('', {method: 'web.product.bdPromotion'}),
+        serviceApi.get('/guang/api/v2/article/getLastArticleList', {limit: 100})];
+
+    return api.all(apiArr).spread((productData, articleData) => {
+
+        _.forEach(_.get(productData, 'data', {}), value => {
+            siteUrls.pcProduct.url.push('https:' + helper.urlFormat(`/${value.id}.html`, null, 'item'));
+            siteUrls.mProduct.url.push('https:' + helper.urlFormat(`/product/${value.id}.html`, null, 'm'));
+        });
+
+        _.forEach(_.get(articleData, 'data.artList', {}), value => {
+            siteUrls.pcGuang.url.push('https:' + helper.urlFormat(`/${value.articleId}.html`, null, 'guang'));
+            siteUrls.mGuang.url.push('https:' + helper.urlFormat(`/mip/guang/info/${value.articleId}.html`
+        , null, 'guang.m'));
+        });
+
+        return siteUrls;
+    });
+};
+
+/**
+ * 发送最新商品详情1000条和逛详情100条推送到相应的站点域名（pc和wap）
+ */
+const sendUrls = () => {
+
+    co(function*() {
+        // 获取pc/wap的商品详情和逛的链接
+        let sendArr = [],
+            urls = yield getUrls();
+
+        _.forEach(urls, value => {
+            sendArr.push(sendUrlsToBaidu({site: value.site, type: value.type}, value.url));
+        });
+
+        // 推送url
+        api.all(sendArr);
+    })();
+};
+
+
+
+/**
+ * 调用接口建议词
+ */
+const getKeywordsApi = (page, limit) => {
+    let params = {
+        page: page || 1,
+        limit: limit || 1000,
+        method: 'web.search.suggestList'
+    };
+
+    return api.get('', params);
+};
+
+/**
+ * 关键词同步到redis
+ */
+const synchronousKeywords = () => {
+
+    return getKeywordsApi(1, 1).then(res => {
+        let start = 0,
+            page = 1,
+            intervalTime = 1000, // 循环调用的时间间隔
+            limit = 1000, // 每次请求接口关键词数量
+            total = _.get(res, 'data.total', 0);
+
+            // 接口调用失败
+        if (total <= 0) {
+            console.log('no data');
+            return;
+        }
+
+            // 循环遍历接口关键词写入redis
+        let interval = setInterval(() => {
+            if (start > total) {
+                clearInterval(interval);
+            }
+
+            getKeywordsApi(page, limit).then(result => {
+                let multi = redis.multi();
+
+                start += limit;
+                page++;
+                console.log(page);
+                _.forEach(_.get(result, 'data.suggest_list', []), value => {
+                    let key = `keywords_mana:${value.keyword}`;
+
+                    multi.set(key, value.keyword);
+                    multi.lrem('keywords_mana_list', 1, key).lpush('keywords_mana_list', key);
+                });
+
+                multiAsync(multi);
+
+            }).catch(()=>{
+                clearInterval(interval);
+            });
+
+        }, intervalTime);
+
+    });
+};
+
+/**
+ * 查询 redis中 关键词
+ * @type {{getKeyWordsUrl}}
+ */
+
+const getRedisKeywords = (start, end) => {
+    return redis.lrangeAsync(redisKey.keywordsList, start, end).then(res => {
+        let urls = {pc: [], wap: []};
+
+        _.forEach(res, keyword => {
+            let buff = new Buffer(keyword).toString('hex').toUpperCase();
+
+            urls.pc.push(`https://www.yohobuy.com/so/${buff}.html`);
+            urls.wap.push(`https://m.yohobuy.com/so/${buff}.html`);
+        });
+
+        return urls;
+    });
+};
+
+/**
+ * 推送url
+ */
+const sendKeywordsUrls = () => {
+    return redis.llenAsync(redisKey.keywordsList).then(total => {
+        console.log(total);
+        if (total <= 0) {
+            return;
+        }
+
+        let start = 0,
+            intervalTime = 1000, // 循环调用的时间间隔
+            count = 1000;
+
+        let interval = setInterval(() => {
+            if (start >= total) {
+                clearInterval(interval);
+            }
+
+            getRedisKeywords(start, start + count).then(urls => {
+                console.log(urls);
+
+                // 发送到百度
+                sendUrlsToBaidu({site: 'https://www.yohobuy.com'}, urls.pc);
+                sendUrlsToBaidu({site: 'https://m.yohobuy.com'}, urls.wap);
+            }).catch(() => {
+                clearInterval(interval);
+            });
+
+            start += count;
+
+        }, intervalTime);
+
+        return [];
+
+    });
+};
+
+module.exports = {
+    sendUrls,
+    synchronousKeywords,
+    sendKeywordsUrls
+};

@@ -17,18 +17,18 @@ module.exports = {
     cookieDomain: '.yohobuy.com',
     domains: {
         // test3
-        // singleApi: 'http://api-test3.yohops.com:9999/',
-        // api: 'http://api-test3.yohops.com:9999/',
-        // service: 'http://service-test3.yohops.com:9999/',
-        // serviceNotify: 'http://service-test3.yohops.com:9999/',
-        // global: 'http://global-test-soa.yohops.com:9999/',
+        singleApi: 'http://api-test3.yohops.com:9999/',
+        api: 'http://api-test3.yohops.com:9999/',
+        service: 'http://service-test3.yohops.com:9999/',
+        serviceNotify: 'http://service-test3.yohops.com:9999/',
+        global: 'http://global-test-soa.yohops.com:9999/',
 
         // prod
-        singleApi: 'http://single.yoho.cn/',
-        api: 'http://api.yoho.cn/',
-        service: 'http://service.yoho.cn/',
-        serviceNotify: 'http://service.yoho.cn/',
-        global: 'http://api-global.yohobuy.com/',
+        // singleApi: 'http://single.yoho.cn/',
+        // api: 'http://api.yoho.cn/',
+        // service: 'http://service.yoho.cn/',
+        // serviceNotify: 'http://service.yoho.cn/',
+        // global: 'http://api-global.yohobuy.com/',
 
         // gray
         // singleApi: 'http://single.gray.yohops.com/',
@@ -133,7 +133,25 @@ module.exports = {
     maxQps: 1200,
     sessionMemcachedPrefix: 'yohobuy_session:',
     baiduToken: '0lSAO4ZxEKsYopMG', // 百度站长推送的token
-    sitemapPath: './files'
+    redis: {
+        connect: {
+            host: '127.0.0.1',
+            port: '6379',
+            retry_strategy(options) {
+                if (options.error && options.error.code === 'ECONNREFUSED') {
+                    console.log('redis连接不成功');
+                }
+                if (options.total_retry_time > 1000 * 60 * 60 * 6) {
+                    console.log('redis连接超时');
+                    return;
+                }
+                if (options.attempt > 10) {
+                    return 1000 * 60 * 60 * 0.5;
+                }
+                return Math.min(options.attempt * 100, 1000);
+            }
+        }
+    }
 };
 
 if (isProduction) {
@@ -167,7 +185,26 @@ if (isProduction) {
             open: false,
             url: 'http://123.206.2.55/strategy'
         },
-        zookeeperServer: 'web.zookeeper.yohoops.org:2181'
+        zookeeperServer: 'web.zookeeper.yohoops.org:2181',
+        redis: {
+            connect: {
+                host: 'web.redis.yohoops.org'
+            },
+            port: '6379',
+            retry_strategy(options) {
+                if (options.error && options.error.code === 'ECONNREFUSED') {
+                    console.log('redis连接不成功');
+                }
+                if (options.total_retry_time > 1000 * 60 * 60 * 6) {
+                    console.log('redis连接超时');
+                    return;
+                }
+                if (options.attempt > 10) {
+                    return 1000 * 60 * 60 * 0.5;
+                }
+                return Math.min(options.attempt * 100, 1000);
+            }
+        }
     });
 } else if (isTest) {
     Object.assign(module.exports, {

+const redis = require('redis');
+const bluebird = require('bluebird');
+const config = require('../config/config');
+let client;
+
+try {
+    client = redis.createClient(config.redis.connect);
+
+    bluebird.promisifyAll(redis.RedisClient.prototype);
+    bluebird.promisifyAll(redis.Multi.prototype);
+
+    client.on('error', function() {
+        global.yoho.redis = '';
+    });
+
+    client.on('connect', function() {
+        global.yoho.redis = client;
+    });
+} catch (e) {
+    global.yoho.redis = '';
+}
+
+module.exports = client;

@@ -23,6 +23,7 @@
     "moment": "^2.18.1",
     "node-schedule": "^1.2.1",
     "nodemon": "1.9.2",
+    "redis": "^2.7.1",
     "request": "^2.79.0",
     "request-promise": "^4.1.1",
     "shelljs": "^0.7.7",