keywords url

@@ -6,7 +6,7 @@ const yohoLib = require('yoho-node-lib');
 
 
 // 全局注册library
 // 全局注册library
 yohoLib.global(config);
 yohoLib.global(config);
-
+global.yoho.redis = require('./libs/redis');
 const logger = global.yoho.logger;
 const logger = global.yoho.logger;
 const app = express();
 const app = express();
 const seo = require('./apps/seo');
 const seo = require('./apps/seo');
@@ -14,10 +14,10 @@ const seo = require('./apps/seo');
 // 定时任务 主动推送和生成xml
 // 定时任务 主动推送和生成xml
 seo.start();
 seo.start();
 
 
-// 提供sitemap给搜索百度访问
-app.use(express.static(config.sitemapPath));
+app.get('/synchronousKeywords', seo.synchronousKeywords);
+app.get('/sendKeywordsUrls', seo.sendKeywordsUrls);
 
 
 app.listen(config.port, function() {
 app.listen(config.port, function() {
-    logger.info('yoho seo start');
+    logger.info(`yoho seo start : ${config.port}`);
 });
 });
 
 

@@ -11,6 +11,8 @@ const helper = global.yoho.helpers;
 const config = require('../config/config');
 const config = require('../config/config');
 const schedule = require('node-schedule');
 const schedule = require('node-schedule');
 const qs = require('querystring');
 const qs = require('querystring');
+const seoModel = require('./seoModel');
+
 const baiduUrls = {
 const baiduUrls = {
     urls: 'http://data.zz.baidu.com/urls',
     urls: 'http://data.zz.baidu.com/urls',
     update: 'http://data.zz.baidu.com/update',
     update: 'http://data.zz.baidu.com/update',
@@ -120,6 +122,22 @@ const sendUrls = () => {
     })();
     })();
 };
 };
 
 
+// 同步建议词（把接口拓展的建议词同步到灰度redis）
+const synchronousKeywords = (req, res) => {
+    seoModel.synchronousKeywords();
+    res.end();
+};
+
+// 定时缓慢爬取关键词页面生成缓存，防止蜘蛛爬取
+
+
+// 向百度推送页面新的页面
+const sendKeywordsUrls = (req, res) => {
+    seoModel.sendKeywordsUrls();
+    res.end();
+
+};
+
 /**
 /**
  * 定时每天1点推送最新商品和文章，更新站点sitemap
  * 定时每天1点推送最新商品和文章，更新站点sitemap
  */
  */
@@ -130,5 +148,7 @@ const start = () => {
 };
 };
 
 
 module.exports = {
 module.exports = {
-    start
+    start,
+    synchronousKeywords,
+    sendKeywordsUrls
 };
 };

+'use strict';
+const api = global.yoho.API;
+const redis = global.yoho.redis;
+const _ = require('lodash');
+
+/**
+ * redis multi command
+ */
+const multiAsync = (multi)=>{
+    return multi.execAsync().then(function(res) {
+        return res;
+    });
+};
+
+/**
+ * 将链接推送到百度站长
+ * @param site string 站点
+ * @param urls object {site: 'https://www.yohobuy.com', type: 'mip'} 默认不需要type
+ */
+// const sendUrlsToBaidu = (params, urls) => {
+//    let paramsDef = {
+//        token: config.baiduToken
+//    };
+//
+//    // 过滤无效的参数
+//    _.forEach(params, (val, key) => {
+//        if (!val) {
+//            delete params[key];
+//        }
+//    });
+//
+//    qs.escape = (str) => {
+//        return str;
+//    };
+//
+//    let options = {
+//        url: `${baiduUrls.urls}?${qs.stringify(Object.assign(paramsDef, params), null, null, {})}`,
+//        headers: {
+//            'Content-Type': 'text/plain'
+//        },
+//        method: 'post',
+//        form: urls.join('\n'),
+//        json: true,
+//        timeout: 10000,
+//        gzip: true
+//    };
+//
+//    return rp(options).then(result => {
+//        logger.info(result);
+//    });
+// };
+
+const getKeywordsApi = (page, limit) => {
+    let params = {
+        page: page || 1,
+        limit: limit || 1000,
+        method: 'web.search.suggestList'
+    };
+
+    return api.get('', params);
+};
+
+/**
+ * 关键词同步到redis
+ */
+const synchronousKeywords = () => {
+
+    return getKeywordsApi(1, 1).then(res => {
+        let start = 0,
+            page = 1,
+            limit = 1000, // 每次请求接口关键词数量
+            total = _.get(res, 'data.total', 0);
+
+            // 接口调用失败
+        if (total <= 0) {
+            console.log('no data');
+            return;
+        }
+
+            // 循环遍历接口关键词写入redis
+        let interval = setInterval(() => {
+            if (start > total) {
+                clearInterval(interval);
+            }
+
+            getKeywordsApi(page, limit).then(result => {
+                let multi = redis.multi();
+
+                start += limit;
+                page++;
+                console.log(page);
+                _.forEach(_.get(result, 'data.suggest_list', []), value => {
+                    let key = `keywords_mana:${value.keyword}`;
+
+                    multi.set(key, value.keyword);
+                    multi.lrem('keywords_mana_list', 1, key).lpush('keywords_mana_list', key);
+                });
+
+                multiAsync(multi);
+
+            }).catch(()=>{
+                clearInterval(interval);
+            });
+
+        }, 1000);
+
+    });
+};
+
+/**
+ * 查询 redis中 关键词
+ * @type {{getKeyWordsUrl}}
+ */
+
+const getRedisKeywords = (start, end) => {
+    return redis.lrangeAsync('keywords_mana_list', start, end).then(res => {
+        let urls = [];
+
+        _.forEach(res, keyword => {
+            let buff = new Buffer(keyword).toString('hex').toUpperCase();
+
+            urls.push(`https://www.yohobuy.com/so/${buff}.html`);
+        });
+
+        return urls;
+    });
+};
+
+/**
+ * 推送url
+ */
+const sendKeywordsUrls = () => {
+    return redis.llenAsync('keywords_mana_list').then(total => {
+        console.log(total);
+        if (total <= 0) {
+            return;
+        }
+
+        let start = 0,
+            count = 1000;
+
+        let interval = setInterval(() => {
+            if (start >= total) {
+                clearInterval(interval);
+            }
+
+            console.log(start);
+            getRedisKeywords(start, start + count).then(urls => {
+                console.log(urls);
+
+                // 发送到百度
+                // sendUrlsToBaidu({site: 'https://www.yohobuy.com'}, urls);
+            }).catch(() => {
+                clearInterval(interval);
+            });
+
+            start += count;
+
+        }, 1000);
+
+        return [];
+
+    });
+};
+
+module.exports = {
+    synchronousKeywords,
+    sendKeywordsUrls
+};

@@ -17,18 +17,18 @@ module.exports = {
     cookieDomain: '.yohobuy.com',
     cookieDomain: '.yohobuy.com',
     domains: {
     domains: {
         // test3
         // test3
-        // singleApi: 'http://api-test3.yohops.com:9999/',
-        // api: 'http://api-test3.yohops.com:9999/',
-        // service: 'http://service-test3.yohops.com:9999/',
-        // serviceNotify: 'http://service-test3.yohops.com:9999/',
-        // global: 'http://global-test-soa.yohops.com:9999/',
+        singleApi: 'http://api-test3.yohops.com:9999/',
+        api: 'http://api-test3.yohops.com:9999/',
+        service: 'http://service-test3.yohops.com:9999/',
+        serviceNotify: 'http://service-test3.yohops.com:9999/',
+        global: 'http://global-test-soa.yohops.com:9999/',
 
 
         // prod
         // prod
-        singleApi: 'http://single.yoho.cn/',
-        api: 'http://api.yoho.cn/',
-        service: 'http://service.yoho.cn/',
-        serviceNotify: 'http://service.yoho.cn/',
-        global: 'http://api-global.yohobuy.com/',
+        // singleApi: 'http://single.yoho.cn/',
+        // api: 'http://api.yoho.cn/',
+        // service: 'http://service.yoho.cn/',
+        // serviceNotify: 'http://service.yoho.cn/',
+        // global: 'http://api-global.yohobuy.com/',
 
 
         // gray
         // gray
         // singleApi: 'http://single.gray.yohops.com/',
         // singleApi: 'http://single.gray.yohops.com/',
@@ -133,7 +133,25 @@ module.exports = {
     maxQps: 1200,
     maxQps: 1200,
     sessionMemcachedPrefix: 'yohobuy_session:',
     sessionMemcachedPrefix: 'yohobuy_session:',
     baiduToken: '0lSAO4ZxEKsYopMG', // 百度站长推送的token
     baiduToken: '0lSAO4ZxEKsYopMG', // 百度站长推送的token
-    sitemapPath: './files'
+    redis: {
+        connect: {
+            host: '127.0.0.1',
+            port: '6379',
+            retry_strategy(options) {
+                if (options.error && options.error.code === 'ECONNREFUSED') {
+                    console.log('redis连接不成功');
+                }
+                if (options.total_retry_time > 1000 * 60 * 60 * 6) {
+                    console.log('redis连接超时');
+                    return;
+                }
+                if (options.attempt > 10) {
+                    return 1000 * 60 * 60 * 0.5;
+                }
+                return Math.min(options.attempt * 100, 1000);
+            }
+        }
+    }
 };
 };
 
 
 if (isProduction) {
 if (isProduction) {
@@ -167,7 +185,26 @@ if (isProduction) {
             open: false,
             open: false,
             url: 'http://123.206.2.55/strategy'
             url: 'http://123.206.2.55/strategy'
         },
         },
-        zookeeperServer: 'web.zookeeper.yohoops.org:2181'
+        zookeeperServer: 'web.zookeeper.yohoops.org:2181',
+        redis: {
+            connect: {
+                host: 'web.redis.yohoops.org'
+            },
+            port: '6379',
+            retry_strategy(options) {
+                if (options.error && options.error.code === 'ECONNREFUSED') {
+                    console.log('redis连接不成功');
+                }
+                if (options.total_retry_time > 1000 * 60 * 60 * 6) {
+                    console.log('redis连接超时');
+                    return;
+                }
+                if (options.attempt > 10) {
+                    return 1000 * 60 * 60 * 0.5;
+                }
+                return Math.min(options.attempt * 100, 1000);
+            }
+        }
     });
     });
 } else if (isTest) {
 } else if (isTest) {
     Object.assign(module.exports, {
     Object.assign(module.exports, {

+const redis = require('redis');
+const bluebird = require('bluebird');
+const config = require('../config/config');
+let client;
+
+try {
+    client = redis.createClient(config.redis.connect);
+
+    bluebird.promisifyAll(redis.RedisClient.prototype);
+    bluebird.promisifyAll(redis.Multi.prototype);
+
+    client.on('error', function() {
+        global.yoho.redis = '';
+    });
+
+    client.on('connect', function() {
+        global.yoho.redis = client;
+    });
+} catch (e) {
+    global.yoho.redis = '';
+}
+
+module.exports = client;

@@ -23,6 +23,7 @@
     "moment": "^2.18.1",
     "moment": "^2.18.1",
     "node-schedule": "^1.2.1",
     "node-schedule": "^1.2.1",
     "nodemon": "1.9.2",
     "nodemon": "1.9.2",
+    "redis": "^2.7.1",
     "request": "^2.79.0",
     "request": "^2.79.0",
     "request-promise": "^4.1.1",
     "request-promise": "^4.1.1",
     "shelljs": "^0.7.7",
     "shelljs": "^0.7.7",