Merge branch 'feature/mars'

@@ -58,6 +58,12 @@ app.get('/xzh-yohoBoysAll', baiduXzh.yohoBoysAll);
 // 百度熊掌号--mars全站推送一遍mip
 app.get('/xzh-marsAll', baiduXzh.marsAll);
 
+// 百度熊掌号--yohoBoysReal实时推送mip
+app.get('/xzh-yohoBoysReal', baiduXzh.yohoBoysReal);
+
+// 百度熊掌号--mars实时推送mip
+app.get('/xzh-marsReal', baiduXzh.marsReal);
+
 // 商品详情-百度链接提交
 app.get('/goodsUrl', cseo.goodsUrl);
 
@@ -82,6 +88,9 @@ app.get('/marsAll', cseo.marsAll);
 // 潮流全站推送一遍mip和百度资源位
 app.get('/yohoBoysAll', cseo.yohoBoysAll);
 
+// mars实时推送百度资源位连接和百度资源位mip
+app.get('/marsReal', cseo.marsReal);
+
 app.listen(config.port, function() {
     logger.info(`yoho seo start : http://127.0.0.1:${config.port}`);
 });

@@ -129,6 +129,12 @@ const marsAll = (req, res) => {
     });
 };
 
+const marsReal = (req, res) => {
+    return req.ctx(BaiduLinkSubmit).marsReal(req.query).then(result => {
+        return res.json(result);
+    });
+};
+
 module.exports = {
     index,
     autoGoodsXml,
@@ -143,6 +149,7 @@ module.exports = {
     yohoGirlsAll,
     yohoBoysAll,
     marsAll,
+    marsReal,
     demoXsd,
     demoXml
 };

@@ -31,9 +31,23 @@ const marsAll = (req, res) => {
     });
 };
 
+const yohoBoysReal = (req, res) => {
+    return req.ctx(XzhIndexModel).yohoBoysReal().then(result => {
+        return res.json(result);
+    });
+};
+
+const marsReal = (req, res) => {
+    return req.ctx(XzhIndexModel).marsReal().then(result => {
+        return res.json(result);
+    });
+};
+
 module.exports = {
     index,
     yohoGirlsAll,
     yohoBoysAll,
-    marsAll
+    marsAll,
+    yohoBoysReal,
+    marsReal
 };

@@ -189,7 +189,7 @@ class BaiduLinkSubmit extends global.yoho.BaseModel {
         })();
     }
 
-    _getYohoNewsLink(url) {
+    _getRp(url, json) {
         return rp({
             method: 'GET',
             uri: url,
@@ -198,7 +198,7 @@ class BaiduLinkSubmit extends global.yoho.BaseModel {
                 encode: false
             },
             timeout: 8 * 1000,
-            json: true,
+            json: json || true,
             gzip: true
         }).then(result => {
             return {
@@ -207,7 +207,6 @@ class BaiduLinkSubmit extends global.yoho.BaseModel {
                 message: 'success'
             };
         }).catch(e => {
-            console.log(`_getYohoNewsLink: err: ${e.message}`);
             return {code: e.statusCode || 400, data: [], message: e.message};
         });
     }
@@ -224,8 +223,9 @@ class BaiduLinkSubmit extends global.yoho.BaseModel {
                 mipUrl: 'http://www.yohoboys.com'
             };
 
-            let rdata = _.get(yield that._getYohoNewsLink(
-                `${initData.pcUrl}/channel/ajax/tab?channel=100&page=1`
+            let rdata = _.get(yield that._getRp(
+                `${initData.pcUrl}/channel/ajax/tab?channel=100&page=1`,
+                true
             ), 'data', []);
             let tlinks = [];
             let links = _.map(rdata, item => {
@@ -294,8 +294,9 @@ class BaiduLinkSubmit extends global.yoho.BaseModel {
                 mipUrl: 'http://www.yohogirls.com'
             };
 
-            let rdata = _.get(yield that._getYohoNewsLink(
-                `${initData.pcUrl}/channel/ajax/getRecoms?channel=1&page=1`
+            let rdata = _.get(yield that._getRp(
+                `${initData.pcUrl}/channel/ajax/getRecoms?channel=1&page=1`,
+                true
             ), 'data', []);
             let tlinks = [];
             let links = _.map(rdata, item => {
@@ -447,6 +448,54 @@ class BaiduLinkSubmit extends global.yoho.BaseModel {
         })();
     }
 
+    // 实时推送mars
+    marsReal() {
+        let marsIds = [];
+        let errId = 0;
+        let initData = {
+            mip: [],
+            pc: [],
+            pcUrl: 'http://www.yohomars.com',
+            mipUrl: 'http://www.yohomars.com'
+        };
+
+        // init global:yoho:seo:linksubmit:marsReal 11972
+        return redis.getAsync('global:yoho:seo:linksubmit:marsReal').then(marsReal => {
+            marsReal = parseInt(marsReal, 10);
+
+            for (let i = 0; i < 5; i++) {
+                marsIds.push(marsReal);
+                marsReal = (marsReal + 2);
+            }
+
+            return Promise.each(marsIds, (id) => this._getRp(
+                    `http://www.yohomars.com/share/store/${id}`,
+                    false
+                ).then(result => {
+
+                    if (result.code === 200) {
+                        initData.pc.push(`http://www.yohomars.com/share/store/${id}`);
+                        initData.mip.push(`http://www.yohomars.com/store/share/mip/${id}.html`);
+                    } else if (!errId && result.code === 404) {
+                        errId = id;
+                    }
+
+                    return result;
+                })
+            );
+        }).then(() => {
+            errId = errId ? errId : (marsIds.pop() + 2);
+
+            // 记录最后一次404或者上次推送成功连接id
+            return redis.setAsync('global:yoho:seo:linksubmit:marsReal', errId);
+        }).then(() => {
+            return Promise.all([
+                this.sendUrl(initData.pc, initData.pcUrl),
+                this.sendUrl(initData.mip, initData.mipUrl, 'mip')
+            ]);
+        });
+    }
+
 }
 
 module.exports = BaiduLinkSubmit;

@@ -58,7 +58,7 @@ class XzhIndexModel extends global.yoho.BaseModel {
                 return {code: 201, data: {}, message: `获取${artice.total}条记录，未推送0条，成功推送0条。`};
             }
 
-            rdata = yield that.sendData(_.map(ids, id => {
+            rdata = yield that._sendData(_.map(ids, id => {
                 return `https://m.yohobuy.com/mip/guang/${id}.html`;
             }), XZHKEY.yohobuy, 'realtime');
 
@@ -86,7 +86,7 @@ class XzhIndexModel extends global.yoho.BaseModel {
 
         return co(function* () {
             let urls = yield redis.lrangeAsync(`${NEWSKEY}:mip:www_yohogirls_com`, 0, 999);
-            let rdata = yield that.sendData(urls, XZHKEY.yohonow, 'batch');
+            let rdata = yield that._sendData(urls, XZHKEY.yohonow, 'batch');
 
             yield redis.ltrimAsync(`${NEWSKEY}:www_yohogirls_com`, 1000, -1);
             yield redis.ltrimAsync(`${NEWSKEY}:mip:www_yohogirls_com`, 1000, -1);
@@ -99,7 +99,7 @@ class XzhIndexModel extends global.yoho.BaseModel {
 
         return co(function* () {
             let urls = yield redis.lrangeAsync(`${NEWSKEY}:mip:www_yohoboys_com`, 0, 999);
-            let rdata = yield that.sendData(urls, XZHKEY.yohonow, 'batch');
+            let rdata = yield that._sendData(urls, XZHKEY.yohonow, 'batch');
 
             yield redis.ltrimAsync(`${NEWSKEY}:www_yohoboys_com`, 1000, -1);
             yield redis.ltrimAsync(`${NEWSKEY}:mip:www_yohoboys_com`, 1000, -1);
@@ -113,7 +113,7 @@ class XzhIndexModel extends global.yoho.BaseModel {
 
         return co(function* () {
             let urls = yield redis.lrangeAsync(`${NEWSKEY}:mip:www_yohomars_com`, 0, 999);
-            let rdata = yield that.sendData(urls, XZHKEY.mars, 'batch');
+            let rdata = yield that._sendData(urls, XZHKEY.mars, 'batch');
 
             yield redis.ltrimAsync(`${NEWSKEY}:www_yohomars_com`, 1000, -1);
             yield redis.ltrimAsync(`${NEWSKEY}:mip:www_yohomars_com`, 1000, -1);
@@ -122,8 +122,107 @@ class XzhIndexModel extends global.yoho.BaseModel {
         })();
     }
 
+    // 实时推送潮流资讯
+    yohoBoysReal() {
+        let that = this;
+
+        return co(function* () {
+            let initData = {
+                mip: [],
+                mipUrl: 'http://www.yohoboys.com'
+            };
+
+            let rdata = _.get(yield that._getRp(
+                `${initData.mipUrl}/channel/ajax/tab?channel=100&page=1`,
+                true
+            ), 'data', []);
+            let tlinks = [];
+            let links = _.map(rdata, item => {
+                return _.get(item, 'content.url', '').replace(/http:|https:/, '');
+            });
+            let artice = {
+                total: links.length,
+                notAlready: 0
+            };
+
+            links = _.difference(
+                links,
+                yield redis.hmgetAsync('global:yoho:seo:xzh:yohoBoysReal', links)
+            );// 去除已经推送的
+            artice.notAlready = links.length;
+            if (artice.notAlready <= 0) {
+                let message = `获取${artice.total}条记录，未推送0条，成功推送0条。`;
+
+                return {code: 201, data: {}, message: `MIP: ${message}`};
+            }
+
+            _.each(links, val => {
+                if (_.isEmpty(val)) {
+                    return true;
+                }
+
+                tlinks.push(val, val);// mset做准备
+
+                initData.mip.push('http:' + val.replace(/\/news\/index/, '/mip/news/mobile'));
+            });
+
+            rdata = yield that._sendData(initData.mip, XZHKEY.yohonow, 'realtime');
+
+            if (rdata.code !== 200) {
+                return rdata;
+            }
+
+            yield redis.hmsetAsync('global:yoho:seo:xzh:yohoBoysReal', tlinks);
+
+            initData = {};
+            links = [];
+            tlinks = [];
+
+            return rdata;
+        })();
+    }
+
+    // 实时推送mars
+    marsReal() {
+        let marsIds = [];
+        let errId = 0;
+        let links = [];
+
+        // init global:yoho:seo:xzh:marsReal 11972
+        return redis.getAsync('global:yoho:seo:xzh:marsReal').then(marsReal => {
+            marsReal = parseInt(marsReal, 10);
+
+            for (let i = 0; i < 5; i++) {
+                marsIds.push(marsReal);
+                marsReal = (marsReal + 2);
+            }
+
+            return Promise.each(marsIds, (id) => this._getRp(
+                    `http://www.yohomars.com/share/store/${id}`,
+                    false
+                ).then(result => {
+
+                    if (result.code === 200) {
+                        links.push(`http://www.yohomars.com/store/share/mip/${id}.html`);
+                    } else if (!errId && result.code === 404) {
+                        errId = id;
+                    }
+
+                    return result;
+                })
+            );
+        }).then(() => {
+            errId = errId ? errId : (marsIds.pop() + 2);
+
+            // 记录最后一次404或者上次推送成功连接id
+            return redis.setAsync('global:yoho:seo:xzh:marsReal', errId);
+        }).then(() => {
+            return this._sendData(links, XZHKEY.mars, 'realtime');
+        });
+    }
+
     // 向百度发送数据
-    sendData(urls, appTokey, type) {
+    _sendData(urls, appTokey, type) {
 
         if (urls.length <= 0) {
             return Promise.resolve({code: 400, data: {}, message: 'urls is empty'});
@@ -158,6 +257,28 @@ class XzhIndexModel extends global.yoho.BaseModel {
             return {code: e.statusCode || 400, data: {}, message: e.message};
         });
     }
+
+    _getRp(url, json) {
+        return rp({
+            method: 'GET',
+            uri: url,
+            qs: {},
+            qsStringifyOptions: {
+                encode: false
+            },
+            timeout: 8 * 1000,
+            json: json || true,
+            gzip: true
+        }).then(result => {
+            return {
+                code: 200,
+                data: result,
+                message: 'success'
+            };
+        }).catch(e => {
+            return {code: e.statusCode || 400, data: [], message: e.message};
+        });
+    }
 }
 
 module.exports = XzhIndexModel;