index.js
4.03 KB
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
'use strict';
const ROOT_PATH = global.ROOT_PATH;
const _ = require('lodash');
const co = Promise.coroutine;
const rp = require('request-promise');
const redis = require(`${ROOT_PATH}/libs/redis`);
const YohoApiModel = require('./../interface/yoho-api');
const senUrl = 'http://data.zz.baidu.com/urls?appid=1583402501013173&token=K0L5PUhk1XOko81r&type=';
const NEWSKEY = 'global:yoho:seo:baidu:mip';
// http://ziyuan.baidu.com/xzh/commit/push?appid=1583402501013173&qq-pf-to=pcqq.c2c
class XzhIndexModel extends global.yoho.BaseModel {
constructor(ctx) {
super(ctx);
this.yohoApiModel = new YohoApiModel(ctx);
}
index() {
let that = this;
return co(function* () {
let ids = [];
// 新增
let rdata = yield that.yohoApiModel.getLastArticleList({page: 1, limit: 100});
rdata = _.get(rdata, 'data.artList', []);
ids = _.map(rdata, item => {
return `${item.articleId}`;
});
let artice = {
total: ids.length,
notAlready: 0
};
ids = _.difference(ids, yield redis.hmgetAsync('global:yoho:seo:xzh:guang', ids));// 去除已经推送的
artice.notAlready = ids.length;
if (artice.notAlready <= 0) {
return {code: 201, data: {}, message: `获取${artice.total}条记录,未推送0条,成功推送0条。`};
}
rdata = yield that.sendData(_.map(ids, id => {
return `https://m.yohobuy.com/mip/guang/${id}.html`;
}), 'realtime');
if (rdata.code !== 200) {
return rdata;
}
let tids = [];
_.each(ids, id => {
tids.push(id, id);
});
yield redis.hmsetAsync('global:yoho:seo:xzh:guang', tids);
ids = [];
tids = [];
return rdata;
})();
}
yohoGirlsAll() {
let that = this;
return co(function* () {
let urls = yield redis.lrangeAsync(`${NEWSKEY}:mip:www_yohogirls_com`, 0, 1000);
let rdata = yield that.sendData(urls, 'batch');
redis.ltrimAsync(`${NEWSKEY}:www_yohogirls_com`, 1000, -1);
redis.ltrimAsync(`${NEWSKEY}:mip:www_yohogirls_com`, 1000, -1);
return rdata;
})();
}
yohoBoysAll() {
let that = this;
return co(function* () {
let urls = yield redis.lrangeAsync(`${NEWSKEY}:mip:www_yohoboys_com`, 0, 1000);
let rdata = yield that.sendData(urls, 'batch');
redis.ltrimAsync(`${NEWSKEY}:www_yohoboys_com`, 1000, -1);
redis.ltrimAsync(`${NEWSKEY}:mip:www_yohoboys_com`, 1000, -1);
return rdata;
})();
}
marsAll() {
let that = this;
return co(function* () {
let urls = yield redis.lrangeAsync(`${NEWSKEY}:mip:www_yohomars_com`, 0, 1000);
let rdata = yield that.sendData(urls, 'batch');
redis.ltrimAsync(`${NEWSKEY}:www_yohomars_com`, 1000, -1);
redis.ltrimAsync(`${NEWSKEY}:mip:www_yohomars_com`, 1000, -1);
return rdata;
})();
}
// 向百度发送数据
sendData(urls, type) {
if (urls.length <= 0) {
return Promise.resolve({code: 400, data: {}, message: 'data is empty'});
}
type = type || 'batch';
return rp({
method: 'POST',
uri: `${senUrl}${type}`,
body: urls.join('\n'),
timeout: 8 * 1000
}).then(result => {
result = JSON.parse(result || '{}');
return Object.assign({
code: 200,
message: `当天剩余${result.remain_realtime}条可推送到mip,本次成功推送${result.success_realtime}条。`
}, result);
}).catch(e => {
console.log(e.message);
return {code: e.statusCode || 400, data: {}, message: e.message};
});
}
}
module.exports = XzhIndexModel;