Authored by 毕凯

增加接口调用缓存

... ... @@ -12,13 +12,16 @@ if (config.useOneapm) {
require('oneapm');
}
let express = require('express'),
path = require('path'),
bodyParser = require('body-parser'),
cookieParser = require('cookie-parser'),
favicon = require('serve-favicon');
const express = require('express');
const path = require('path');
const bodyParser = require('body-parser');
const cookieParser = require('cookie-parser');
const favicon = require('serve-favicon');
const session = require('express-session');
const memcached = require('connect-memcached');
let app = express();
const app = express();
const MemcachedStore = memcached(session);
app.set('view engine', '.hbs');
... ... @@ -27,6 +30,17 @@ app.use(express.static(path.join(__dirname, 'public')));
app.use(bodyParser.json());
app.use(bodyParser.urlencoded({extended: false}));
app.use(cookieParser());
app.use(session({
secret: '3e5fec7deca0b8305cefe2ad9d90ff5e',
name: 'PHPSESSID',
prefix: 'yohobuy',
proxy: true,
resave: true,
saveUninitialized: true,
store: new MemcachedStore({
hosts: config.memcache.session
})
}));
// dispatcher
require('./dispatch')(app);
... ...
... ... @@ -10,8 +10,7 @@ const headerModel = require('../../../doraemon/models/header');
exports.index = (req, res) => {
headerModel.getHeaderData()
.then(function(modelRes) {
let response = JSON.parse(modelRes);
.then(function(response) {
let data = headerModel.getAllHeaderData(response.data, 'boy');
data.headerData.navbars[0].active = true;
... ...
... ... @@ -16,8 +16,11 @@ module.exports = {
search: 'http://192.168.10.64:8080/yohosearch/'
},
useOneapm: false,
useCache: true,
memcache: {
server: ['192.168.102.168:12580'],
master: ['192.168.102.168:12580'],
slave: ['192.168.102.168:12580'],
session: ['192.168.102.168:12580'],
timeout: 5000
},
loggers: {
... ...
... ... @@ -210,7 +210,5 @@ exports.getHeaderData = function() {
/* eslint-enable */
});
return serviceApi.get('/operations/api/v6/category/getCategory', data);
return serviceApi.get('/operations/api/v6/category/getCategory', data, true);
};
... ...
... ... @@ -15,7 +15,7 @@
<link rel="dns-prefetch" href="//img12.static.yhbimg.com">
<link rel="dns-prefetch" href="//img13.static.yhbimg.com">
{{#if devEnv}}
<link rel="stylesheet" href="//localhost:5000/css/index.css">
<link rel="stylesheet" href="//localhost:5002/css/index.css">
{{^}}
<link rel="stylesheet" href="//cdn.yoho.cn/m-yohobuy-node/{{version}}/index.css">
{{/if}}
... ... @@ -29,7 +29,7 @@
{{/if}}
{{> footer}}
{{#if devEnv}}
<script src="//localhost:5000/{{module}}.{{page}}.js"></script>
<script src="//localhost:5002/{{module}}.{{page}}.js"></script>
{{^}}
<script src="//cdn.yoho.cn/m-yohobuy-node/{{version}}/{{module}}.{{page}}.js"></script>
{{/if}}
... ...
... ... @@ -7,12 +7,16 @@
'use strict';
const rp = require('request-promise');
const qs = require('querystring');
const md5 = require('md5');
const _ = require('lodash');
const log = require('./logger');
const api = require('../config/common').domains.api;
const serviceApi = require('../config/common').domains.service;
const searchApi = require('../config/common').domains.search;
const cache = require('./cache');
const Timer = require('./timer');
const config = require('../config/common');
const api = config.domains.api;
const serviceApi = config.domains.service;
const searchApi = config.domains.search;
let ApiUrl;
... ... @@ -24,54 +28,107 @@ class API {
}
/**
* get
* @param url String
* @param data Obejct
* 获取请求 ID
*/
get(url, data) {
let options = {
url: `${ApiUrl}${url}`,
qs: data
};
_getReqId(options) {
return md5(`${options.url}?${qs.stringify(options.qs || options.form)}`);
}
/**
* 调用接口
*/
_requestFromAPI(options, cacheOption, reqId) {
let timer = new Timer();
timer.put('getApi');// 统计时间开始
let ret = rp(options);
ret.then((body)=>{
let duration = timer.put('getApi');// 接口返回
log.info('API GET: %s, parms: %j , durationMs: %d ms , body: %s', options.url, options.qs, duration, body);
log.info(`get api: ${options.url}?${qs.stringify(options.qs)}`);
return rp(options).then((result) => {
let duration = timer.put('getApi');// 统计时间结束
log.info(`get api success: use: ${duration}ms`);
if (config.useCache && cacheOption) {
reqId = reqId || this._getReqId(options);
// 数据校验无误,写缓存, 否则返回 Slave 缓存服务器的数据
if (result && result.code) {
let cacheTime = _.isNumber(cacheOption) ? cacheOption : 60;
cache.set(`apiCache:${reqId}`, result, cacheTime);
cache.setSlave(`apiCache:${reqId}`, result, cacheTime);
} else {
return this._requestFromCache(options, true);
}
}
return result;
}).catch((error)=>{
let duration = timer.put('getApi');// 接口返回
let duration = timer.put('getApi');// 统计时间结束
log.error('API GET: %s, parms: %j , durationMs: %d ms error: %s , statusCode: %d',
options.url, options.qs, duration, error.message, error.statusCode);
});
log.error(`get api fail: use: ${duration}ms, statusCode: ${error.statusCode}, error: ${error.message}`);
return ret;
// 使用缓存的时候,读取二级缓存
if (config.useCache) {
return this._requestFromCache(options, true);
}
return Promise.reject({
error: '接口调用失败'
});
});
}
/**
* multi get
* @params: urls => Array[Object[url[string], data[object]]]
* 读取缓存
* @param {[object]} options
* @param {[boolean]} slave true: 读取二级缓存
* @return {[type]}
*/
multiGet(urls) {
var rps = [];
_.forEach(urls, function(el) {
rps.push(rp({
url: `${ApiUrl}${el.url}`,
qs: el.data
}));
_requestFromCache(options, slave) {
let reqId = this._getReqId(options);
let getCache = slave ? cache.getFromSlave : cache.get;
log.info(`get cache: ${reqId}, url: ${options.url}?${qs.stringify(options.qs)}`);
return getCache(`apiCache:${reqId}`).then((result) => {
if (!_.isNil(result)) {
try {
result = JSON.parse(result);
} finally {
log.info(slave ? 'get slave cache success' : 'get master cache success');
return result;
}
}
if (!slave) {
return this._requestFromAPI(options, true, reqId);
}
}).catch(() => {
log.error(slave ? 'get slave cache fail' : 'get master cache fail');
if (!slave) {
return this._requestFromAPI(options, true, reqId);
}
});
}
return Promise.all(rps).then((d) => {
return d;
});
/**
* 使用 get 请求获取接口
* @param {[string]} url
* @param {[object]} data
* @param {[bool or number]} cacheOption 使用数字时,数字表示缓存时间
* @return {[type]}
*/
get(url, data, cacheOption) {
let options = {
url: `${ApiUrl}${url}`,
qs: data,
json: true,
timeout: 3000
};
// 从缓存获取数据
if (config.useCache && cacheOption) {
return this._requestFromCache(options);
}
return this._requestFromAPI(options, cacheOption);
}
/**
... ... @@ -80,11 +137,22 @@ class API {
* @param data Obejct
*/
post(url, data) {
return rp({
let options = {
url: `${ApiUrl}${url}`,
form: data,
method: 'post',
form: data
});
json: true,
timeout: 3000
};
return this._requestFromAPI(options);
}
all(list) {
if (_.isArray(list)) {
return Promise.all(list);
}
throw Error('the parameters of api all method should be Array!');
}
}
... ...
... ... @@ -9,11 +9,14 @@
const Promise = require('bluebird');
const Memcached = require('memcached');
const _ = require('lodash');
const log = require('./logger');
const config = require('../config/common');
let cache = new Memcached(config.memcache.server, config.memcache);
let master = new Memcached(config.memcache.master, config.memcache);
let slave = new Memcached(config.memcache.slave, config.memcache);
cache = Promise.promisifyAll(cache);
master = Promise.promisifyAll(master);
slave = Promise.promisifyAll(slave);
/**
* 获取缓存
... ... @@ -22,8 +25,9 @@ cache = Promise.promisifyAll(cache);
*/
exports.get = (key) => {
if (_.isString(key)) {
return cache.getAsync();
return master.getAsync(key);
}
return Promise.resolve();
};
... ... @@ -34,7 +38,31 @@ exports.get = (key) => {
*/
exports.getMulti = (list) => {
if (_.isArray(list)) {
return cache.getMultiAsync();
return master.getMultiAsync(list);
}
return Promise.resolve();
};
/**
* 获取缓存(从 Slave 服务器)
* @param {[string]} key 键
* @return {[type]}
*/
exports.getFromSlave = (key) => {
if (_.isString(key)) {
return slave.getAsync(key);
}
return Promise.resolve();
};
/**
* 批量获取缓存(从 Slave 服务器)
* @param {[array]} list 字符串数组
* @return {[type]}
*/
exports.getMultiFromSlave = (list) => {
if (_.isArray(list)) {
return slave.getMultiAsync(list);
}
return Promise.resolve();
};
... ... @@ -47,14 +75,35 @@ exports.getMulti = (list) => {
* @return {[type]}
*/
exports.set = (key, value, lifetime) => {
lifetime = lifetime || 3600;
lifetime = lifetime || 86400;
log.info(`write cache: ${key}`);
if (_.isObject(value)) {
value = JSON.stringify(value);
}
if (_.isString(key)) {
return master.setAsync(key, value, lifetime);
}
return Promise.resolve();
};
// if (_.isObject(value)) {
// value = JSON.stringify(value);
// }
/**
* 写缓存(到 Slave 服务器)
* @param {[type]} key 键
* @param {[type]} value 值
* @param {[type]} lifetime 生命周期
* @return {[type]}
*/
exports.setSlave = (key, value, lifetime) => {
lifetime = lifetime || 86400;
if (_.isObject(value)) {
value = JSON.stringify(value);
}
if (_.isString(key)) {
return cache.setAsync(key, value, lifetime);
return slave.setAsync(key, value, lifetime);
}
return Promise.resolve();
};
... ... @@ -66,7 +115,7 @@ exports.set = (key, value, lifetime) => {
*/
exports.del = (key) => {
if (_.isString(key)) {
return cache.delAsync();
return master.delAsync(key);
}
return Promise.resolve();
};
... ...
... ... @@ -20,9 +20,11 @@
"dependencies": {
"bluebird": "^3.3.5",
"body-parser": "^1.15.0",
"connect-memcached": "^0.2.0",
"cookie-parser": "^1.4.1",
"express": "^4.13.1",
"express-handlebars": "^3.0.0",
"express-session": "^1.13.0",
"influxdb-winston": "^1.0.1",
"lodash": "^4.12.0",
"md5": "^2.1.0",
... ... @@ -32,7 +34,8 @@
"request-promise": "^3.0.0",
"serve-favicon": "^2.3.0",
"winston": "^2.2.0",
"winston-daily-rotate-file": "^1.0.1"
"winston-daily-rotate-file": "^1.0.1",
"yoho-handlebars": "0.0.1"
},
"devDependencies": {
"autoprefixer": "^6.3.6",
... ...
... ... @@ -167,16 +167,19 @@ gulp.task('webpack-dev-server', () => {
new WebpackDevServer(webpack(devConfig), {
contentBase: '.',
publicPath: '//localhost:5000',
publicPath: '//localhost:5002',
hot: true,
stats: {
colors: true
},
headers: {
'Access-Control-Allow-Origin': '*'
}
}).listen(5000, 'localhost', (err) => {
}).listen(5002, 'localhost', (err) => {
if (err) {
throw new gutil.PluginError('webpack-dev-server', err);
}
gutil.log('[webpack-serve]', 'http://localhost:5000/');
gutil.log('[webpack-serve]', 'http://localhost:5002/');
});
});
... ...