Authored by 陈峰

pagecache支持多云

/**
*
* @author: chenfeng<feng.chen@yoho.cn>
* @date: 16/10/19
*/
'use strict';
import Model from './model';
import utils from 'utility';
import rp from 'request-promise';
import ws from '../../lib/ws';
class CdnCache extends Model {
constructor() {
super('cdn_page');
}
async removeCache(queryUris, objectType) {
let self = this;
self._requestApi({
Action: 'RefreshObjectCaches',
ObjectPath: queryUris,
ObjectType: objectType
})
}
async _requestApi(params) {
let self = this;
let cdns = await self.findAll();
if (!cdns.length) {
return;
}
params = Object.assign(params, {
Format: 'JSON',
Version: '2014-11-11',
AccessKeyId: cdns[0].keyId,
SignatureMethod: 'HMAC-SHA1',
TimeStamp: self._getUtcTime(new Date()),
SignatureVersion: '1.0',
SignatureNonce: parseInt(Math.random() * 100000, 10) + ''
});
let sign = self._sign(params, cdns[0].keySecret);
params.Signature = sign;
rp({
uri: `${cdns[0].scheme}://${cdns[0].cdnApi}`,
qs: params,
json: true
})
.then(function (res) {
self._broadcast(`清理提交成功,RefreshTaskId:${res.RefreshTaskId}`)
})
.catch(function (err) {
self._broadcast(`清理失败:${err.response.body}`)
});
}
_getUtcTime(date) {
let year = date.getUTCFullYear();
let month = date.getUTCMonth() + 1;
let day = date.getUTCDate() < 10 ? '0' + date.getUTCDate() : date.getUTCDate();
let hour = date.getUTCHours() < 10 ? '0' + date.getUTCHours() : date.getUTCHours();
let minute = date.getUTCMinutes() < 10 ? '0' + date.getUTCMinutes() : date.getUTCMinutes();
let second = date.getUTCSeconds() < 10 ? '0' + date.getUTCSeconds() : date.getUTCSeconds();
month = month < 10 ? '0' + month : month;
return `${year}-${month}-${day}T${hour}:${minute}:${second}Z`
}
_sign(params, keySecret) {
let data = {};
Object.keys(params).sort().forEach(k => data[k]=params[k]);
let signString = '';
Object.keys(data).forEach(k => {
data[k] = encodeURIComponent(data[k])
signString += `&${k}=${data[k]}`;
});
if (signString) {
signString = `GET&%2F&${encodeURIComponent(signString.substring(1, signString.length))}`;
let sign = utils.hmac('sha1', keySecret + '&', signString);
return sign;
}
return '';
}
_broadcast(message) {
console.log(message)
ws.broadcast(`/cdn_cache/log`, {
message: message
});
}
async init() {
let count = await this.count({});
if (count === 0) {
await this.insert({
scheme: 'https',
cdnApi: 'cdn.aliyuncs.com',
keyId: 'Mt6m3xVdWddj8bDe',
keySecret: 'ExHpmJJ7ayJEEMz2LUffKRe3ehMGDs'
})
}
}
}
export default CdnCache;
\ No newline at end of file
... ...
... ... @@ -9,6 +9,8 @@ import UserModel from './user';
import HotfixModel from './hotfix';
import OperationLoggerModel from './operation_logger';
import PageCacheModel from './page_cache';
import CdnCacheModel from './cdn_cache';
import ProductCacheModel from './product_cache';
shelljs.mkdir('-p', config.dbDir);
... ... @@ -20,9 +22,13 @@ const User = new UserModel();
const Hotfix = new HotfixModel();
const OperationLogger = new OperationLoggerModel();
const PageCache = new PageCacheModel();
const CdnCache = new CdnCacheModel();
const ProductCache = new ProductCacheModel();
User.init();
PageCache.init();
CdnCache.init();
ProductCache.init();
export {
Server,
... ... @@ -32,5 +38,7 @@ export {
User,
Hotfix,
OperationLogger,
PageCache
PageCache,
CdnCache,
ProductCache
};
\ No newline at end of file
... ...
... ... @@ -28,7 +28,7 @@ class PageCahe extends Model {
* @param {[array]} keys [key列表]
* @param {[string]} storeTableName [pc和wap的标识]
*/
async removeCache(queryUris, storeTableName) {
async removeCache(queryUris, storeTableName, serverType) {
let self = this;
let queryUriList = queryUris.split('\n');
... ... @@ -49,8 +49,7 @@ class PageCahe extends Model {
// }
}
let servers = await self.findAll();
let servers = await self.find({server: serverType});
self._broadcast(`共${servers.length}nginx准备清理..`)
for (let i = 0; i < servers.length; i++) {
try {
... ... @@ -67,9 +66,9 @@ class PageCahe extends Model {
* [全量清理缓存]
* @param {[string]} storeTableName [pc和wap的标识]
*/
async removeAllCache(storeTableName) {
async removeAllCache(storeTableName, serverType) {
let self = this;
let servers = await self.findAll();
let servers = await self.find({server: serverType});
self._broadcast(`共${servers.length}nginx准备清理`)
for (let i = 0; i < servers.length; i++) {
... ... @@ -216,14 +215,24 @@ class PageCahe extends Model {
async init() {
let count = await this.count({});
if (count === 0) {
// await this.insert({
// host: '127.0.0.1',
// username: 'chenfeng',
// password: '1',
// port: 22,
// tag: 'nginx2',
// cachepath: '/usr/local/nginx'
// });
await this.insert({
host: '127.0.0.1',
username: 'chenfeng',
password: '1',
port: 22,
tag: 'nginx2',
cachepath: '/usr/local/nginx',
server: 'qCloud'
});
await this.insert({
host: '127.0.0.1',
username: 'chenfeng',
password: '1',
port: 22,
tag: 'nginx2',
cachepath: '/usr/local/nginx',
server: 'AWS'
});
// await this.insert({
// host: '10.66.1.2',
// username: 'node',
... ... @@ -232,14 +241,15 @@ class PageCahe extends Model {
// tag: 'nginx2',
// cachepath: '/usr/local/openresty/nginx'
// });
await this.insert({
host: '10.66.1.3',
username: 'www',
password: 'yoho9646',
port: 22,
tag: 'nginx3',
cachepath: '/usr/local/openresty/nginx'
});
// await this.insert({
// host: '10.66.1.3',
// username: 'www',
// password: 'yoho9646',
// port: 22,
// tag: 'nginx3',
// cachepath: '/usr/local/openresty/nginx',
// server: 'qCloud'
// });
// await this.insert({
// host: '10.66.1.15',
// username: 'node',
... ...
/**
*
* @author: chenfeng<feng.chen@yoho.cn>
* @date: 16/10/20
*/
import Model from './model';
import rp from 'request-promise';
import ws from '../../lib/ws';
class ProductCache extends Model{
constructor() {
super('product_cache');
}
async removePriceCache(list) {
let self = this;
let apis = await self.findAll();
if (!apis.length) {
return;
}
let url = apis[0].priceCacheApi;
await self._batchPost(list, url);
}
async removeProductCache(list) {
let self = this;
let apis = await self.findAll();
if (!apis.length) {
return;
}
let url = apis[0].productCacheApi;
await self._batchPost(list, url);
}
async _batchPost(list, url) {
let self = this;
let interval = 2;
return new Promise(async (resolve, reject) => {
let tick = parseInt(list.length / interval, 10) + (list.length % interval > 0 ? 1 : 0);
let post = async (i, len) => {
if (i < len) {
let limit = list.length > (i + 1) * interval ? (i + 1) * interval : list.length;
let datas = list.slice(i * interval, limit);
try {
await self._postApi(datas, url);
self._broadcast(`进度:${limit}/${list.length}`)
} catch(err) {
self._broadcast(`错误:${err.message}`)
}
setTimeout(async () => {
await post(++i, tick);
}, 500);
} else {
resolve();
}
}
await post(0, tick);
});
}
async _postApi(data, url) {
return new Promise((resolve, reject) => {
rp({
method: 'POST',
uri: url,
body: data,
json: true
})
.then(function (res) {
res.code === 200 ? resolve(res) : reject(res);
})
.catch(function (err) {
console.log(err.response.body);
reject(err.response.body)
});
})
}
_broadcast(message) {
console.log(message)
ws.broadcast(`/product_cache/log`, {
message: message
});
}
async init() {
let count = await this.count({});
if (count === 0) {
await this.insert({
priceCacheApi: 'http://service-test3.yohops.com:9999/erp/clear/batch/productPriceCache',
productCacheApi: 'http://service-test3.yohops.com:9999/erp/clear/batch/productCache'
});
}
}
}
export default ProductCache;
\ No newline at end of file
... ...
/**
*
* @author: chenfeng<feng.chen@yoho.cn>
* @date: 16/10/19
*/
'use strict';
import Router from 'koa-router';
import {
CdnCache
} from '../../models'
const r = new Router();
const cdnCache = {
async query(ctx) {
let date = {
typeList: [{
name: '文件',
typeName: 'File',
}, {
name: '目录',
typeName: 'Directory',
}]
}
await ctx.render('action/cdn_cache', date);
},
async clear(ctx) {
let queryUris = ctx.request.body.query_uri;
let objectType = ctx.request.body.object_type;
if (queryUris.trim() && objectType) {
CdnCache.removeCache(queryUris, objectType);
}
return ctx.body = {
code: 200
};
}
}
r.get('/query', cdnCache.query);
r.post('/clear', cdnCache.clear);
export default r;
\ No newline at end of file
... ...
... ... @@ -24,6 +24,11 @@ const pageCahe = {
tableName: 'ngx_cache_wap',
name: 'Wap'
}],
serverList: [{
name: 'AWS'
}, {
name: 'qCloud'
}],
count: 0
};
await ctx.render('action/page_cache', data);
... ... @@ -31,16 +36,23 @@ const pageCahe = {
async clear(ctx) {
let queryUris = ctx.request.body.query_uri;
let storeTableName = ctx.request.body.table_name;
let serverType = ctx.request.body.server;
if (queryUris && storeTableName) {
await PageCache.removeCache(queryUris, storeTableName);
PageCache.removeCache(queryUris, storeTableName, serverType);
}
return ctx.body = {
code: 200
};
},
async clearAll(ctx) {
let storeTableName = ctx.request.body.table_name;
let serverType = ctx.request.body.server;
if (storeTableName) {
await PageCache.removeAllCache(storeTableName);
PageCache.removeAllCache(storeTableName, serverType);
}
return ctx.body = {
code: 200
};
}
}
r.get('/query', pageCahe.query);
... ...
/**
*
* @author: chenfeng<feng.chen@yoho.cn>
* @date: 16/10/19
*/
'use strict';
import Router from 'koa-router';
import {
ProductCache
} from '../../models'
const r = new Router();
const productCache = {
async query(ctx) {
await ctx.render('action/product_cache');
},
async clear(ctx) {
let type = ctx.request.body.type;
if (type) {
ProductCache.removePriceCache([51068877, 51068893, 51117073])
}
return ctx.body = {
code: 200
};
}
}
r.get('/query', productCache.query);
r.post('/clear', productCache.clear);
export default r;
\ No newline at end of file
... ...
... ... @@ -9,7 +9,9 @@ import monitor from './actions/monitor';
import users from './actions/users';
import hotfix from './actions/hotfix';
import operationLog from './actions/operation_log';
import pageCahe from './actions/page_cache';
import pageCache from './actions/page_cache';
import cdnCache from './actions/cdn_cache';
import productCache from './actions/product_cache';
const noAuth = new Router();
const base = new Router();
... ... @@ -35,7 +37,9 @@ export default function (app) {
base.use('/users', users.routes(), users.allowedMethods());
base.use('/hotfix', hotfix.routes(), hotfix.allowedMethods());
base.use('/operation', operationLog.routes(), operationLog.allowedMethods());
base.use('/page_cache', pageCahe.routes(), pageCahe.allowedMethods());
base.use('/page_cache', pageCache.routes(), pageCache.allowedMethods());
base.use('/cdn_cache', cdnCache.routes(), cdnCache.allowedMethods());
base.use('/product_cache', productCache.routes(), productCache.allowedMethods());
base.use('', index.routes(), index.allowedMethods());
... ...
<div class="pageheader">
<div class="media">
<div class="pageicon pull-left">
<i class="fa fa-th-list"></i>
</div>
<div class="media-body">
<ul class="breadcrumb">
<li><a href="/"><i class="glyphicon glyphicon-home"></i></a></li>
<li><a href="/servers">缓存管理</a></li>
</ul>
<h4>清理CdnCahe缓存</h4>
</div>
</div>
<!-- media -->
</div>
<div class="contentpanel">
<div class="row cache-panel">
<div class="panel panel-default">
<div class="panel-body">
<div class="col-sm-6">
<div class="operations mb20">
</div>
<div class="query">
<textarea name="" id="uri" cols="30" rows="20" class="form-control" placeholder="支持多条换行输入"></textarea>
</div>
</div>
<div class="col-sm-6">
<div class="panel">
<div class="panel-heading">
<h4 class="panel-title">日志</h4>
</div><!-- panel-heading -->
<div class="panel-body yoho-log-dark">
<div class="results-list ">
</div><!-- results-list -->
</div><!-- panel-body -->
</div><!-- panel -->
</div>
</div>
<div class="panel-footer">
<select id="objectType" class="form-control input-sm selcet-auto pull-left mr20">
{{#each typeList}}
<option value="{{typeName}}">{{name}}</option>
{{/each}}
</select>
<button class="btn btn-warning btn-clear pull-left">清除</button>
<div class="result pull-left ml20"></div>
</div>
</div>
</div>
</div>
<script>
var posing = false;
$(document).on('ready pjax:success', function() {
$('.btn-clear').click(function() {
var typeName = $('#objectType').val();
var uri = $('#uri').val();
if (uri && typeName) {
$logs.empty();
$.post('/cdn_cache/clear', {
query_uri: uri,
object_type: typeName
}, function(res) {
});
}
})
function layoutResize() {
$('.yoho-log-dark').height($('body').height() - 450);
}
$(window).resize(function() {
layoutResize();
});
layoutResize();
var $logs = $('.yoho-log-dark .results-list');
var $dark = $('.yoho-log-dark');
function appendLog(message) {
var html = '<p><span class="message">- ' + message + '</span></p>';
$logs.append(html);
$dark.scrollTop($logs[0].scrollHeight);
}
var ws = io();
ws.on('connect', function() {
ws.on('/cdn_cache/log', function(data) {
appendLog(data.message)
});
});
});
</script>
\ No newline at end of file
... ...
... ... @@ -8,7 +8,7 @@
<li><a href="/"><i class="glyphicon glyphicon-home"></i></a></li>
<li><a href="/servers">缓存管理</a></li>
</ul>
<h4>清理缓存</h4>
<h4>清理PageCahe缓存</h4>
</div>
</div>
<!-- media -->
... ... @@ -23,7 +23,7 @@
</div>
<div class="query">
<textarea name="" id="uri" cols="30" rows="20" class="form-control" placeholder="支持多条换行输入">http://m.yohobuy.com</textarea>
<textarea name="" id="uri" cols="30" rows="20" class="form-control" placeholder="支持多条换行输入"></textarea>
</div>
</div>
<div class="col-sm-6">
... ... @@ -40,6 +40,11 @@
</div>
</div>
<div class="panel-footer">
<select id="servers" class="form-control input-sm selcet-auto pull-left mr20">
{{#each serverList}}
<option value="{{name}}">{{name}}</option>
{{/each}}
</select>
<select id="stores" class="form-control input-sm selcet-auto pull-left mr20">
{{#each storeList}}
<option value="{{tableName}}">{{name}}</option>
... ... @@ -56,39 +61,16 @@
<script>
var posing = false;
$(document).on('ready pjax:success', function() {
$('.btn-success').click(function() {
if (posing) {
return;
}
posing = true;
$('.page-count>span').text('查询中..');
$('.btn-search').addClass('disabled');
var tableName = $('#stores').val();
var uri = $('#uri').val();
if (uri && tableName) {
$.post('/page_cache/search', {
query_uri: uri,
table_name: tableName
}, function (res) {
posing = false;
$('.btn-search').removeClass('disabled');
if (res.code === 200) {
$('.page-count>span').text(res.data);
} else {
$('.page-count>span').text('查询失败');
}
})
}
});
$('.btn-clear').click(function() {
var tableName = $('#stores').val();
var server = $('#servers').val();
var uri = $('#uri').val();
if (uri && tableName) {
$logs.empty();
$.post('/page_cache/clear', {
query_uri: uri,
table_name: tableName
table_name: tableName,
server: server
}, function(res) {
});
}
... ...
<div class="pageheader">
<div class="media">
<div class="pageicon pull-left">
<i class="fa fa-th-list"></i>
</div>
<div class="media-body">
<ul class="breadcrumb">
<li><a href="/"><i class="glyphicon glyphicon-home"></i></a></li>
<li><a href="/servers">缓存管理</a></li>
</ul>
<h4>清理商品缓存</h4>
</div>
</div>
<!-- media -->
</div>
<div class="contentpanel">
<div class="row cache-panel">
<div class="panel panel-default">
<div class="panel-body">
<div class="col-sm-6">
<div class="operations mb20">
</div>
<div class="query">
<textarea name="" id="uri" cols="30" rows="20" class="form-control" placeholder="支持多条换行输入"></textarea>
</div>
</div>
<div class="col-sm-6">
<div class="panel">
<div class="panel-heading">
<h4 class="panel-title">日志</h4>
</div><!-- panel-heading -->
<div class="panel-body yoho-log-dark">
<div class="results-list ">
</div><!-- results-list -->
</div><!-- panel-body -->
</div><!-- panel -->
</div>
</div>
<div class="panel-footer">
<select id="selectType" class="form-control input-sm selcet-auto pull-left mr20">
<option value="1">批量变价</option>
<option value="2">其它批量</option>
</select>
<button class="btn btn-warning btn-clear pull-left">清除</button>
<div class="result pull-left ml20"></div>
</div>
</div>
</div>
</div>
<script>
var posing = false;
$(document).on('ready pjax:success', function() {
$('.btn-clear').click(function() {
var selectType = $('#selectType').val();
if (selectType) {
$logs.empty();
$.post('/product_cache/clear', {
type: selectType
}, function(res) {
});
}
})
function layoutResize() {
$('.yoho-log-dark').height($('body').height() - 450);
}
$(window).resize(function() {
layoutResize();
});
layoutResize();
var $logs = $('.yoho-log-dark .results-list');
var $dark = $('.yoho-log-dark');
function appendLog(message) {
var html = '<p><span class="message">- ' + message + '</span></p>';
$logs.append(html);
$dark.scrollTop($logs[0].scrollHeight);
}
var ws = io();
ws.on('connect', function() {
ws.on('/product_cache/log', function(data) {
appendLog(data.message)
});
});
});
</script>
\ No newline at end of file
... ...
... ... @@ -28,7 +28,8 @@
<li class="parent"><a href=""><i class="fa fa-history"></i> <span>缓存管理</span></a>
<ul class="children">
<li><a href="/page_cache/query">PageCahe清理</a></li>
{{!-- <li><a href="/cdn_cache/query">CDN清理</a></li> --}}
<li><a href="/cdn_cache/query">CDN清理</a></li>
<li><a href="/product_cache/query">商品清理</a></li>
</ul>
</li>
{{#if is_master}}
... ...
0 info it worked if it ends with ok
1 verbose cli [ '/usr/local/bin/node',
1 verbose cli '/usr/local/bin/npm',
1 verbose cli 'run',
1 verbose cli 'babel',
1 verbose cli 'app.js' ]
2 info using npm@3.8.9
3 info using node@v6.2.0
4 verbose run-script [ 'prebabel', 'babel', 'postbabel' ]
5 info lifecycle yoho-node-ci@0.0.1~prebabel: yoho-node-ci@0.0.1
6 silly lifecycle yoho-node-ci@0.0.1~prebabel: no script for prebabel, continuing
7 info lifecycle yoho-node-ci@0.0.1~babel: yoho-node-ci@0.0.1
8 verbose lifecycle yoho-node-ci@0.0.1~babel: unsafe-perm in lifecycle true
9 verbose lifecycle yoho-node-ci@0.0.1~babel: PATH: /usr/local/lib/node_modules/npm/bin/node-gyp-bin:/Users/chenfeng/Documents/source/yoho/yoho-node-ci/node_modules/.bin:/usr/local/bin:/usr/local/lib/node_modules/npm/bin/node-gyp-bin:/Users/chenfeng/Documents/source/yoho/yoho-node-ci/node_modules/.bin:/usr/local/bin:/usr/local/bin:/usr/bin:/bin:/usr/sbin:/sbin:/usr/local/share/dotnet:/Users/chenfeng/Documents/tools/android-sdk-macosx/platform-tools:/Users/chenfeng/Documents/tools/android-sdk-macosx/tools:/usr/local/nginx/sbin
10 verbose lifecycle yoho-node-ci@0.0.1~babel: CWD: /Users/chenfeng/Documents/source/yoho/yoho-node-ci
11 silly lifecycle yoho-node-ci@0.0.1~babel: Args: [ '-c', 'babel-node "app.js"' ]
12 silly lifecycle yoho-node-ci@0.0.1~babel: Returned: code: 1 signal: null
13 info lifecycle yoho-node-ci@0.0.1~babel: Failed to exec babel script
14 verbose stack Error: yoho-node-ci@0.0.1 babel: `babel-node "app.js"`
14 verbose stack Exit status 1
14 verbose stack at EventEmitter.<anonymous> (/usr/local/lib/node_modules/npm/lib/utils/lifecycle.js:245:16)
14 verbose stack at emitTwo (events.js:106:13)
14 verbose stack at EventEmitter.emit (events.js:191:7)
14 verbose stack at ChildProcess.<anonymous> (/usr/local/lib/node_modules/npm/lib/utils/spawn.js:24:14)
14 verbose stack at emitTwo (events.js:106:13)
14 verbose stack at ChildProcess.emit (events.js:191:7)
14 verbose stack at maybeClose (internal/child_process.js:850:16)
14 verbose stack at Process.ChildProcess._handle.onexit (internal/child_process.js:215:5)
15 verbose pkgid yoho-node-ci@0.0.1
16 verbose cwd /Users/chenfeng/Documents/source/yoho/yoho-node-ci
17 error Darwin 16.0.0
18 error argv "/usr/local/bin/node" "/usr/local/bin/npm" "run" "babel" "app.js"
19 error node v6.2.0
20 error npm v3.8.9
21 error code ELIFECYCLE
22 error yoho-node-ci@0.0.1 babel: `babel-node "app.js"`
22 error Exit status 1
23 error Failed at the yoho-node-ci@0.0.1 babel script 'babel-node "app.js"'.
23 error Make sure you have the latest version of node.js and npm installed.
23 error If you do, this is most likely a problem with the yoho-node-ci package,
23 error not with npm itself.
23 error Tell the author that this fails on your system:
23 error babel-node "app.js"
23 error You can get information on how to open an issue for this project with:
23 error npm bugs yoho-node-ci
23 error Or if that isn't available, you can get their info via:
23 error npm owner ls yoho-node-ci
23 error There is likely additional logging output above.
24 verbose exit [ 1, true ]
... ...
... ... @@ -56,6 +56,7 @@
"nedb-promise": "^2.0.0",
"qn": "^1.3.0",
"qs": "^6.2.0",
"request-promise": "^4.1.1",
"shelljs": "^0.7.0",
"socket.io": "^1.4.6",
"ssh2": "^0.5.0",
... ...