Authored by xuqi

Merge branch 'feature/page-cache' of http://git.yoho.cn/OPENTECH/yoho-node-ci in…

…to feature/page-cache
/**
*
* @author: chenfeng<feng.chen@yoho.cn>
* @date: 16/10/19
*/
'use strict';
import Model from './model';
import utils from 'utility';
import rp from 'request-promise';
import ws from '../../lib/ws';
class CdnCache extends Model {
constructor() {
super('cdn_page');
}
async removeCache(queryUris, objectType) {
let self = this;
self._requestApi({
Action: 'RefreshObjectCaches',
ObjectPath: queryUris,
ObjectType: objectType
})
}
async _requestApi(params) {
let self = this;
let cdns = await self.findAll();
if (!cdns.length) {
return;
}
params = Object.assign(params, {
Format: 'JSON',
Version: '2014-11-11',
AccessKeyId: cdns[0].keyId,
SignatureMethod: 'HMAC-SHA1',
TimeStamp: self._getUtcTime(new Date()),
SignatureVersion: '1.0',
SignatureNonce: parseInt(Math.random() * 100000, 10) + ''
});
let sign = self._sign(params, cdns[0].keySecret);
params.Signature = sign;
rp({
uri: `${cdns[0].scheme}://${cdns[0].cdnApi}`,
qs: params,
json: true
})
.then(function (res) {
self._broadcast(`清理提交成功,RefreshTaskId:${res.RefreshTaskId}`)
})
.catch(function (err) {
self._broadcast(`清理失败:${err.response.body}`)
});
}
_getUtcTime(date) {
let year = date.getUTCFullYear();
let month = date.getUTCMonth() + 1;
let day = date.getUTCDate() < 10 ? '0' + date.getUTCDate() : date.getUTCDate();
let hour = date.getUTCHours() < 10 ? '0' + date.getUTCHours() : date.getUTCHours();
let minute = date.getUTCMinutes() < 10 ? '0' + date.getUTCMinutes() : date.getUTCMinutes();
let second = date.getUTCSeconds() < 10 ? '0' + date.getUTCSeconds() : date.getUTCSeconds();
month = month < 10 ? '0' + month : month;
return `${year}-${month}-${day}T${hour}:${minute}:${second}Z`
}
_sign(params, keySecret) {
let data = {};
Object.keys(params).sort().forEach(k => data[k]=params[k]);
let signString = '';
Object.keys(data).forEach(k => {
data[k] = encodeURIComponent(data[k])
signString += `&${k}=${data[k]}`;
});
if (signString) {
signString = `GET&%2F&${encodeURIComponent(signString.substring(1, signString.length))}`;
let sign = utils.hmac('sha1', keySecret + '&', signString);
return sign;
}
return '';
}
_broadcast(message) {
console.log(message)
ws.broadcast(`/cdn_cache/log`, {
message: message
});
}
async init() {
let count = await this.count({});
if (count === 0) {
await this.insert({
scheme: 'https',
cdnApi: 'cdn.aliyuncs.com',
keyId: 'Mt6m3xVdWddj8bDe',
keySecret: 'ExHpmJJ7ayJEEMz2LUffKRe3ehMGDs'
})
}
}
}
export default CdnCache;
\ No newline at end of file
... ...
... ... @@ -11,6 +11,8 @@ import UserModel from './user';
import HotfixModel from './hotfix';
import OperationLoggerModel from './operation_logger';
import PageCacheModel from './page_cache';
import CdnCacheModel from './cdn_cache';
import ProductCacheModel from './product_cache';
import MemcachedHostModel from './memcached_host';
import DegradeModel from './degrade';
import DegradeServerModel from './degrade_server';
... ... @@ -27,12 +29,16 @@ const User = new UserModel();
const Hotfix = new HotfixModel();
const OperationLogger = new OperationLoggerModel();
const PageCache = new PageCacheModel();
const CdnCache = new CdnCacheModel();
const ProductCache = new ProductCacheModel();
const MemcachedHost = new MemcachedHostModel();
const Degrade = new DegradeModel();
const DegradeServer = new DegradeServerModel();
User.init();
PageCache.init();
CdnCache.init();
ProductCache.init();
Degrade.init();
... ... @@ -45,6 +51,8 @@ export {
Hotfix,
OperationLogger,
PageCache,
CdnCache,
ProductCache,
MemcachedHost,
RestartInfo,
DeleteRestartInfo,
... ...
... ... @@ -28,7 +28,7 @@ class PageCahe extends Model {
* @param {[array]} keys [key列表]
* @param {[string]} storeTableName [pc和wap的标识]
*/
async removeCache(queryUris, storeTableName) {
async removeCache(queryUris, storeTableName, serverType) {
let self = this;
let queryUriList = queryUris.split('\n');
... ... @@ -49,8 +49,7 @@ class PageCahe extends Model {
// }
}
let servers = await self.findAll();
let servers = await self.find({server: { $in: serverType }});
self._broadcast(`共${servers.length}nginx准备清理..`)
for (let i = 0; i < servers.length; i++) {
try {
... ... @@ -67,9 +66,9 @@ class PageCahe extends Model {
* [全量清理缓存]
* @param {[string]} storeTableName [pc和wap的标识]
*/
async removeAllCache(storeTableName) {
async removeAllCache(storeTableName, serverType) {
let self = this;
let servers = await self.findAll();
let servers = await self.find({server: { $in: serverType }});
self._broadcast(`共${servers.length}nginx准备清理`)
for (let i = 0; i < servers.length; i++) {
... ... @@ -222,15 +221,26 @@ class PageCahe extends Model {
// password: '1',
// port: 22,
// tag: 'nginx2',
// cachepath: '/usr/local/nginx'
// cachepath: '/usr/local/nginx',
// server: 'qCloud'
// });
// await this.insert({
// host: '127.0.0.1',
// username: 'chenfeng',
// password: '1',
// port: 22,
// tag: 'nginx2',
// cachepath: '/usr/local/nginx',
// server: 'AWS'
// });
// await this.insert({
// host: '10.66.1.2',
// username: 'node',
// username: 'www',
// password: 'yoho9646',
// port: 22,
// tag: 'nginx2',
// cachepath: '/usr/local/openresty/nginx'
// cachepath: '/usr/local/openresty/nginx',
// server: 'qCloud'
// });
await this.insert({
host: '10.66.1.3',
... ... @@ -238,32 +248,54 @@ class PageCahe extends Model {
password: 'yoho9646',
port: 22,
tag: 'nginx3',
cachepath: '/usr/local/openresty/nginx'
cachepath: '/usr/local/openresty/nginx',
server: 'qCloud'
});
// await this.insert({
// host: '10.66.1.15',
// username: 'node',
// username: 'www',
// password: 'yoho9646',
// port: 22,
// tag: 'nginx15',
// cachepath: '/usr/local/openresty/nginx'
// cachepath: '/usr/local/openresty/nginx',
// server: 'qCloud'
// });
// await this.insert({
// host: '10.66.1.84',
// username: 'node',
// username: 'www',
// password: 'yoho9646',
// port: 22,
// tag: 'nginx84',
// cachepath: '/usr/local/openresty/nginx'
// cachepath: '/usr/local/openresty/nginx',
// server: 'qCloud'
// });
// await this.insert({
// host: '10.66.1.97',
// username: 'node',
// username: 'www',
// password: 'yoho9646',
// port: 22,
// tag: 'nginx97',
// cachepath: '/usr/local/openresty/nginx'
// cachepath: '/usr/local/openresty/nginx',
// server: 'qCloud'
// });
await this.insert({
host: '172.31.23.111',
username: 'www',
password: 'yoho9646',
port: 22,
tag: 'aws1',
cachepath: '/usr/local/openresty/nginx',
server: 'AWS'
});
await this.insert({
host: '172.31.21.139',
username: 'www',
password: 'yoho9646',
port: 22,
tag: 'aws2',
cachepath: '/usr/local/openresty/nginx',
server: 'AWS'
});
}
}
}
... ...
/**
*
* @author: chenfeng<feng.chen@yoho.cn>
* @date: 16/10/20
*/
import Model from './model';
import rp from 'request-promise';
import ws from '../../lib/ws';
class ProductCache extends Model{
constructor() {
super('product_cache');
}
async removePriceCache(list) {
let self = this;
let apis = await self.findAll();
if (!apis.length) {
return;
}
let url = apis[0].priceCacheApi;
await self._batchPost(list, url);
}
async removeProductCache(list) {
let self = this;
let apis = await self.findAll();
if (!apis.length) {
return;
}
let url = apis[0].productCacheApi;
await self._batchPost(list, url);
}
async _batchPost(list, url) {
let self = this;
let interval = 2;
return new Promise(async (resolve, reject) => {
let tick = parseInt(list.length / interval, 10) + (list.length % interval > 0 ? 1 : 0);
let post = async (i, len) => {
if (i < len) {
let limit = list.length > (i + 1) * interval ? (i + 1) * interval : list.length;
let datas = list.slice(i * interval, limit);
try {
await self._postApi(datas, url);
self._broadcast(`进度:${limit}/${list.length}`)
} catch(err) {
self._broadcast(`错误:${err.message}`)
}
setTimeout(async () => {
await post(++i, tick);
}, 500);
} else {
resolve();
}
}
await post(0, tick);
});
}
async _postApi(data, url) {
return new Promise((resolve, reject) => {
rp({
method: 'POST',
uri: url,
body: data,
json: true
})
.then(function (res) {
res.code === 200 ? resolve(res) : reject(res);
})
.catch(function (err) {
console.log(err.response.body);
reject(err.response.body)
});
})
}
_broadcast(message) {
console.log(message)
ws.broadcast(`/product_cache/log`, {
message: message
});
}
async init() {
let count = await this.count({});
if (count === 0) {
await this.insert({
priceCacheApi: 'http://service-test3.yohops.com:9999/erp/clear/batch/productPriceCache',
productCacheApi: 'http://service-test3.yohops.com:9999/erp/clear/batch/productCache'
});
}
}
}
export default ProductCache;
\ No newline at end of file
... ...
/**
*
* @author: chenfeng<feng.chen@yoho.cn>
* @date: 16/10/19
*/
'use strict';
import Router from 'koa-router';
import {
CdnCache
} from '../../models'
const r = new Router();
const cdnCache = {
async query(ctx) {
let date = {
typeList: [{
name: '文件',
typeName: 'File',
}, {
name: '目录',
typeName: 'Directory',
}]
}
await ctx.render('action/cdn_cache', date);
},
async clear(ctx) {
let queryUris = ctx.request.body.query_uri;
let objectType = ctx.request.body.object_type;
if (queryUris.trim() && objectType) {
CdnCache.removeCache(queryUris, objectType);
}
return ctx.body = {
code: 200
};
}
}
r.get('/query', cdnCache.query);
r.post('/clear', cdnCache.clear);
export default r;
\ No newline at end of file
... ...
... ... @@ -24,6 +24,11 @@ const pageCahe = {
tableName: 'ngx_cache_wap',
name: 'Wap'
}],
serverList: [{
name: 'AWS'
}, {
name: 'qCloud'
}],
count: 0
};
await ctx.render('action/page_cache', data);
... ... @@ -31,16 +36,25 @@ const pageCahe = {
async clear(ctx) {
let queryUris = ctx.request.body.query_uri;
let storeTableName = ctx.request.body.table_name;
if (queryUris && storeTableName) {
await PageCache.removeCache(queryUris, storeTableName);
let serverType = ctx.request.body.server;
let servers = serverType.split(',').filter(server => server);
if (queryUris && storeTableName && servers.length) {
PageCache.removeCache(queryUris, storeTableName, servers);
}
return ctx.body = {
code: 200
};
},
async clearAll(ctx) {
let storeTableName = ctx.request.body.table_name;
if (storeTableName) {
await PageCache.removeAllCache(storeTableName);
let serverType = ctx.request.body.server;
let servers = serverType.split(',').filter(server => server);
if (storeTableName && servers.length) {
PageCache.removeAllCache(storeTableName, servers);
}
return ctx.body = {
code: 200
};
}
}
r.get('/query', pageCahe.query);
... ...
/**
*
* @author: chenfeng<feng.chen@yoho.cn>
* @date: 16/10/19
*/
'use strict';
import Router from 'koa-router';
import {
ProductCache
} from '../../models'
const r = new Router();
const productCache = {
async query(ctx) {
await ctx.render('action/product_cache');
},
async clear(ctx) {
let type = ctx.request.body.type;
if (type) {
ProductCache.removePriceCache([51068877, 51068893, 51117073])
}
return ctx.body = {
code: 200
};
}
}
r.get('/query', productCache.query);
r.post('/clear', productCache.clear);
export default r;
\ No newline at end of file
... ...
... ... @@ -9,7 +9,9 @@ import monitor from './actions/monitor';
import users from './actions/users';
import hotfix from './actions/hotfix';
import operationLog from './actions/operation_log';
import pageCahe from './actions/page_cache';
import pageCache from './actions/page_cache';
import cdnCache from './actions/cdn_cache';
import productCache from './actions/product_cache';
import apiCache from './actions/api_cache';
import degrade from './actions/degrade';
... ... @@ -37,9 +39,10 @@ export default function (app) {
base.use('/users', users.routes(), users.allowedMethods());
base.use('/hotfix', hotfix.routes(), hotfix.allowedMethods());
base.use('/operation', operationLog.routes(), operationLog.allowedMethods());
base.use('/page_cache', pageCahe.routes(), pageCahe.allowedMethods());
base.use('/page_cache', pageCache.routes(), pageCache.allowedMethods());
base.use('/cdn_cache', cdnCache.routes(), cdnCache.allowedMethods());
base.use('/product_cache', productCache.routes(), productCache.allowedMethods());
base.use('/api_cache', apiCache.routes(), apiCache.allowedMethods());
base.use('/degrade', degrade.routes(), degrade.allowedMethods());
base.use('', index.routes(), index.allowedMethods());
... ...
<div class="pageheader">
<div class="media">
<div class="pageicon pull-left">
<i class="fa fa-th-list"></i>
</div>
<div class="media-body">
<ul class="breadcrumb">
<li><a href="/"><i class="glyphicon glyphicon-home"></i></a></li>
<li><a href="/servers">缓存管理</a></li>
</ul>
<h4>清理CdnCahe缓存</h4>
</div>
</div>
<!-- media -->
</div>
<div class="contentpanel">
<div class="row cache-panel">
<div class="panel panel-default">
<div class="panel-body">
<div class="col-sm-6">
<div class="operations mb20">
</div>
<div class="query">
<textarea name="" id="uri" cols="30" rows="20" class="form-control" placeholder="支持多条换行输入"></textarea>
</div>
</div>
<div class="col-sm-6">
<div class="panel">
<div class="panel-heading">
<h4 class="panel-title">日志</h4>
</div><!-- panel-heading -->
<div class="panel-body yoho-log-dark">
<div class="results-list ">
</div><!-- results-list -->
</div><!-- panel-body -->
</div><!-- panel -->
</div>
</div>
<div class="panel-footer">
<select id="objectType" class="form-control input-sm selcet-auto pull-left mr20">
{{#each typeList}}
<option value="{{typeName}}">{{name}}</option>
{{/each}}
</select>
<button class="btn btn-warning btn-clear pull-left">清除</button>
<div class="result pull-left ml20"></div>
</div>
</div>
</div>
</div>
<script>
var posing = false;
$(document).on('ready pjax:success', function() {
$('.btn-clear').click(function() {
var typeName = $('#objectType').val();
var uri = $('#uri').val();
if (uri && typeName) {
$logs.empty();
$.post('/cdn_cache/clear', {
query_uri: uri,
object_type: typeName
}, function(res) {
});
}
})
function layoutResize() {
$('.yoho-log-dark').height($('body').height() - 450);
}
$(window).resize(function() {
layoutResize();
});
layoutResize();
var $logs = $('.yoho-log-dark .results-list');
var $dark = $('.yoho-log-dark');
function appendLog(message) {
var html = '<p><span class="message">- ' + message + '</span></p>';
$logs.append(html);
$dark.scrollTop($logs[0].scrollHeight);
}
var ws = io();
ws.on('connect', function() {
ws.on('/cdn_cache/log', function(data) {
appendLog(data.message)
});
});
});
</script>
\ No newline at end of file
... ...
... ... @@ -8,7 +8,7 @@
<li><a href="/"><i class="glyphicon glyphicon-home"></i></a></li>
<li><a href="/servers">缓存管理</a></li>
</ul>
<h4>清理缓存</h4>
<h4>清理PageCahe缓存</h4>
</div>
</div>
<!-- media -->
... ... @@ -23,7 +23,7 @@
</div>
<div class="query">
<textarea name="" id="uri" cols="30" rows="20" class="form-control" placeholder="支持多条换行输入">http://m.yohobuy.com</textarea>
<textarea name="" id="uri" cols="30" rows="20" class="form-control" placeholder="支持多条换行输入"></textarea>
</div>
</div>
<div class="col-sm-6">
... ... @@ -39,7 +39,17 @@
</div><!-- panel -->
</div>
</div>
<div class="panel-footer">
<div class="panel-footer ">
<div class="ckbox ckbox-primary pull-left mr20">
<input type="checkbox" id="chk_aws" value="AWS" name="server" checked="checked">
<label for="chk_aws">AWS</label>
</div>
<div class="ckbox ckbox-primary pull-left mr20">
<input type="checkbox" value="qCloud" id="chk_qcloud" name="server" checked="checked">
<label for="chk_qcloud">qCloud</label>
</div>
<select id="stores" class="form-control input-sm selcet-auto pull-left mr20">
{{#each storeList}}
<option value="{{tableName}}">{{name}}</option>
... ... @@ -56,49 +66,34 @@
<script>
var posing = false;
$(document).on('ready pjax:success', function() {
$('.btn-success').click(function() {
if (posing) {
return;
}
posing = true;
$('.page-count>span').text('查询中..');
$('.btn-search').addClass('disabled');
var tableName = $('#stores').val();
var uri = $('#uri').val();
if (uri && tableName) {
$.post('/page_cache/search', {
query_uri: uri,
table_name: tableName
}, function (res) {
posing = false;
$('.btn-search').removeClass('disabled');
if (res.code === 200) {
$('.page-count>span').text(res.data);
} else {
$('.page-count>span').text('查询失败');
}
})
}
});
$('.btn-clear').click(function() {
var servers = "";
$(':checkbox[name="server"]:checked').each(function() {
servers += $(this).val() + ',';
})
var tableName = $('#stores').val();
var uri = $('#uri').val();
if (uri && tableName) {
if (uri && tableName && servers) {
$logs.empty();
$.post('/page_cache/clear', {
query_uri: uri,
table_name: tableName
table_name: tableName,
server: servers
}, function(res) {
});
}
})
$('.btn-all-clear').click(function() {
var servers = "";
$(':checkbox[name="server"]:checked').each(function() {
servers += $(this).val() + ',';
})
var tableName = $('#stores').val();
if (tableName) {
if (tableName && servers) {
$logs.empty();
$.post('/page_cache/clear/all', {
table_name: tableName
table_name: tableName,
server: servers
}, function(res) {
});
}
... ...
<div class="pageheader">
<div class="media">
<div class="pageicon pull-left">
<i class="fa fa-th-list"></i>
</div>
<div class="media-body">
<ul class="breadcrumb">
<li><a href="/"><i class="glyphicon glyphicon-home"></i></a></li>
<li><a href="/servers">缓存管理</a></li>
</ul>
<h4>清理商品缓存</h4>
</div>
</div>
<!-- media -->
</div>
<div class="contentpanel">
<div class="row cache-panel">
<div class="panel panel-default">
<div class="panel-body">
<div class="col-sm-6">
<div class="operations mb20">
</div>
<div class="query">
<textarea name="" id="uri" cols="30" rows="20" class="form-control" placeholder="支持多条换行输入"></textarea>
</div>
</div>
<div class="col-sm-6">
<div class="panel">
<div class="panel-heading">
<h4 class="panel-title">日志</h4>
</div><!-- panel-heading -->
<div class="panel-body yoho-log-dark">
<div class="results-list ">
</div><!-- results-list -->
</div><!-- panel-body -->
</div><!-- panel -->
</div>
</div>
<div class="panel-footer">
<select id="selectType" class="form-control input-sm selcet-auto pull-left mr20">
<option value="1">批量变价</option>
<option value="2">其它批量</option>
</select>
<button class="btn btn-warning btn-clear pull-left">清除</button>
<div class="result pull-left ml20"></div>
</div>
</div>
</div>
</div>
<script>
var posing = false;
$(document).on('ready pjax:success', function() {
$('.btn-clear').click(function() {
var selectType = $('#selectType').val();
if (selectType) {
$logs.empty();
$.post('/product_cache/clear', {
type: selectType
}, function(res) {
});
}
})
function layoutResize() {
$('.yoho-log-dark').height($('body').height() - 450);
}
$(window).resize(function() {
layoutResize();
});
layoutResize();
var $logs = $('.yoho-log-dark .results-list');
var $dark = $('.yoho-log-dark');
function appendLog(message) {
var html = '<p><span class="message">- ' + message + '</span></p>';
$logs.append(html);
$dark.scrollTop($logs[0].scrollHeight);
}
var ws = io();
ws.on('connect', function() {
ws.on('/product_cache/log', function(data) {
appendLog(data.message)
});
});
});
</script>
\ No newline at end of file
... ...
... ... @@ -75,14 +75,20 @@
<span class="label label-serstatus"></span>
</div>
<div class="clearfix mt5">
<div class="col-xs-3 project-env" data-env="test">
<div class="col-xs-4 project-env" data-env="test">
<h5 class="md-title mt10">当前状态</h5>
</div>
<div class="col-xs-4">
<h5 class="md-title mt10">进程状态</h5>
</div>
</div>
<div class="clearfix mt5">
<div class="col-xs-4 project-env" data-env="test">
<span class="label label-success deploy-log-btn" data-host="{{host}}"><i
class="fa fa-spinner fa-spin fa-fw margin-bottom"></i> <b>{{#if info}}{{info.state}}{{^}}
未知部署{{/if}}</b></span>
</div>
<div class="col-xs-3">
<h5 class="md-title mt10">进程状态</h5>
<div class="col-xs-4">
<span class="label label-status"></span>
</div>
</div>
... ...
... ... @@ -28,8 +28,9 @@
<li class="parent"><a href=""><i class="fa fa-history"></i> <span>缓存管理</span></a>
<ul class="children">
<li><a href="/page_cache/query">PageCahe清理</a></li>
{{!-- <li><a href="/cdn_cache/query">CDN清理</a></li>
<li><a href="/product_cache/query">商品清理</a></li> --}}
<li><a href="/api_cache">ApiCahe清理</a></li>
{{!-- <li><a href="/cdn_cache/query">CDN清理</a></li> --}}
</ul>
</li>
{{#if is_master}}
... ...
... ... @@ -62,7 +62,8 @@
"shelljs": "^0.7.0",
"socket.io": "^1.4.6",
"ssh2": "^0.5.0",
"tar": "^2.2.1"
"tar": "^2.2.1",
"utility": "^1.8.0"
},
"devDependencies": {
"ava": "^0.14.0",
... ...