OPENTECH / yoho-node-ci · Commits

Go to a project

Authored by 沈志敏
Commit 9f8c436a54eee87c8f726b46fa566269985d2b55 2 parents 2fa34def cae93466

Merge branch 'feature/page-cache' of git.yoho.cn:OPENTECH/yoho-node-ci into feature/page-cache 

# Conflicts:
#	apps/models/index.js
Inline Side-by-side
Showing 14 changed files with 540 additions and 3 deletions
@@ -22,6 +22,17 @@ class ApiCache { @@ -22,6 +22,17 @@ class ApiCache {
22 }); 22 });
23 } 23 }
24 24
  25 + flushAll() {
  26 + this.memcached.flush(err => {
  27 + if (err) {
  28 + this._log(`flush all fail.` + err.toString())
  29 + } else {
  30 + this._log(`flush all success`)
  31 + }
  32 + this.memcached.end();
  33 + });
  34 + }
  35 +
25 clean(key) { 36 clean(key) {
26 let begin = new Date(); 37 let begin = new Date();
27 38
@@ -58,7 +69,7 @@ class ApiCache { @@ -58,7 +69,7 @@ class ApiCache {
58 if (_.isArray(response)) { 69 if (_.isArray(response)) {
59 _.each(response, keyObj => { 70 _.each(response, keyObj => {
60 count ++ ; 71 count ++ ;
61 - if (keyObj.key && (keyObj.key.indexOf(key1) >= 0 || keyObj.key.indexOf(key2) >= 0 )) { 72 + if (keyObj.key && (keyObj.key.indexOf(key1) >= 0 || keyObj.key.indexOf(key2) >= 0 || keyObj.key.indexOf(key) >= 0)) {
62 this.delKey(keyObj.key); 73 this.delKey(keyObj.key);
63 } else { 74 } else {
64 // this._log(`skip ${keyObj.key}`) 75 // this._log(`skip ${keyObj.key}`)
@@ -66,7 +77,7 @@ class ApiCache { @@ -66,7 +77,7 @@ class ApiCache {
66 }); 77 });
67 } else { 78 } else {
68 count ++; 79 count ++;
69 - if (response.key && (response.key.indexOf(key1) >= 0 || response.key.indexOf(key2) >= 0 )) { 80 + if (response.key && (response.key.indexOf(key1) >= 0 || response.key.indexOf(key2) >= 0 || response.key.indexOf(key) >= 0)) {
70 this.delKey(response.key); 81 this.delKey(response.key);
71 } else { 82 } else {
72 // this._log(`skip ${response.key}`) 83 // this._log(`skip ${response.key}`)
  1 +'use strict';
  2 +
  3 +import Model from './model';
  4 +import _ from 'lodash';
  5 +
  6 +const defaultDegrades = [
  7 + {
  8 + path: '/pc/common/disMyYohoHover',
  9 + name: '【禁止】公共头部>>> MY有货鼠标移入显示账户信息'
  10 + },
  11 + {
  12 + path: '/pc/common/removeCartCount',
  13 + name: '【移除】公共头部>>> 购物车图标显示的购物车商品数目'
  14 + },
  15 + {
  16 + path: '/pc/common/disSearchAssociation',
  17 + name: '【禁止】公共头部>>> 搜索框执行搜索联想'
  18 + },
  19 + {
  20 + path: '/pc/common/disCartHover',
  21 + name: '【禁止】公共头部>>> 购物车图标鼠标移入请求且不显示购物车商品列表'
  22 + },
  23 + {
  24 + path: '/pc/brands/disBrandNameHover',
  25 + name: '【禁止】品牌一览>>> 品牌名移入显示品牌简介Tip'
  26 + },
  27 + {
  28 + path: '/pc/product/removeRecentView',
  29 + name: '【移除】商品列表/商品详情>>> 最近浏览'
  30 + },
  31 + {
  32 + path: '/pc/guang/removeHotTag',
  33 + name: '【移除】逛>>> 热门标签'
  34 + },
  35 + {
  36 + path: '/pc/guang/removeAd',
  37 + name: '【移除】逛>>> 广告banner'
  38 + },
  39 + {
  40 + path: '/pc/guang/removeItemComment',
  41 + name: '【移除】逛>>> 详情页评论'
  42 + },
  43 + //wap
  44 + {
  45 + path: '/wap/plustar/removeCollect',
  46 + name: '【移除】Plustar>>> 品牌收藏'
  47 + },
  48 + {
  49 + path: '/wap/plustar/removeRelatedPost',
  50 + name: '【移除】Plustar>>> 相关资讯'
  51 + },
  52 + {
  53 + path: '/wap/search/removeHotSearch',
  54 + name: '【移除】搜索>>> 热门搜索'
  55 + },
  56 + {
  57 + path: '/wap/ucenter/removePrefer',
  58 + name: '【移除】个人中心>>> 为您优选'
  59 + },
  60 + {
  61 + path: '/wap/common/removeCartCount',
  62 + name: '【移除】公共>>> 购物车icon商品数目'
  63 + },
  64 + {
  65 + path: '/wap/cart/removePrefer',
  66 + name: '【移除】购物车>>> 为您优选新品'
  67 + }
  68 +];
  69 +
  70 +class Degrade extends Model {
  71 +
  72 + constructor() {
  73 + super('degrade');
  74 + }
  75 +
  76 + async init() {
  77 + for (let i of defaultDegrades) {
  78 + let count = await this.count({
  79 + path: i.path
  80 + });
  81 +
  82 + if (count === 0) {
  83 + await this.insert(i);
  84 + }
  85 + }
  86 + }
  87 +}
  88 +
  89 +export default Degrade;
  1 +'use strict';
  2 +
  3 +import Model from './model';
  4 +
  5 +class DegradeServer extends Model {
  6 + constructor() {
  7 + super('degrade_server');
  8 + }
  9 +}
  10 +
  11 +export default DegradeServer;
@@ -12,6 +12,8 @@ import HotfixModel from './hotfix'; @@ -12,6 +12,8 @@ import HotfixModel from './hotfix';
12 import OperationLoggerModel from './operation_logger'; 12 import OperationLoggerModel from './operation_logger';
13 import PageCacheModel from './page_cache'; 13 import PageCacheModel from './page_cache';
14 import MemcachedHostModel from './memcached_host'; 14 import MemcachedHostModel from './memcached_host';
  15 +import DegradeModel from './degrade';
  16 +import DegradeServerModel from './degrade_server';
15 17
16 shelljs.mkdir('-p', config.dbDir); 18 shelljs.mkdir('-p', config.dbDir);
17 19
@@ -26,10 +28,14 @@ const Hotfix = new HotfixModel(); @@ -26,10 +28,14 @@ const Hotfix = new HotfixModel();
26 const OperationLogger = new OperationLoggerModel(); 28 const OperationLogger = new OperationLoggerModel();
27 const PageCache = new PageCacheModel(); 29 const PageCache = new PageCacheModel();
28 const MemcachedHost = new MemcachedHostModel(); 30 const MemcachedHost = new MemcachedHostModel();
  31 +const Degrade = new DegradeModel();
  32 +const DegradeServer = new DegradeServerModel();
29 33
30 User.init(); 34 User.init();
31 PageCache.init(); 35 PageCache.init();
32 36
  37 +Degrade.init();
  38 +
33 export { 39 export {
34 Server, 40 Server,
35 Building, 41 Building,
@@ -41,5 +47,7 @@ export { @@ -41,5 +47,7 @@ export {
41 PageCache, 47 PageCache,
42 MemcachedHost, 48 MemcachedHost,
43 RestartInfo, 49 RestartInfo,
44 - DeleteRestartInfo 50 + DeleteRestartInfo,
  51 + Degrade,
  52 + DegradeServer
45 }; 53 };
@@ -51,6 +51,18 @@ const api_cache = { @@ -51,6 +51,18 @@ const api_cache = {
51 return ctx.body = { 51 return ctx.body = {
52 code: 200 52 code: 200
53 }; 53 };
  54 + },
  55 +
  56 + async flushAll(ctx) {
  57 + let hosts = await MemcachedHost.findAll();
  58 +
  59 + _.each(hosts, (h) => {
  60 + (new ApiCache(h.host)).flushAll();
  61 + });
  62 +
  63 + return ctx.body = {
  64 + code: 200
  65 + };
54 } 66 }
55 }; 67 };
56 68
@@ -58,5 +70,6 @@ r.get('/', api_cache.index); @@ -58,5 +70,6 @@ r.get('/', api_cache.index);
58 r.post('/host/add', api_cache.addHost); 70 r.post('/host/add', api_cache.addHost);
59 r.post('/host/del', api_cache.delHost); 71 r.post('/host/del', api_cache.delHost);
60 r.post('/clean', api_cache.cleanKey); 72 r.post('/clean', api_cache.cleanKey);
  73 +r.post('/flush', api_cache.flushAll);
61 74
62 export default r; 75 export default r;
  1 +'use strict';
  2 +
  3 +import Router from 'koa-router';
  4 +import moment from 'moment';
  5 +import _ from 'lodash';
  6 +
  7 +import {Degrade, DegradeServer} from '../../models';
  8 +
  9 +import getter from '../../zookeeper/getter';
  10 +import setter from '../../zookeeper/setter';
  11 +
  12 +const router = new Router();
  13 +
  14 +const ctl = {
  15 + async getServer() {
  16 + let server = await DegradeServer.findAll({});
  17 +
  18 + server = _.last(server);
  19 +
  20 + if (server) {
  21 + return `${server.ip}:${server.port}`;
  22 + } else {
  23 + return 'localhost:2181';
  24 + }
  25 + },
  26 + async index (ctx) {
  27 + let count = await DegradeServer.count({});
  28 + let render;
  29 +
  30 + if (count) {
  31 + let serverPath = await ctl.getServer();
  32 + let degrades = await Degrade.findAll();
  33 +
  34 + for (let i of degrades) {
  35 + i.checked = await getter(serverPath, i.path);
  36 + }
  37 +
  38 + let pc = _.filter(degrades, o => _.startsWith(o.path, '/pc'));
  39 + let wap = _.filter(degrades, o => _.startsWith(o.path, '/wap'));
  40 +
  41 + let serverSplit = serverPath.split(':');
  42 + render = {
  43 + ip: serverSplit[0],
  44 + port: serverSplit[1],
  45 + render: {
  46 + pc: pc,
  47 + wap: wap
  48 + }
  49 + }
  50 + }
  51 +
  52 + await ctx.render('action/degrade', render);
  53 + },
  54 + async server(ctx) {
  55 + let ip = ctx.request.body.ip;
  56 + let port = ctx.request.body.port;
  57 +
  58 + let serverCount = await DegradeServer.count({});
  59 +
  60 + // keep one server
  61 + if (serverCount) {
  62 + let serverConfig = await DegradeServer.findAll({});
  63 + let id = _.last(serverConfig)._id; // get the latest item
  64 +
  65 + await DegradeServer.update({
  66 + _id: id
  67 + }, {
  68 + $set: {
  69 + ip: ip,
  70 + port: port
  71 + }
  72 + });
  73 + } else {
  74 + await DegradeServer.insert({
  75 + ip: ip,
  76 + port: port
  77 + });
  78 + }
  79 +
  80 + ctx.body = {
  81 + code: 200,
  82 + message: `${serverCount ? 'update' : 'new'} server success`
  83 + };
  84 + },
  85 + async setter(ctx) {
  86 + let {checked, id} = ctx.query;
  87 +
  88 + let theDegrade = await Degrade.findById(id);
  89 +
  90 + let path = theDegrade.path;
  91 +
  92 + let serverPath = await ctl.getServer();
  93 +
  94 + await setter(serverPath, path, checked.toString());
  95 +
  96 + ctx.body = {
  97 + code: 200,
  98 + message: 'update success'
  99 + };
  100 + }
  101 +};
  102 +
  103 +router.get('/', ctl.index);
  104 +router.post('/server', ctl.server);
  105 +router.get('/setter', ctl.setter);
  106 +
  107 +export default router;
@@ -11,6 +11,7 @@ import hotfix from './actions/hotfix'; @@ -11,6 +11,7 @@ import hotfix from './actions/hotfix';
11 import operationLog from './actions/operation_log'; 11 import operationLog from './actions/operation_log';
12 import pageCahe from './actions/page_cache'; 12 import pageCahe from './actions/page_cache';
13 import apiCache from './actions/api_cache'; 13 import apiCache from './actions/api_cache';
  14 +import degrade from './actions/degrade';
14 15
15 const noAuth = new Router(); 16 const noAuth = new Router();
16 const base = new Router(); 17 const base = new Router();
@@ -39,6 +40,8 @@ export default function (app) { @@ -39,6 +40,8 @@ export default function (app) {
39 base.use('/page_cache', pageCahe.routes(), pageCahe.allowedMethods()); 40 base.use('/page_cache', pageCahe.routes(), pageCahe.allowedMethods());
40 base.use('/api_cache', apiCache.routes(), apiCache.allowedMethods()); 41 base.use('/api_cache', apiCache.routes(), apiCache.allowedMethods());
41 42
  43 + base.use('/degrade', degrade.routes(), degrade.allowedMethods());
  44 +
42 base.use('', index.routes(), index.allowedMethods()); 45 base.use('', index.routes(), index.allowedMethods());
43 46
44 app.use(base.routes(), base.allowedMethods()); 47 app.use(base.routes(), base.allowedMethods());
@@ -19,6 +19,9 @@ @@ -19,6 +19,9 @@
19 19
20 <div class="row"> 20 <div class="row">
21 <div class="col-sm-4 col-md-3"> 21 <div class="col-sm-4 col-md-3">
  22 + <div>
  23 + <button class="btn btn-danger" id="flush-btn">Flush All</button>
  24 + </div>
22 25
23 <h4 class="md-title mb5">memcached列表</h4> 26 <h4 class="md-title mb5">memcached列表</h4>
24 <div class="host-list"> 27 <div class="host-list">
@@ -29,7 +32,9 @@ @@ -29,7 +32,9 @@
29 32
30 <div class="mb20"></div> 33 <div class="mb20"></div>
31 34
  35 + <div>
32 <button class="btn btn-success" id="add-btn">添加</button> 36 <button class="btn btn-success" id="add-btn">添加</button>
  37 + </div>
33 38
34 <br> 39 <br>
35 40
@@ -102,6 +107,18 @@ @@ -102,6 +107,18 @@
102 }); 107 });
103 }); 108 });
104 109
  110 + $('#flush-btn').click(function(){
  111 + var i = layer.confirm('确定清楚所有缓存吗？', {
  112 + btn: ['确定', '取消']
  113 + }, function() {
  114 + $.post('/api_cache/flush', function(ret) {
  115 + if (ret.code === 200) {
  116 + layer.close(i);
  117 + }
  118 + });
  119 + });
  120 + });
  121 +
105 function layoutResize() { 122 function layoutResize() {
106 $('.yoho-log-dark').height($('body').height() - 340); 123 $('.yoho-log-dark').height($('body').height() - 340);
107 } 124 }
  1 +<style>
  2 + .degrade-page {
  3 + padding: 20px;
  4 + }
  5 +
  6 + .degrade-tab li {
  7 + cursor: pointer;
  8 + }
  9 +
  10 + .pc-degrade,
  11 + .wap-degrade {
  12 + list-style: none;
  13 + padding: 20px;
  14 + }
  15 +
  16 + .err-tip {
  17 + color: #a94442;
  18 + }
  19 +</style>
  20 +<div class="degrade-page">
  21 + <div class="panel panel-info">
  22 + <div class="panel-heading">zookeeper server</div>
  23 + <div class="panel-body">
  24 + <div class="form-inline">
  25 + <div class="form-group">
  26 + <label for="server-ip">Server IP</label>
  27 + <input id="server-ip" class="form-control" type="text" placeholder="input server ip" value="{{ip}}">
  28 + </div>
  29 + <div class="form-group">
  30 + <label for="server-port">Server Port</label>
  31 + <input id="server-port" class="form-control" type="text" placeholder="input server port" value="{{port}}">
  32 + </div>
  33 + <span id="server-sure" class="btn btn-default">确定</span>
  34 + <span id="err-tip" class="err-tip hide">
  35 + <i class="glyphicon glyphicon-remove-sign"></i>
  36 + 小哥，填错了吧
  37 + </span>
  38 + </div>
  39 + </div>
  40 + </div>
  41 + <div class="panel panel-danger">
  42 + <div class="panel-heading">degrade point</div>
  43 + <div id="pjax-container" class="panel-body">
  44 + {{#render}}
  45 + <ul id="degrade-tab" class="nav nav-tabs degrade-tab" role="tablist">
  46 + <li role="presentation" class="active">
  47 + <a>PC</a>
  48 + </li>
  49 + <li role="presentation">
  50 + <a>WAP</a>
  51 + </li>
  52 + </ul>
  53 +
  54 + <ul class="pc-degrade degrade-content">
  55 + {{#each pc}}
  56 + <li data-id="{{_id}}">
  57 + <div class="checkbox">
  58 + <label>
  59 + <input type="checkbox"{{#if checked}} checked{{/if}}>
  60 + {{name}}
  61 + </label>
  62 + </div>
  63 + </li>
  64 + {{/each}}
  65 + </ul>
  66 +
  67 + <ul class="wap-degrade degrade-content hide">
  68 + {{#each wap}}
  69 + <li data-id="{{_id}}">
  70 + <div class="checkbox">
  71 + <label>
  72 + <input type="checkbox"{{#if checked}} checked{{/if}}>
  73 + {{name}}
  74 + </label>
  75 + </div>
  76 + </li>
  77 + {{/each}}
  78 + </ul>
  79 + {{^}}
  80 + Waitting for connecting to server...
  81 + {{/render}}
  82 + </div>
  83 + </div>
  84 +</div>
  85 +<script>
  86 + $(function() {
  87 + var $ip = $('#server-ip');
  88 + var $port = $('#server-port');
  89 + var $tip = $('#err-tip');
  90 +
  91 + function validateIP(ip) {
  92 + if (ip === 'localhost' ||
  93 + /^(0|[1-9]?|1\d\d?|2[0-4]\d|25[0-5])\.(0|[1-9]?|1\d\d?|2[0-4]\d|25[0-5])\.(0|[1-9]?|1\d\d?|2[0-4]\d|25[0-5])\.(0|[1-9]?|1\d\d?|2[0-4]\d|25[0-5])$/.test(ip))
  94 + {
  95 + $ip.closest('.form-group').removeClass('has-error');
  96 + return true;
  97 + }
  98 +
  99 + $ip.closest('.form-group').addClass('has-error');
  100 + return false;
  101 + }
  102 +
  103 + function validatePort(port) {
  104 + if (/^[1-9]\d*$/.test(port) && +port >= 1 && +port <= 65535) {
  105 + $port.closest('.form-group').removeClass('has-error');
  106 + return true;
  107 + }
  108 +
  109 + $port.closest('.form-group').addClass('has-error');
  110 + return false;
  111 + }
  112 +
  113 + // server
  114 + $('#server-sure').click(function() {
  115 + var ip = $.trim($ip.val());
  116 + var port = $.trim($port.val());
  117 +
  118 + if (!validateIP(ip) || !validatePort(port)) {
  119 + $tip.removeClass('hide');
  120 + return;
  121 + }
  122 +
  123 + $tip.addClass('hide');
  124 +
  125 + var serverPrompt = confirm('小伙砸，你确认修改zookeeper的server到：' + ip + ':' + port + '吗？')
  126 +
  127 + if (!serverPrompt) {
  128 + return;
  129 + }
  130 +
  131 + $.ajax({
  132 + type: 'POST',
  133 + url: '/degrade/server',
  134 + data: {
  135 + ip: ip,
  136 + port: port
  137 + }
  138 + }).then(function(data) {
  139 + if (data.code === 200) {
  140 + $.pjax.reload('#pjax-container');
  141 + }
  142 + });
  143 + });
  144 +
  145 + $('#degrade-tab').on('click', 'li', function() {
  146 + var $this = $(this);
  147 +
  148 + if ($this.hasClass('active')) {
  149 + return;
  150 + }
  151 +
  152 + $('li', $('#degrade-tab')).toggleClass('active');
  153 +
  154 + var index = $this.index();
  155 +
  156 + if (index === 0) {
  157 +
  158 + //PC active
  159 + $('.pc-degrade').removeClass('hide');
  160 + $('.wap-degrade').addClass('hide');
  161 + } else {
  162 +
  163 + // wap active
  164 + $('.wap-degrade').removeClass('hide');
  165 + $('.pc-degrade').addClass('hide');
  166 + }
  167 + });
  168 +
  169 + // change
  170 + $('.degrade-content input[type="checkbox"]').change(function() {
  171 + var $checkbox = $(this),
  172 + $li = $checkbox.closest('li');
  173 +
  174 + var checked = $checkbox.prop('checked');
  175 +
  176 + var id = $li.data('id');
  177 +
  178 + $.ajax({
  179 + url: '/degrade/setter',
  180 + data: {
  181 + checked: checked,
  182 + id: id
  183 + }
  184 + });
  185 + });
  186 + })
  187 +</script>
@@ -41,6 +41,7 @@ @@ -41,6 +41,7 @@
41 </ul> 41 </ul>
42 </li> 42 </li>
43 {{/if}} 43 {{/if}}
  44 + <li><a href="/degrade"><i class="fa fa-hand-o-down"></i> <span>降级配置</span></a></li>
44 </ul> 45 </ul>
45 46
46 </div> 47 </div>
  1 +'use strict';
  2 +
  3 +import zookeeper from 'node-zookeeper-client';
  4 +
  5 +module.exports = (server, path) => {
  6 + const client = zookeeper.createClient(server);
  7 +
  8 + client.once('connected', () => {
  9 + client.mkdirp(path, new Buffer('false'), (err, path) => {
  10 + if (err) {
  11 + console.log('Node %s create err', path, err.stack);
  12 + } else {
  13 + console.log('Node %s is created', path);
  14 + }
  15 +
  16 + client.close();
  17 + });
  18 + });
  19 +
  20 + client.connect();
  21 +};
  1 +'usu strict';
  2 +
  3 +import _ from 'lodash';
  4 +import zookeeper from 'node-zookeeper-client';
  5 +
  6 +import creator from './creator';
  7 +
  8 +const getter = (server, client, path, resolve, reject) => {
  9 + client.exists(path, (err, stat) => {
  10 + if (err) {
  11 + console.log('path %s exits error', path, err.stack);
  12 + resolve(false);
  13 + return;
  14 + }
  15 +
  16 + if (stat) {
  17 + client.getData(
  18 + path,
  19 + (err, data, stat) => {
  20 + if (err) {
  21 + console.log('Got path %s data error', path, err.stack);
  22 + }
  23 + client.close();
  24 +
  25 + resolve((data && data.toString('utf8') === 'true') ? true : false);
  26 + }
  27 + )
  28 + } else {
  29 + // 不存在的路径
  30 + console.log('no path %s, we will create it with value "false" automatic', path);
  31 + client.close();
  32 +
  33 + // create path
  34 + creator(server, path);
  35 +
  36 + resolve(false);
  37 + }
  38 + });
  39 +
  40 +};
  41 +
  42 +module.exports = (server, path) => new Promise((resolve, reject) => {
  43 + const client = zookeeper.createClient(server);
  44 +
  45 + client.once('connected', () => {
  46 + getter(server, client, path, resolve, reject);
  47 + });
  48 +
  49 + client.connect();
  50 +});
  1 +'usu strict';
  2 +
  3 +import _ from 'lodash';
  4 +import zookeeper from 'node-zookeeper-client';
  5 +
  6 +module.exports = (server, path, val) => new Promise((resolve, reject) => {
  7 + const client = zookeeper.createClient(server);
  8 +
  9 + client.once('connected', function () {
  10 + client.setData(path, new Buffer(val.toString()), function(err, data, stat) {
  11 + console.log('path %s data change to', path, val);
  12 + resolve();
  13 + client.close();
  14 + });
  15 + });
  16 +
  17 + client.connect();
  18 +});
@@ -55,6 +55,7 @@ @@ -55,6 +55,7 @@
55 "moment": "^2.13.0", 55 "moment": "^2.13.0",
56 "nedb": "^1.8.0", 56 "nedb": "^1.8.0",
57 "nedb-promise": "^2.0.0", 57 "nedb-promise": "^2.0.0",
  58 + "node-zookeeper-client": "^0.2.2",
58 "qn": "^1.3.0", 59 "qn": "^1.3.0",
59 "qs": "^6.2.0", 60 "qs": "^6.2.0",
60 "request-promise": "^4.1.1", 61 "request-promise": "^4.1.1",