OPENTECH / yoho-node-ci · Commits

Go to a project

Authored by 陈峰
Commit 1298b05aad7963b53dbb2f70f682cde429104603 1 parent 31fd3da3

商品缓存清理增加根据skn导出商品url功能

Inline Side-by-side
Showing 12 changed files with 339 additions and 27 deletions
No preview for this file type
No preview for this file type
@@ -13,6 +13,7 @@ import OperationLoggerModel from './operation_logger'; @@ -13,6 +13,7 @@ import OperationLoggerModel from './operation_logger';
13 import PageCacheModel from './page_cache'; 13 import PageCacheModel from './page_cache';
14 import CdnCacheModel from './cdn_cache'; 14 import CdnCacheModel from './cdn_cache';
15 import ProductCacheModel from './product_cache'; 15 import ProductCacheModel from './product_cache';
  16 +import TempProductImportModel from './temp_product_import';
16 import MemcachedHostModel from './memcached_host'; 17 import MemcachedHostModel from './memcached_host';
17 import DegradeModel from './degrade'; 18 import DegradeModel from './degrade';
18 import DegradeServerModel from './degrade_server'; 19 import DegradeServerModel from './degrade_server';
@@ -31,6 +32,7 @@ const OperationLogger = new OperationLoggerModel(); @@ -31,6 +32,7 @@ const OperationLogger = new OperationLoggerModel();
31 const PageCache = new PageCacheModel(); 32 const PageCache = new PageCacheModel();
32 const CdnCache = new CdnCacheModel(); 33 const CdnCache = new CdnCacheModel();
33 const ProductCache = new ProductCacheModel(); 34 const ProductCache = new ProductCacheModel();
  35 +const TempProductImport = new TempProductImportModel();
34 const MemcachedHost = new MemcachedHostModel(); 36 const MemcachedHost = new MemcachedHostModel();
35 const Degrade = new DegradeModel(); 37 const Degrade = new DegradeModel();
36 const DegradeServer = new DegradeServerModel(); 38 const DegradeServer = new DegradeServerModel();
@@ -53,6 +55,7 @@ export { @@ -53,6 +55,7 @@ export {
53 PageCache, 55 PageCache,
54 CdnCache, 56 CdnCache,
55 ProductCache, 57 ProductCache,
  58 + TempProductImport,
56 MemcachedHost, 59 MemcachedHost,
57 RestartInfo, 60 RestartInfo,
58 DeleteRestartInfo, 61 DeleteRestartInfo,
@@ -7,6 +7,8 @@ @@ -7,6 +7,8 @@
7 import Model from './model'; 7 import Model from './model';
8 import rp from 'request-promise'; 8 import rp from 'request-promise';
9 import ws from '../../lib/ws'; 9 import ws from '../../lib/ws';
  10 +import sign from '../../lib/sign'
  11 +import _ from 'lodash';
10 12
11 class ProductCache extends Model{ 13 class ProductCache extends Model{
12 constructor() { 14 constructor() {
@@ -34,6 +36,78 @@ class ProductCache extends Model{ @@ -34,6 +36,78 @@ class ProductCache extends Model{
34 self._broadcast(`开始提交：其它批量`); 36 self._broadcast(`开始提交：其它批量`);
35 await self._batchPost(list, url); 37 await self._batchPost(list, url);
36 } 38 }
  39 + async generateProductUrl(list) {
  40 + let self = this;
  41 + let interval = 50;
  42 + let tick = parseInt(list.length / interval, 10) + (list.length % interval > 0 ? 1 : 0);
  43 + let result = [];
  44 + self._broadcast(`获取商品信息中：`);
  45 + for (var tickIndex = 0; tickIndex < tick; tickIndex++) {
  46 + let datas = list.slice(tickIndex * interval, (tickIndex + 1) * interval > list.length ? list.length : (tickIndex + 1) * interval);
  47 + try {
  48 + let sknList = [];
  49 + _.forEach(datas, data => {
  50 + sknList.push(data.skn)
  51 + })
  52 + let skns = sknList.join(',');
  53 + self._broadcast(`进度：${tickIndex+1}/${tick}`)
  54 + let res = await self._batchProductInfo(skns);
  55 + _.forEach(res.data.product_list, (product) => {
  56 + result.push(self._joinProductUrl(product));
  57 + });
  58 + } catch(exp) {
  59 + }
  60 + }
  61 + self._broadcast(`获取完成，正在生成文件`);
  62 + return result;
  63 + }
  64 + async _batchProductInfo(skns) {
  65 + let params = {
  66 + productSkn: skns,
  67 + method: 'h5.product.batch'
  68 + }
  69 + params = sign.apiSign(params, 'h5', '5.1.0', '');
  70 + return new Promise((resolve, reject) => {
  71 + rp({
  72 + url: 'http://api.yoho.cn/',
  73 + qs: params,
  74 + json: true,
  75 + timeout: 3000,
  76 + gzip: true
  77 + })
  78 + .then(function (res) {
  79 + res.code === 200 ? resolve(res) : reject(res);
  80 + })
  81 + .catch(function (err) {
  82 + if (err.response) {
  83 + console.log(err.response.body);
  84 + reject(err.response.body)
  85 + }
  86 + });
  87 + })
  88 + }
  89 + _joinProductUrl(product) {
  90 + if (product && product.goods_list && product.goods_list.length) {
  91 + return `/product/pro_${product.product_id}_${product.goods_list[0].goods_id}/${product.cn_alphabet}.html`;
  92 + }
  93 + return '';
  94 + }
  95 + rendExcel(sheets) {
  96 + if (sheets.length) {
  97 + let rows = sheets[0].data;
  98 + if (rows.length > 1) {
  99 + let data = [];
  100 + for (var i = 1; i < rows.length; i++) {
  101 + var row = rows[i];
  102 + if (row.length) {
  103 + data.push({skn: row[0]});
  104 + }
  105 + }
  106 + return data;
  107 + }
  108 + }
  109 + return [];
  110 + }
37 async _batchPost(list, url) { 111 async _batchPost(list, url) {
38 let self = this; 112 let self = this;
39 let interval = 50; 113 let interval = 50;
@@ -44,7 +118,11 @@ class ProductCache extends Model{ @@ -44,7 +118,11 @@ class ProductCache extends Model{
44 let limit = list.length > (i + 1) * interval ? (i + 1) * interval : list.length; 118 let limit = list.length > (i + 1) * interval ? (i + 1) * interval : list.length;
45 let datas = list.slice(i * interval, limit); 119 let datas = list.slice(i * interval, limit);
46 try { 120 try {
47 - await self._postApi(datas, url); 121 + let skns = [];
  122 + _.forEach(datas, data => {
  123 + skns.push(data.skn)
  124 + })
  125 + await self._postApi(skns, url);
48 self._broadcast(`进度：${limit}/${list.length}`) 126 self._broadcast(`进度：${limit}/${list.length}`)
49 } catch(err) { 127 } catch(err) {
50 self._broadcast(`错误：${err.message}`) 128 self._broadcast(`错误：${err.message}`)
@@ -72,8 +150,10 @@ class ProductCache extends Model{ @@ -72,8 +150,10 @@ class ProductCache extends Model{
72 res.code === 200 ? resolve(res) : reject(res); 150 res.code === 200 ? resolve(res) : reject(res);
73 }) 151 })
74 .catch(function (err) { 152 .catch(function (err) {
  153 + if (err.response) {
75 console.log(err.response.body); 154 console.log(err.response.body);
76 reject(err.response.body) 155 reject(err.response.body)
  156 + }
77 }); 157 });
78 }) 158 })
79 159
  1 +/**
  2 + *
  3 + * @author: chenfeng<feng.chen@yoho.cn>
  4 + * @date: 16/11/01
  5 + */
  6 +'use strict';
  7 +
  8 +import Model from './model';
  9 +
  10 +class TempProductImport extends Model {
  11 + constructor() {
  12 + super('temp_product_import');
  13 + }
  14 +}
  15 +export default TempProductImport;
No preview for this file type
@@ -9,47 +9,94 @@ @@ -9,47 +9,94 @@
9 import Router from 'koa-router'; 9 import Router from 'koa-router';
10 import xlsx from 'node-xlsx'; 10 import xlsx from 'node-xlsx';
11 import fs from 'fs'; 11 import fs from 'fs';
  12 +import _ from 'lodash';
12 import { 13 import {
13 - ProductCache 14 + ProductCache,
  15 + TempProductImport
14 } from '../../models' 16 } from '../../models'
15 17
16 const r = new Router(); 18 const r = new Router();
17 19
18 const productCache = { 20 const productCache = {
19 async query(ctx) { 21 async query(ctx) {
  22 + if (await TempProductImport.count({}) > 0) {
  23 + await TempProductImport.remove({});
  24 + }
20 await ctx.render('action/product_cache'); 25 await ctx.render('action/product_cache');
21 }, 26 },
22 - async clear(ctx) { 27 + async import(ctx) {
  28 + if (await TempProductImport.count({}) > 0) {
  29 + await TempProductImport.remove({});
  30 + }
23 let excelFile = ctx.request.body._files && ctx.request.body._files.excelFile; 31 let excelFile = ctx.request.body._files && ctx.request.body._files.excelFile;
24 - let type = ctx.request.body.type;  
25 - if (excelFile && type) { 32 + if (excelFile) {
26 let sheets = xlsx.parse(fs.readFileSync(excelFile.path)); 33 let sheets = xlsx.parse(fs.readFileSync(excelFile.path));
27 - if (sheets.length) {  
28 - let rows = sheets[0].data;  
29 - if (rows.length > 1) {  
30 - let data = [];  
31 - for (var i = 1; i < rows.length; i++) {  
32 - var row = rows[i];  
33 - if (row.length) {  
34 - data.push(row[0]); 34 + let data = ProductCache.rendExcel(sheets);
  35 + await TempProductImport.insert(data);
  36 + let count = await TempProductImport.count({});
  37 + ctx.body = {
  38 + code: 200,
  39 + count
35 } 40 }
  41 + return;
36 } 42 }
  43 + ctx.body = {
  44 + code: 400
  45 + }
  46 + },
  47 + async clear(ctx) {
  48 + let type = ctx.request.body.type;
  49 + if (await TempProductImport.count({}) > 0) {
  50 + let data = await TempProductImport.findAll();
  51 + if (type) {
37 if (type === '1') { 52 if (type === '1') {
38 ProductCache.removePriceCache(data) 53 ProductCache.removePriceCache(data)
39 } else if (type === '2') { 54 } else if (type === '2') {
40 ProductCache.removeProductCache(data) 55 ProductCache.removeProductCache(data)
41 } 56 }
42 - 57 + ctx.body = {
  58 + code: 200
43 } 59 }
  60 + return;
44 } 61 }
  62 + } else {
  63 + ProductCache._broadcast('请导入商品');
45 } 64 }
46 ctx.body = { 65 ctx.body = {
47 - code: 200 66 + code: 400
  67 + }
  68 + },
  69 + async downloadProductUrl(ctx) {
  70 + if (await TempProductImport.count({}) > 0) {
  71 + let data = await TempProductImport.findAll();
  72 + let productUrls = await ProductCache.generateProductUrl(data);
  73 + let domains = {'PC': 'http://www.yohobuy.com', 'WAP': 'https://m.yohobuy.com'};
  74 + let resultData = [];
  75 + _.forEach(domains, (value, key) => {
  76 + let rows = [['URL']];
  77 + _.forEach(productUrls, url => {
  78 + rows.push([value + url]);
  79 + });
  80 + resultData.push({name: key, data: rows});
  81 + });
  82 + var buffer = xlsx.build(resultData);
  83 + ctx.status = 200;
  84 + ctx.set('Content-disposition', 'attachment; filename=output.xlsx');
  85 + ctx.set('Content-type', 'application/vnd.ms-excel');
  86 + ctx.body = buffer;
  87 + return;
  88 + } else {
  89 + ProductCache._broadcast('请导入商品');
  90 + }
  91 + ctx.body = {
  92 + code: 400
48 } 93 }
49 } 94 }
50 } 95 }
51 96
52 r.get('/query', productCache.query); 97 r.get('/query', productCache.query);
  98 +r.post('/import', productCache.import);
53 r.post('/clear', productCache.clear); 99 r.post('/clear', productCache.clear);
  100 +r.get('/product_url', productCache.downloadProductUrl);
54 101
55 export default r; 102 export default r;
@@ -22,6 +22,7 @@ @@ -22,6 +22,7 @@
22 <div class="operations mb20"> 22 <div class="operations mb20">
23 <label class="control-label">导入文件：</label> 23 <label class="control-label">导入文件：</label>
24 <div class="file-upload"></div> 24 <div class="file-upload"></div>
  25 + <p class="skn-count"></p>
25 <select id="selectType" class="form-control input-sm selcet-auto pull-left mr20"> 26 <select id="selectType" class="form-control input-sm selcet-auto pull-left mr20">
26 <option value="1">批量变价</option> 27 <option value="1">批量变价</option>
27 <option value="2">其它批量</option> 28 <option value="2">其它批量</option>
@@ -31,6 +32,8 @@ @@ -31,6 +32,8 @@
31 </div> 32 </div>
32 <div class="download"> 33 <div class="download">
33 <a href="/template/product_cache_template.xlsx" class="block">模板文件下载</a> 34 <a href="/template/product_cache_template.xlsx" class="block">模板文件下载</a>
  35 + &nbsp;
  36 + <a href="javascript:;" class="block product-url">生成商品url</a>
34 </div> 37 </div>
35 </div> 38 </div>
36 <div class="col-sm-6"> 39 <div class="col-sm-6">
@@ -59,9 +62,8 @@ @@ -59,9 +62,8 @@
59 $(document).on('ready pjax:success', function() { 62 $(document).on('ready pjax:success', function() {
60 63
61 var fileUpload = $('.file-upload').uploadFile({ 64 var fileUpload = $('.file-upload').uploadFile({
62 - url: '/product_cache/clear', 65 + url: '/product_cache/import',
63 method: 'POST', 66 method: 'POST',
64 - autoSubmit: false,  
65 uploadStr: '浏览', 67 uploadStr: '浏览',
66 multiple: false, 68 multiple: false,
67 maxFileCount: 1, 69 maxFileCount: 1,
@@ -72,6 +74,9 @@ @@ -72,6 +74,9 @@
72 }, 74 },
73 onSuccess: function(files, data, xhr, pd) { 75 onSuccess: function(files, data, xhr, pd) {
74 console.log('success') 76 console.log('success')
  77 + if(data.code === 200) {
  78 + $('.skn-count').text('导入数量：' + data.count).data('count', data.count);
  79 + }
75 fileUpload.reset(); 80 fileUpload.reset();
76 }, 81 },
77 onError: function(files, data, xhr, pd) { 82 onError: function(files, data, xhr, pd) {
@@ -80,16 +85,19 @@ @@ -80,16 +85,19 @@
80 } 85 }
81 }) 86 })
82 $('.btn-clear').click(function() { 87 $('.btn-clear').click(function() {
83 - // var selectType = $('#selectType').val();  
84 - // if (selectType) {  
85 - // $logs.empty();  
86 - // // $.post('/product_cache/clear', {  
87 - // // type: selectType  
88 - // // }, function(res) {  
89 - // // });  
90 -  
91 - // }  
92 - fileUpload.startUpload(); 88 + var count = $('.skn-count').data('count');
  89 + var selectType = $('#selectType').val();
  90 + if(count > 0 && selectType) {
  91 + $.post('/product_cache/clear', {
  92 + type: selectType,
  93 + });
  94 + }
  95 + })
  96 + $('.product-url').click(function() {
  97 + var count = $('.skn-count').data('count');
  98 + if(count > 0) {
  99 + window.location.href = '/product_cache/product_url';
  100 + }
93 }) 101 })
94 102
95 function layoutResize() { 103 function layoutResize() {
No preview for this file type
  1 +/**
  2 + * 签名
  3 + * @author: bikai
  4 + * @date: 2016/5/6
  5 + */
  6 +
  7 +'use strict';
  8 +const _ = require('lodash');
  9 +const md5 = require('md5');
  10 +
  11 +const privateKey = {
  12 + android: 'fd4ad5fcfa0de589ef238c0e7331b585',
  13 + iphone: 'a85bb0674e08986c6b115d5e3a4884fa',
  14 + ipad: 'ad9fcda2e679cf9229e37feae2cdcf80',
  15 + web: '0ed29744ed318fd28d2c07985d3ba633',
  16 + yoho: 'fd4ad5fcsa0de589af23234ks1923ks',
  17 + h5: 'fd4ad5fcfa0de589ef238c0e7331b585'
  18 +};
  19 +
  20 +/**
  21 + * 排序参数
  22 + * @param {Object} argument 需要排序的参数对象
  23 + * @return {Object} 排序之后的参数对象
  24 + */
  25 +const packageSort = argument => {
  26 + const newObj = {};
  27 +
  28 + for (const k of Object.keys(argument).sort()) {
  29 + newObj[k] = argument[k];
  30 + }
  31 +
  32 + return newObj;
  33 +};
  34 +
  35 +/**
  36 + * 生成签名
  37 + * @param {Object} argument 需要签名的数据
  38 + * @return {string} 生成的签名字符串
  39 + */
  40 +const makeSign = argument => {
  41 + const qs = [];
  42 +
  43 + _.forEach(argument, (value, key) => {
  44 + value = _.trim(value);
  45 + qs.push(`${key}=${value}`);
  46 + argument[key] = value;
  47 + });
  48 +
  49 + return md5(qs.join('&')).toLowerCase();
  50 +};
  51 +
  52 +// 生成API签名，调用后端接口的时候有私钥校验
  53 +exports.apiSign = (params, app, appVersion, signExtend) => {
  54 + const clientType = params.client_type || app;
  55 +
  56 + appVersion = appVersion || '4.6.0';
  57 + signExtend = signExtend || {};
  58 +
  59 + /* eslint-disable */
  60 + let sign = packageSort(Object.assign({
  61 + client_type: clientType,
  62 + private_key: privateKey[clientType],
  63 + app_version: appVersion,
  64 + os_version: `yohobuy:${app}`,
  65 + screen_size: '720x1280',
  66 + v: '7'
  67 + }, signExtend, params));
  68 + /* eslint-enable */
  69 +
  70 + sign = Object.assign(sign, {
  71 + client_secret: makeSign(sign) // eslint-disable-line camelcase
  72 + });
  73 + delete sign.private_key;
  74 + return sign;
  75 +};
  76 +
  77 +// 检查签名，APP 访问 H5 页面的时候需要检查
  78 +exports.checkSign = (params) => {
  79 + const clientSecret = params.client_secret; // eslint-disable-line camelcase
  80 +
  81 + let sortedParams;
  82 +
  83 + // 忽略部分参数
  84 + delete params.client_secret;
  85 + delete params.q;
  86 + delete params.debug_data;
  87 + delete params['/api'];
  88 +
  89 + params.private_key = privateKey[params.client_type]; // eslint-disable-line camelcase
  90 + sortedParams = packageSort(params);
  91 +
  92 + return clientSecret === makeSign(sortedParams);
  93 +};
  94 +
  95 +// 检查签名，APP 访问 H5 页面的时候需要检查, 有可能不同于上边的签名方式
  96 +exports.webSign = (params) => {
  97 + const webPrivateKey = 'yohobuyapp';
  98 +
  99 + return params.key === md5(md5(webPrivateKey) + params.uid);
  100 +};
  101 +
  102 +// 生成 token
  103 +exports.makeToken = (string) => {
  104 + return md5(md5(string + '#@!@#'));
  105 +};
  106 +
  107 +// 校验 token
  108 +exports.verifyToken = (string, token) => {
  109 + return exports.makeToken(string) === token;
  110 +};
  1 +0 info it worked if it ends with ok
  2 +1 verbose cli [ '/usr/local/bin/node',
  3 +1 verbose cli '/Users/chenfeng/Documents/source/yoho/yoho-node-ci/node_modules/.bin/npm',
  4 +1 verbose cli 'run',
  5 +1 verbose cli 'babel',
  6 +1 verbose cli 'app.js' ]
  7 +2 info using npm@3.10.9
  8 +3 info using node@v6.2.0
  9 +4 verbose run-script [ 'prebabel', 'babel', 'postbabel' ]
  10 +5 info lifecycle yoho-node-ci@0.0.1~prebabel: yoho-node-ci@0.0.1
  11 +6 silly lifecycle yoho-node-ci@0.0.1~prebabel: no script for prebabel, continuing
  12 +7 info lifecycle yoho-node-ci@0.0.1~babel: yoho-node-ci@0.0.1
  13 +8 verbose lifecycle yoho-node-ci@0.0.1~babel: unsafe-perm in lifecycle true
  14 +9 verbose lifecycle yoho-node-ci@0.0.1~babel: PATH: /Users/chenfeng/Documents/source/yoho/yoho-node-ci/node_modules/npm/bin/node-gyp-bin:/Users/chenfeng/Documents/source/yoho/yoho-node-ci/node_modules/.bin:/usr/local/lib/node_modules/npm/bin/node-gyp-bin:/Users/chenfeng/Documents/source/yoho/yoho-node-ci/node_modules/.bin:/usr/local/bin:/usr/local/bin:/usr/bin:/bin:/usr/sbin:/sbin:/usr/local/share/dotnet:/Users/chenfeng/Documents/tools/android-sdk-macosx/platform-tools:/Users/chenfeng/Documents/tools/android-sdk-macosx/tools:/usr/local/nginx/sbin
  15 +10 verbose lifecycle yoho-node-ci@0.0.1~babel: CWD: /Users/chenfeng/Documents/source/yoho/yoho-node-ci
  16 +11 silly lifecycle yoho-node-ci@0.0.1~babel: Args: [ '-c', 'babel-node "app.js"' ]
  17 +12 silly lifecycle yoho-node-ci@0.0.1~babel: Returned: code: 1 signal: null
  18 +13 info lifecycle yoho-node-ci@0.0.1~babel: Failed to exec babel script
  19 +14 verbose stack Error: yoho-node-ci@0.0.1 babel: `babel-node "app.js"`
  20 +14 verbose stack Exit status 1
  21 +14 verbose stack at EventEmitter.<anonymous> (/Users/chenfeng/Documents/source/yoho/yoho-node-ci/node_modules/npm/lib/utils/lifecycle.js:255:16)
  22 +14 verbose stack at emitTwo (events.js:106:13)
  23 +14 verbose stack at EventEmitter.emit (events.js:191:7)
  24 +14 verbose stack at ChildProcess.<anonymous> (/Users/chenfeng/Documents/source/yoho/yoho-node-ci/node_modules/npm/lib/utils/spawn.js:40:14)
  25 +14 verbose stack at emitTwo (events.js:106:13)
  26 +14 verbose stack at ChildProcess.emit (events.js:191:7)
  27 +14 verbose stack at maybeClose (internal/child_process.js:850:16)
  28 +14 verbose stack at Process.ChildProcess._handle.onexit (internal/child_process.js:215:5)
  29 +15 verbose pkgid yoho-node-ci@0.0.1
  30 +16 verbose cwd /Users/chenfeng/Documents/source/yoho/yoho-node-ci
  31 +17 error Darwin 16.1.0
  32 +18 error argv "/usr/local/bin/node" "/Users/chenfeng/Documents/source/yoho/yoho-node-ci/node_modules/.bin/npm" "run" "babel" "app.js"
  33 +19 error node v6.2.0
  34 +20 error npm v3.10.9
  35 +21 error code ELIFECYCLE
  36 +22 error yoho-node-ci@0.0.1 babel: `babel-node "app.js"`
  37 +22 error Exit status 1
  38 +23 error Failed at the yoho-node-ci@0.0.1 babel script 'babel-node "app.js"'.
  39 +23 error Make sure you have the latest version of node.js and npm installed.
  40 +23 error If you do, this is most likely a problem with the yoho-node-ci package,
  41 +23 error not with npm itself.
  42 +23 error Tell the author that this fails on your system:
  43 +23 error babel-node "app.js"
  44 +23 error You can get information on how to open an issue for this project with:
  45 +23 error npm bugs yoho-node-ci
  46 +23 error Or if that isn't available, you can get their info via:
  47 +23 error npm owner ls yoho-node-ci
  48 +23 error There is likely additional logging output above.
  49 +24 verbose exit [ 1, true ]
No preview for this file type