马力 / yoho-ansible-roles · Commits

Go to a project

Authored by chunhua.zhang
Commit b27c528cf008e2f0f7471930160b16eaf5d054fe 1 parent d7008cbd

add openresty

Inline Side-by-side
Showing 27 changed files with 2669 additions and 1 deletions
- hosts: server1
roles:
- resolv
\ No newline at end of file
- openresty
\ No newline at end of file
... ...
Role Name
=========
A brief description of the role goes here.
Requirements
------------
Any pre-requisites that may not be covered by Ansible itself or the role should be mentioned here. For instance, if the role uses the EC2 module, it may be a good idea to mention in this section that the boto package is required.
Role Variables
--------------
A description of the settable variables for this role should go here, including any variables that are in defaults/main.yml, vars/main.yml, and any variables that can/should be set via parameters to the role. Any variables that are read from other roles and/or the global scope (ie. hostvars, group vars, etc.) should be mentioned here as well.
Dependencies
------------
A list of other roles hosted on Galaxy should go here, plus any details in regards to parameters that may need to be set for other roles, or variables that are used from other roles.
Example Playbook
----------------
Including an example of how to use your role (for instance, with variables passed in as parameters) is always nice for users too:
- hosts: servers
roles:
- { role: username.rolename, x: 42 }
License
-------
BSD
Author Information
------------------
An optional section for the role authors to include contact information, or a website (HTML is not allowed).
... ...
---
# defaults file for openresty
dc : "yoho-qcloud"
path: "/Data/local/openresty-1.9.15.1"
upstream_brower:
- 10.66.202.7:8092
- 10.66.202.32:8092
upstream_gateway:
- 10.66.202.13:8080
upstream_activity:
- 10.66.202.14:8090
upstream_wechat:
- 10.66.202.7:8094
upstream_erpgateway:
- 10.66.202.7:8089
upstream_union:
- 10.66.70.21:8088
- 10.66.70.49:8088
- 10.66.103.20:8088
... ...
module("resty.http", package.seeall)
_VERSION = '0.2'
-- constants
-- connection timeout in seconds
local TIMEOUT = 60
-- default ports for document retrieval
local PORT = 80
local SSL_PORT = 443
-- user agent field sent in request
local USERAGENT = 'resty.http/' .. _VERSION
-- default url parts
local default = {
host = "",
path ="/",
scheme = "http"
}
-- global variables
local url = require("resty.url")
local mt = { __index = resty.http }
local tcp = ngx.socket.tcp
local base64 = ngx.encode_base64
local function adjusturi(reqt)
local u = reqt
-- if there is a proxy, we need the full url. otherwise, just a part.
if not reqt.proxy and not PROXY then
u = {
path = reqt.path,
params = reqt.params,
query = reqt.query,
fragment = reqt.fragment
}
end
return url.build(u)
end
local function adjustheaders(reqt)
-- default headers
local lower = {
["user-agent"] = USERAGENT,
["host"] = reqt.host,
["connection"] = "close, TE",
["te"] = "trailers"
}
-- if we have authentication information, pass it along
if reqt.user and reqt.password then
lower["authorization"] =
"Basic " .. (base64(reqt.user .. ":" .. reqt.password))
end
-- override with user headers
for i,v in pairs(reqt.headers or lower) do
lower[string.lower(i)] = v
end
return lower
end
local function adjustproxy(reqt)
local proxy = reqt.proxy or PROXY
if proxy then
proxy = url.parse(proxy)
return proxy.host, proxy.port or 3128
else
return reqt.host, reqt.port
end
end
local function adjustrequest(reqt)
-- parse url if provided
local nreqt = reqt.url and url.parse(reqt.url, default) or {}
-- explicit components override url
for i,v in pairs(reqt) do nreqt[i] = v end
if nreqt.port == nil or nreqt.port == "" then
if nreqt.scheme == "https" then
nreqt.port = SSL_PORT
else
nreqt.port = PORT
end
end
-- compute uri if user hasn't overriden
nreqt.uri = reqt.uri or adjusturi(nreqt)
-- ajust host and port if there is a proxy
nreqt.host, nreqt.port = adjustproxy(nreqt)
-- adjust headers in request
nreqt.headers = adjustheaders(nreqt)
nreqt.timeout = reqt.timeout or TIMEOUT * 1000;
nreqt.fetch_size = reqt.fetch_size or 16*1024 -- 16k
nreqt.max_body_size = reqt.max_body_size or 1024*1024*1024 -- 1024mb
if reqt.keepalive then
nreqt.headers['connection'] = 'keep-alive'
end
return nreqt
end
local function receivestatusline(sock)
local status_reader = sock:receiveuntil("\r\n")
local data, err, partial = status_reader()
if not data then
return nil, "read status line failed " .. err
end
local t1, t2, code = string.find(data, "HTTP/%d*%.%d* (%d%d%d)")
return tonumber(code), data
end
local function receiveheaders(sock, headers)
local line, name, value, err, tmp1, tmp2
headers = headers or {}
-- get first line
line, err = sock:receive()
if err then return nil, err end
-- headers go until a blank line is found
while line ~= "" do
-- get field-name and value
tmp1, tmp2, name, value = string.find(line, "^(.-):%s*(.*)")
if not (name and value) then return nil, "malformed reponse headers" end
name = string.lower(name)
-- get next line (value might be folded)
line, err = sock:receive()
if err then return nil, err end
-- unfold any folded values
while string.find(line, "^%s") do
value = value .. line
line = sock:receive()
if err then return nil, err end
end
-- save pair in table
if headers[name] then
if name == "set-cookie" then
headers[name] = headers[name] .. "," .. value
else
headers[name] = headers[name] .. ", " .. value
end
else headers[name] = value end
end
return headers
end
local function read_body_data(sock, size, fetch_size, callback)
local p_size = fetch_size
while size and size > 0 do
if size < p_size then
p_size = size
end
local data, err, partial = sock:receive(p_size)
if not err then
if data then
callback(data)
end
elseif err == "closed" then
if partial then
callback(partial)
end
return 1 -- 'closed'
else
return nil, err
end
size = size - p_size
end
return 1
end
local function receivebody(sock, headers, nreqt)
local t = headers["transfer-encoding"] -- shortcut
local body = {} -- data chunks of response body
local callback = nreqt.body_callback
if not callback then
local function bc(data, chunked_header, ...)
if chunked_header then return end
body[#body+1] = data
end
callback = bc
end
if t and t ~= "identity" then
-- chunked
while true do
local chunk_header = sock:receiveuntil("\r\n")
local data, err, partial = chunk_header()
if not data then
return nil,err
else
if data == "0" then
return table.concat(body) -- end of chunk
else
local length = tonumber(data, 16)
-- TODO check nreqt.max_body_size !!
local ok, err = read_body_data(sock,length, nreqt.fetch_size, callback)
if err then
return nil,err
end
end
end
end
elseif headers["content-length"] ~= nil and tonumber(headers["content-length"]) >= 0 then
-- content length
local length = tonumber(headers["content-length"])
if length > nreqt.max_body_size then
ngx.log(ngx.INFO, 'content-length > nreqt.max_body_size !! Tail it !')
length = nreqt.max_body_size
end
local ok, err = read_body_data(sock,length, nreqt.fetch_size, callback)
if not ok then
return nil,err
end
else
-- connection close
local ok, err = read_body_data(sock,nreqt.max_body_size, nreqt.fetch_size, callback)
if not ok then
return nil,err
end
end
return table.concat(body)
end
local function shouldredirect(reqt, code, headers)
return headers.location and
string.gsub(headers.location, "%s", "") ~= "" and
(reqt.redirect ~= false) and
(code == 301 or code == 302) and
(not reqt.method or reqt.method == "GET" or reqt.method == "HEAD")
and (not reqt.nredirects or reqt.nredirects < 5)
end
local function shouldreceivebody(reqt, code)
if reqt.method == "HEAD" then return nil end
if code == 204 or code == 304 then return nil end
if code >= 100 and code < 200 then return nil end
return 1
end
function new(self)
return setmetatable({}, mt)
end
function request(self, reqt)
local code, headers, status, body, bytes, ok, err
local nreqt = adjustrequest(reqt)
local sock = tcp()
if not sock then
return nil, "create sock failed"
end
sock:settimeout(nreqt.timeout)
-- connect
ok, err = sock:connect(nreqt.host, nreqt.port)
if err then
return nil, "sock connected failed " .. err
end
-- check type of req_body, maybe string, file, function
local req_body = nreqt.body
local req_body_type = nil
if req_body then
req_body_type = type(req_body)
if req_body_type == 'string' then -- fixed Content-Length
nreqt.headers['content-length'] = #req_body
end
end
-- send request line and headers
local reqline = string.format("%s %s HTTP/1.1\r\n", nreqt.method or "GET", nreqt.uri)
local h = ""
for i, v in pairs(nreqt.headers) do
-- fix cookie is a table value
if type(v) == "table" then
if i == "cookie" then
v = table.concat(v, "; ")
else
v = table.concat(v, ", ")
end
end
h = i .. ": " .. v .. "\r\n" .. h
end
h = h .. '\r\n' -- close headers
-- @modify: add ssl support
if nreqt.scheme == 'https' then
local sess, err = sock:sslhandshake();
if err then
return nil, err;
end
end
bytes, err = sock:send(reqline .. h)
if err then
sock:close()
return nil, err
end
-- send req_body, if exists
if req_body_type == 'string' then
bytes, err = sock:send(req_body)
if err then
sock:close()
return nil, err
end
elseif req_body_type == 'file' then
local buf = nil
while true do -- TODO chunked maybe better
buf = req_body:read(8192)
if not buf then break end
bytes, err = sock:send(buf)
if err then
sock:close()
return nil, err
end
end
elseif req_body_type == 'function' then
err = req_body(sock) -- as callback(sock)
if err then
return err
end
end
-- receive status line
code, status = receivestatusline(sock)
if not code then
sock:close()
if not status then
return nil, "read status line failed "
else
return nil, "read status line failed " .. status
end
end
-- ignore any 100-continue messages
while code == 100 do
headers, err = receiveheaders(sock, {})
code, status = receivestatusline(sock)
end
-- notify code_callback
if nreqt.code_callback then
nreqt.code_callback(code)
end
-- receive headers
headers, err = receiveheaders(sock, {})
if err then
sock:close()
return nil, "read headers failed " .. err
end
-- notify header_callback
if nreqt.header_callback then
nreqt.header_callback(headers)
end
-- TODO rediret check
-- receive body
if shouldreceivebody(nreqt, code) then
body, err = receivebody(sock, headers, nreqt)
if err then
sock:close()
if code == 200 then
return 1, code, headers, status, nil
end
return nil, "read body failed " .. err
end
end
-- read CR/LF or LF otherwise thorw 'unread data in buffer' error
sock:receive("*l")
if nreqt.keepalive then
local ok, err = sock:setkeepalive(nreqt.keepalive)
if not ok then
ngx.log(ngx.WARN, "failed to set keepalive: " .. err)
end
else
sock:close()
end
return 1, code, headers, status, body
end
function proxy_pass(self, reqt)
local nreqt = {}
for i,v in pairs(reqt) do nreqt[i] = v end
if not nreqt.code_callback then
nreqt.code_callback = function(code, ...)
ngx.status = code
end
end
if not nreqt.header_callback then
nreqt.header_callback = function (headers, ...)
for i, v in pairs(headers) do
ngx.header[i] = v
end
end
end
if not nreqt.body_callback then
nreqt.body_callback = function (data, ...)
ngx.print(data) -- Will auto package as chunked format!!
end
end
return request(self, nreqt)
end
-- to prevent use of casual module global variables
getmetatable(resty.http).__newindex = function (table, key, val)
error('attempt to write to undeclared variable "' .. key .. '": '
.. debug.traceback())
end
... ...
-----------------------------------------------------------------------------
-- URI parsing, composition and relative URL resolution
-- LuaSocket toolkit.
-- Author: Diego Nehab
-- RCS ID: $Id: url.lua,v 1.38 2006/04/03 04:45:42 diego Exp $
-----------------------------------------------------------------------------
-----------------------------------------------------------------------------
-- Declare module
-----------------------------------------------------------------------------
local string = require("string")
local base = _G
local table = require("table")
module("resty.url", package.seeall)
-----------------------------------------------------------------------------
-- Module version
-----------------------------------------------------------------------------
_VERSION = "URL 1.0.1"
-----------------------------------------------------------------------------
-- Encodes a string into its escaped hexadecimal representation
-- Input
-- s: binary string to be encoded
-- Returns
-- escaped representation of string binary
-----------------------------------------------------------------------------
function escape(s)
return string.gsub(s, "([^A-Za-z0-9_])", function(c)
return string.format("%%%02x", string.byte(c))
end)
end
-----------------------------------------------------------------------------
-- Protects a path segment, to prevent it from interfering with the
-- url parsing.
-- Input
-- s: binary string to be encoded
-- Returns
-- escaped representation of string binary
-----------------------------------------------------------------------------
local function make_set(t)
local s = {}
for i,v in base.ipairs(t) do
s[t[i]] = 1
end
return s
end
-- these are allowed withing a path segment, along with alphanum
-- other characters must be escaped
local segment_set = make_set {
"-", "_", ".", "!", "~", "*", "'", "(",
")", ":", "@", "&", "=", "+", "$", ",",
}
local function protect_segment(s)
return string.gsub(s, "([^A-Za-z0-9_])", function (c)
if segment_set[c] then return c
else return string.format("%%%02x", string.byte(c)) end
end)
end
-----------------------------------------------------------------------------
-- Encodes a string into its escaped hexadecimal representation
-- Input
-- s: binary string to be encoded
-- Returns
-- escaped representation of string binary
-----------------------------------------------------------------------------
function unescape(s)
return string.gsub(s, "%%(%x%x)", function(hex)
return string.char(base.tonumber(hex, 16))
end)
end
-----------------------------------------------------------------------------
-- Builds a path from a base path and a relative path
-- Input
-- base_path
-- relative_path
-- Returns
-- corresponding absolute path
-----------------------------------------------------------------------------
local function absolute_path(base_path, relative_path)
if string.sub(relative_path, 1, 1) == "/" then return relative_path end
local path = string.gsub(base_path, "[^/]*$", "")
path = path .. relative_path
path = string.gsub(path, "([^/]*%./)", function (s)
if s ~= "./" then return s else return "" end
end)
path = string.gsub(path, "/%.$", "/")
local reduced
while reduced ~= path do
reduced = path
path = string.gsub(reduced, "([^/]*/%.%./)", function (s)
if s ~= "../../" then return "" else return s end
end)
end
path = string.gsub(reduced, "([^/]*/%.%.)$", function (s)
if s ~= "../.." then return "" else return s end
end)
return path
end
-----------------------------------------------------------------------------
-- Parses a url and returns a table with all its parts according to RFC 2396
-- The following grammar describes the names given to the URL parts
-- <url> ::= <scheme>://<authority>/<path>;<params>?<query>#<fragment>
-- <authority> ::= <userinfo>@<host>:<port>
-- <userinfo> ::= <user>[:<password>]
-- <path> :: = {<segment>/}<segment>
-- Input
-- url: uniform resource locator of request
-- default: table with default values for each field
-- Returns
-- table with the following fields, where RFC naming conventions have
-- been preserved:
-- scheme, authority, userinfo, user, password, host, port,
-- path, params, query, fragment
-- Obs:
-- the leading '/' in {/<path>} is considered part of <path>
-----------------------------------------------------------------------------
function parse(url, default)
-- initialize default parameters
local parsed = {}
for i,v in base.pairs(default or parsed) do parsed[i] = v end
-- empty url is parsed to nil
if not url or url == "" then return nil, "invalid url" end
-- remove whitespace
-- url = string.gsub(url, "%s", "")
-- get fragment
url = string.gsub(url, "#(.*)$", function(f)
parsed.fragment = f
return ""
end)
-- get scheme
url = string.gsub(url, "^([%w][%w%+%-%.]*)%:",
function(s) parsed.scheme = s; return "" end)
-- get authority
url = string.gsub(url, "^//([^/]*)", function(n)
parsed.authority = n
return ""
end)
-- get query stringing
url = string.gsub(url, "%?(.*)", function(q)
parsed.query = q
return ""
end)
-- get params
url = string.gsub(url, "%;(.*)", function(p)
parsed.params = p
return ""
end)
-- path is whatever was left
if url ~= "" then parsed.path = url end
local authority = parsed.authority
if not authority then return parsed end
authority = string.gsub(authority,"^([^@]*)@",
function(u) parsed.userinfo = u; return "" end)
authority = string.gsub(authority, ":([^:]*)$",
function(p) parsed.port = p; return "" end)
if authority ~= "" then parsed.host = authority end
local userinfo = parsed.userinfo
if not userinfo then return parsed end
userinfo = string.gsub(userinfo, ":([^:]*)$",
function(p) parsed.password = p; return "" end)
parsed.user = userinfo
return parsed
end
-----------------------------------------------------------------------------
-- Rebuilds a parsed URL from its components.
-- Components are protected if any reserved or unallowed characters are found
-- Input
-- parsed: parsed URL, as returned by parse
-- Returns
-- a stringing with the corresponding URL
-----------------------------------------------------------------------------
function build(parsed)
local ppath = parse_path(parsed.path or "")
local url = build_path(ppath)
if parsed.params then url = url .. ";" .. parsed.params end
if parsed.query then url = url .. "?" .. parsed.query end
local authority = parsed.authority
if parsed.host then
authority = parsed.host
if parsed.port then authority = authority .. ":" .. parsed.port end
local userinfo = parsed.userinfo
if parsed.user then
userinfo = parsed.user
if parsed.password then
userinfo = userinfo .. ":" .. parsed.password
end
end
if userinfo then authority = userinfo .. "@" .. authority end
end
if authority then url = "//" .. authority .. url end
if parsed.scheme then url = parsed.scheme .. ":" .. url end
if parsed.fragment then url = url .. "#" .. parsed.fragment end
-- url = string.gsub(url, "%s", "")
return url
end
-----------------------------------------------------------------------------
-- Builds a absolute URL from a base and a relative URL according to RFC 2396
-- Input
-- base_url
-- relative_url
-- Returns
-- corresponding absolute url
-----------------------------------------------------------------------------
function absolute(base_url, relative_url)
if base.type(base_url) == "table" then
base_parsed = base_url
base_url = build(base_parsed)
else
base_parsed = parse(base_url)
end
local relative_parsed = parse(relative_url)
if not base_parsed then return relative_url
elseif not relative_parsed then return base_url
elseif relative_parsed.scheme then return relative_url
else
relative_parsed.scheme = base_parsed.scheme
if not relative_parsed.authority then
relative_parsed.authority = base_parsed.authority
if not relative_parsed.path then
relative_parsed.path = base_parsed.path
if not relative_parsed.params then
relative_parsed.params = base_parsed.params
if not relative_parsed.query then
relative_parsed.query = base_parsed.query
end
end
else
relative_parsed.path = absolute_path(base_parsed.path or "",
relative_parsed.path)
end
end
return build(relative_parsed)
end
end
-----------------------------------------------------------------------------
-- Breaks a path into its segments, unescaping the segments
-- Input
-- path
-- Returns
-- segment: a table with one entry per segment
-----------------------------------------------------------------------------
function parse_path(path)
local parsed = {}
path = path or ""
--path = string.gsub(path, "%s", "")
string.gsub(path, "([^/]+)", function (s) table.insert(parsed, s) end)
for i = 1, #parsed do
parsed[i] = unescape(parsed[i])
end
if string.sub(path, 1, 1) == "/" then parsed.is_absolute = 1 end
if string.sub(path, -1, -1) == "/" then parsed.is_directory = 1 end
return parsed
end
-----------------------------------------------------------------------------
-- Builds a path component from its segments, escaping protected characters.
-- Input
-- parsed: path segments
-- unsafe: if true, segments are not protected before path is built
-- Returns
-- path: corresponding path stringing
-----------------------------------------------------------------------------
function build_path(parsed, unsafe)
local path = ""
local n = #parsed
if unsafe then
for i = 1, n-1 do
path = path .. parsed[i]
path = path .. "/"
end
if n > 0 then
path = path .. parsed[n]
if parsed.is_directory then path = path .. "/" end
end
else
for i = 1, n-1 do
path = path .. protect_segment(parsed[i])
path = path .. "/"
end
if n > 0 then
path = path .. protect_segment(parsed[n])
if parsed.is_directory then path = path .. "/" end
end
end
if parsed.is_absolute then path = "/" .. path end
return path
end
... ...
local http = require "resty.http"
local cjson = require "cjson"
local rate_limit_conf_url="http://config.server.yohoops.org/nginx_limit_api/default"
local limit_ip_access_url="http://config.server.yohoops.org/nginx_limit_ip/default"
local nginx_common_conf_url="http://config.server.yohoops.org/nginx_common_conf/default"
local limit_ip_redis=lua_context["redis_limit_ip"]
function split_str_list(str,spliter)
local ips={}
if not str then
return ips
end
string.gsub(str,'[^' .. spliter ..']+',function(w) table.insert(ips, w) end )
return ips
end
function toboolean(str,default)
if str == nil then
return default
end
if str == "true" then
return true
else
return false
end
end
local query_common_conf=function()
local httpc = http:new()
local ok, code, headers, status, body = httpc:request{
url=nginx_common_conf_url,
method="GET",
}
if ok then
local rate_limit_conf=cjson.decode(body)
local property=rate_limit_conf["propertySources"]
if property then
local property=property[1]
if property then
local source=property["source"]
if source then
local common_conf={}
common_conf.lua_golbal_switch=source["lua_golbal_switch"]
lua_context.lua_conf_cache:set("common_conf",cjson.encode(common_conf))
end
end
end
else
ngx.log(ngx.ERR, "request error:" .. tostring(code))
end
end
-->>begin: query rate limit config function definition
local query_rate_limit_conf=function()
local httpc = http:new()
local ok, code, headers, status, body = httpc:request{
url=rate_limit_conf_url,
method="GET",
}
if ok then
local rate_limit_conf=cjson.decode(body)
local property=rate_limit_conf["propertySources"]
if property then
local property=property[1]
if property then
local api_rate_limit_conf={api_rate_limit={}}
local source=property["source"]
api_rate_limit_conf["is_open"]=source["open_limit_flow"]
for k,v in pairs(source) do
if string.find(k,"^api_rate_limit*") then
local t=split_str_list(k,".")
table.remove(t,1)
local key=table.concat(t,".")
local vals=split_str_list(v,",")
api_rate_limit_conf.api_rate_limit[key]={tonumber(vals[1]),tonumber(vals[2]),vals[3]}
end
end
lua_context.lua_conf_cache:set("api_rate_limit_conf",cjson.encode(api_rate_limit_conf))
end
end
else
ngx.log(ngx.ERR, "request error:" .. tostring(code))
end
end
--<<end: query rate limit config function definition
-->>begin: query limit ip access config function definition
local query_limit_ip_access_conf=function()
local httpc = http:new()
local ok, code, headers, status, body = httpc:request{
url=limit_ip_access_url,
method="GET",
}
if ok then
--ngx.log(ngx.ERR, ">>>>>>>>>>>>>>" .. tostring(body))
local rate_limit_conf=cjson.decode(body)
local property=rate_limit_conf["propertySources"]
if property then
local property=property[1]
if property then
local source=property["source"]
if source then
local limit_ip_access={}
local is_open=source["is_open"]
limit_ip_access["is_open"]=is_open
local ip_qps_limit=source["ip_qps_limit"]
ip_qps_limit=split_str_list(ip_qps_limit,",")
local ip_qps_limit_table={}
table.insert(ip_qps_limit_table,tonumber(ip_qps_limit[1]))
table.insert(ip_qps_limit_table,tonumber(ip_qps_limit[2]))
limit_ip_access["ip_qps_limit"]=ip_qps_limit_table
local interface_ip_qps_limit={}
local white_ips={}
for k,v in pairs(source) do
if string.find(k,"^interface_ip_qps_limit%[*") then
local t=split_str_list(k,".")
table.remove(t,1)
local key=table.concat(t,".")
local vals=split_str_list(v,",")
interface_ip_qps_limit[key]={tonumber(vals[1]),tonumber(vals[2])}
end
if string.find(k,"^white_ips%[*") then
table.insert(white_ips,v)
end
end
limit_ip_access["white_ips"]=white_ips
limit_ip_access["interface_ip_qps_limit"]=interface_ip_qps_limit
lua_context.lua_conf_cache:set("limit_ip_access",cjson.encode(limit_ip_access))
end
end
end
else
ngx.log(ngx.ERR, "request error:" .. tostring(code))
end
end
--<<end: query limit ip access config function definition
-->>begin: subscribe ip blacklist event
local cache=lua_context.mal_ip_cache
local subscribe_mal_ips=function()
if ngx.worker.id() ~= 0 then
return false
end
local connect=limit_ip_redis:getConnect()
if not connect then
ngx.log(ngx.ERR,"subscribe blacklist ip get connection err" )
return false
end
local res, err=connect:subscribe("mal_ips")
if not res then
connect:close()
ngx.log(ngx.ERR,"subscribe blacklist ip connection subscribe err:" .. tostring(err))
return false
end
connect:set_timeout(86400000)
while true do
local res, err = connect:read_reply()
if res then
if res[3] then
local t=cjson.decode(res[3])
local ips=t.ips
local expire=(not t.expire) and 86400 or t.expire
for ip in string.gmatch(ips,"[^',']+") do
cache:set("yh:mip:" .. ip,"1",expire)
ngx.log(ngx.INFO,"nginx subscribe mal ip:" .. tostring(ip) .. ":" .. tostring(expire))
end
end
elseif err ~= "timeout" then
connect:close()
ngx.log(ngx.ERR,"subscribe blacklist ip socket timeout")
return false
end
if ngx.worker.exiting() then
connect:close()
ngx.log(ngx.ERR,"subscribe blacklist ip ngx worker exit")
return false
end
end
return false
end
--<< end: subscribe ip blacklist event
function subscribe_mal_ips_loop()
if ngx.worker.id() ~= 0 then
return
end
local b = ture
while true do
local res=subscribe_mal_ips()
-- subscribe error sleep 10 seconds and then retry
ngx.sleep(10)
if ngx.worker.exiting() then
return
end
end
end
-->>begin: timer at fix rate call function.
local timer_handler
timer_handler=function(premature,t,f,id)
if id then
if ngx.worker.id() == id then
local b,errinfo=pcall(f)
if not b then
ngx.log(ngx.ERR, "task request error:" .. tostring(errinfo))
end
end
else
local b,errinfo=pcall(f)
if not b then
ngx.log(ngx.ERR, "task request error:" .. tostring(errinfo))
end
end
ngx.timer.at(t,timer_handler,t,f)
end
--<<end: timer at fix rate call function.
-- subscribe mal ips task
ngx.timer.at(2,subscribe_mal_ips_loop)
timer_handler(true,5,query_rate_limit_conf,0)
timer_handler(true,7,query_limit_ip_access_conf,0)
timer_handler(true,9,query_common_conf,0)
-- every worker timing schedule configs from share cache
function rate_limit_conf_to_worker()
local t=lua_context.lua_conf_cache:get("api_rate_limit_conf")
if t then
local r=cjson.decode(t)
if r then
lua_context.configs["api_rate_limit_conf"]=r
--ngx.log(ngx.INFO,"++++++++++++++" .. cjson.encode(r.api_rate_limit["web.passport.getUserVerifyInfo"]))
end
end
end
function limit_ip_access_conf_to_worker()
local t=lua_context.lua_conf_cache:get("limit_ip_access")
if t then
local r=cjson.decode(t)
if r then
lua_context.configs["limit_ip_access"]=r
--ngx.log(ngx.INFO,"++++++++++++++" .. cjson.encode(lua_context.configs["limit_ip_access"]))
end
end
end
function query_common_conf_to_worker()
local t=lua_context.lua_conf_cache:get("common_conf")
if t then
local r=cjson.decode(t)
if r then
lua_context.configs["common_conf"]=r
end
end
end
timer_handler(true,2,rate_limit_conf_to_worker)
timer_handler(true,2,limit_ip_access_conf_to_worker)
timer_handler(true,2,query_common_conf_to_worker)
... ...
local lrucache = require "resty.lrucache"
-- init redis twemproxy config
local redis_config1={host="127.0.0.1",port="6379",auth=nil,timeout=20,max_idle_timeout=60000,pool_size=200}
local ip_limit_redis_config={host="127.0.0.1",port="6379",auth=nil,timeout=20,max_idle_timeout=60000,pool_size=100}
local redis_util=require("redisutil")
local redis1=redis_util:new(redis_config1)
local redis_limit_ip=redis_util:new(ip_limit_redis_config)
-- global variable
lua_context={}
lua_context["redises"]={redis1}
lua_context["redis_limit_ip"]=redis_limit_ip
lua_context.mal_ip_cache=ngx.shared.malips
lua_context.lua_conf_cache=ngx.shared.ngxconf
lua_context.configs={}
-- api limit rate default conf
local api_rate_limit_conf={is_open=true,
api_rate_limit={
["app.yohocoin.total"]={100},
["app.yohocoin.subtract"]={100},
["app.yohocoin.add"]={100},
["app.yohocoin.lists"]={100},
["resources.simple.pice"]={600},
["app.product.data"]={600},
["app.product.promotion"]={600},
["app.coupons.queryProdPageCoupons"]={600},
["h5.product.intro"]={600},
["web.brand.banner"]={600},
["app.product.queryShopsInfoById"]={600},
["app.product.queryBuyLimit"]={600},
["app.product.intro"]={600},
["app.product.preference"]={160},
["app.product.shopPreference"]={120},
["app.product.supportService"]={600},
["app.consult.common"]={600},
["app.shop.queryShopsByBrandId"]={600},
["shop.coupons.list"]={600},
["show.recentShareOrderByProductId"]={600},
["app.consult.lastTwo"]={600},
["app.shopsdecorator.getList"]={600},
["app.shops.getIntro"]={600},
["app.shop.banner"]={600},
["web.productBanner.data"]={600},
["web.productModelcard.list"]={600},
["web.productComfort.data"]={600},
["web.productModelTry.data"]={600},
["app.passport.profile"]={400},
["app.activity.getlist"]={400},
["app.home.getInfoNum"]={100},
["app.invitecode.my"]={400},
["app.promotion.getCoupon"]={40,9999992,"人太多啦,稍后再试!"},
["app.coupons.couponSend"]={40,9999992,"人太多啦,稍后再试!"},
["app.coupons.sendCouponByConfig"]={40,9999992,"人太多啦,稍后再试!"},
["app.promotion.getACoupon"]={40,9999992,"人太多啦,稍后再试!"},
["app.coupons.couponsSend"]={40,9999992,"人太多啦,稍后再试!"},
["app.promotion.queryCouponCenter"]={160,9999992,"人太多啦,稍后再试!"},
["app.search.li"]={200},
["app.search.sales"]={160},
["app.search.word"]={400},
["app.limitProduct.hotLimitProduct"]={400},
["app.limitProduct.soonToSaleLimitProduct"]={400},
["app.limitProduct.alreadySaleLimitProduct"]={400},
["app.limitProduct.limitProductHtml"]={400},
["app.limitProduct.limitProductDetail"]={400},
["app.limitProduct.productStatus"]={400},
["app.Shopping.count"]={400},
["app.home.newPreference"]={200},
["app.Shopping.easyPayment"]={40,9999992,"人太多啦,稍后再试!"},
["app.Shopping.easyCompute"]={40,9999992,"人太多啦,稍后再试!"},
["app.Shopping.easySubmit"]={40,9999992,"人太多啦,稍后再试!"},
["app.inbox.getTotal"]={80},
["web.regular.groupsort"]={600},
["web.search.search"]={500},
["web.search.shopListInfo"]={600},
["web.brand.byDomain"]={600},
["web.search.newshelve"]={1000},
["web.search.newShelveBatch"]={600},
["web.regular.groupsort.sale"]={600},
["app.Seckill.payment"]={60,9999992,"活动太火爆了,商品已被秒完!"},
["app.Seckill.compute"]={60,9999992,"活动太火爆了,商品已被秒完!"},
["app.Seckill.submit"]={40,9999992,"活动太火爆了,商品已被秒完!"},
["app.Seckill.countUsableCoupon"]={100},
["app.seckill.queryActivity"]={300,9999992,"人太多啦,稍后再试!"},
["app.seckill.queryProductList"]={300,9999992,"人太多啦,稍后再试!"},
["app.seckill.data"]={300,9999992,"人太多啦,稍后再试!"},
["app.shopping.submitTicket"]={80,9999992,"人太多啦,稍后再试!"},
["app.shopping.ticket"]={80,9999992,"人太多啦,稍后再试!"},
["app.Shopping.submit"]={100,9999992,"人太多啦,稍后再试!"},
["app.Shopping.compute"]={120,9999992,"人太多啦,稍后再试!"},
["app.Shopping.payment"]={120,9999992,"人太多啦,稍后再试!"},
["app.Shopping.enhancedPayment"]={120,9999992,"人太多啦,稍后再试!"},
["app.Shopping.selectCoupon"]={120,9999992,"人太多啦,稍后再试!"},
["app.Shopping.useCoupon"]={120,9999992,"人太多啦,稍后再试!"},
["app.Shopping.countUsableCoupon"]={200},
["app.Shopping.countUsableGiftCard"]={200},
["app.product.seckill.data"]={240,9999992,"人太多啦,稍后再试!"},
["app.product.seckill.promotion"]={240},
["app.product.seckill.intro"]={240},
["app.consult.seckill.common"]={240},
["app.shop.seckill.queryShopsByBrandId"]={240},
["app.consult.seckill.lastTwo"]={240},
["show.seckill.recentShareOrderByProductId"]={240},
["app.depositAdvance.data"]={400},
["app.limitTimeAdvance.data"]={400},
["app.product.shopRecommend"]={160},
["app.product.sizehelper.show"]={600},
["app.product.sizehelper.detail"]={600},
["app.show.instalment"]={600},
["clear.productCache"]={2000},
["app.helper.getServiceOnline"]={600},
["app.Shopping.queryCart"]={300},
["app.Shopping.queryCartCouponCenter"]={300},
["app.Shopping.selectedAndQryCart"]={300,9999992,"人太多啦,稍后再试!"},
["app.Shopping.add"]={100,9999992,"人太多啦,稍后再试!"},
["web.product.shopRecommend"]={40},
["app.coupons.queryUserCouponRemind"]={100},
["app.search.findLike"]={40},
["web.search.findLike"]={40},
["web.search.forseo"]={40},
["app.inbox.getAllInboxCatInfo"]={40},
["app.inbox.getlistnew"]={40},
["app.favorite.product"]={40},
["web.favorite.product"]={40},
["app.favorite.brand"]={40},
["app.coupons.total"]={80},
["app.promotion.couponStatus"]={80},
["app.activity.friendhelp"]={60,9999992,"人太多啦,稍后再试!"},
["user.instalment.activate"]={40},
["user.instalment.getStatus"]={200},
["user.instalment.getBankCards"]={10},
["user.instalment.queryCreditLimit"]={10,9999992,"人太多啦,稍后再试!"},
["user.instalment.getSnsCheckCode"]={10},
["user.instalment.getBankInfoByCardNo"]={10},
["user.instalment.getCardInfoById"]={10},
["app.search.instalment"]={20},
["app.promotion.instalment"]={50},
["app.order.queryAmtInfo"]={10,9999992,"人太多啦,稍后再试!"},
["app.order.queryAmtList"]={10,9999992,"人太多啦,稍后再试!"},
["app.order.queryRePayList"]={10,9999992,"人太多啦,稍后再试!"},
["app.instalment.condition"]={10,9999992,"人太多啦,稍后再试!"},
["app.instalment.pay"]={10,9999992,"人太多啦,稍后再试!"},
["app.instalment.prerepay"]={10,9999992,"人太多啦,稍后再试!"},
["app.instalment.confirmrepay"]={10,9999992,"人太多啦,稍后再试!"},
["app.Buynow.payment"]={10,9999992,"人太多啦,稍后再试!"},
["app.Buynow.countUsableCoupon"]={20},
["app.Buynow.enhancedPayment"]={20,9999992,"人太多啦,稍后再试!"},
["app.Buynow.selectCoupon"]={20},
["app.Buynow.listCoupon"]={20},
["app.Buynow.compute"]={20,9999992,"人太多啦,稍后再试!"},
["app.Buynow.submit"]={10,9999992,"人太多啦,稍后再试!"},
["app.Buynow.useCoupon"]={10},
["app.Buynow.usePromotionCode"]={10},
["app.SpaceOrders.getInstallment"]={10},
["app.SpaceOrders.installDetail"]={10},
["user.instalment.validateSnsCheckCode"]={10},
["user.instalment.reSendSms4FcpService"]={10},
["user.instalment.bindingCards"]={5},
["app.search.promotion"]={80},
["app.search.newPromotion"]={120},
["app.search.zq.filter"]={40},
["app.search.promotion.filter"]={40},
["app.search.coupon.filter"]={40},
["app.search.fuzzy.filter"]={40},
["app.search.category.filter"]={40},
["app.search.filter"]={40},
["app.search.breakSize.filter"]={40},
["app.search.pool.filter"]={40},
["app.search.shop.filter"]={40},
["app.search.good"]={40},
["app.search.fuzzy"]={120},
["app.search.category.productList"]={120},
["app.search.productList"]={120},
["app.search.recomandLi"]={60},
["app.search.shop.productList"]={120},
["app.search.promotion.productList"]={80},
["app.search.zq.productList"]={80},
["app.search.pool.productList"]={80},
["app.search.newProduct.shop"]={80},
["app.search.coupon.productList"]={80},
["app.search.newLast7day"]={120},
["app.resources.holidayGifConfig"]={400},
["app.passport.verify"]={80},
["clear.productCache.batch"]={200},
["/operations/api/v5/resource/get"]={800},
["/operations/api/v5/resource/home"]={400,9999992},
["/operations/api/v5/webshare/getShare"]={400},
["/operations/api/v5/entrance/getEntrance"]={400},
["/operations/api/v6/category/getCategory"]={400},
["/operations/api/v4/icon/getIcon"]={400},
["/guang/api/v1/article/getArticleNotice"]={100},
["/payment/alipay_data"]={200},
["/payment/applepay_data"]={200},
["/payment/weixin_data"]={200},
["/payment/alipay_notify"]={200},
["/payment/weixin_notify"]={200},
["/payment/applepay_notify"]={200},
["/erp/sync/promotion/update"]={600},
["/erp/clear/batch/productPriceCache"]={800}
}
}
-- limit ip access conf
local limit_ip_access={
is_open=true, white_ips={
"172.31.0.0/16",
"10.66.0.0/16",
"54.222.0.0/16",
"118.89.221.106/32",
"123.206.21.19/32",
"123.206.73.107/32",
"54.223.0.0/16"
},
ip_qps_limit={60,1},
interface_ip_qps_limit={
["app.bind.sendThirdBindMobileCodeOnly"]={15,2},
["app.bind.sendThirdBindMobileCodeOnlyImg"]={15,2},
["app.bind.sendChangeBindMobileCodeOnlyImg"]={15,2},
["/smart/way"]={20,3}
}
}
local common_conf={
-- this variable control limit ip & limit rate flow execute
-- true: open ,false: close
lua_golbal_switch = true
}
lua_context.configs["common_conf"]=common_conf
lua_context.configs["limit_ip_access"]=limit_ip_access
lua_context.configs["api_rate_limit_conf"]=api_rate_limit_conf
... ...
function getIpNumber(ip)
local ipArray={};
for w in string.gmatch(ip,"%d+") do
ipArray[#ipArray+1]=w;
end
local ipNumber=0;
for k,v in ipairs(ipArray) do
if k==1 then
ipNumber=ipNumber+tonumber(v)*167777216;
elseif k==2 then
ipNumber=ipNumber+tonumber(v)*65536;
elseif k==3 then
ipNumber=ipNumber+tonumber(v)*256;
else
ipNumber=ipNumber+tonumber(v);
end
end
return ipNumber;
end
function allow()
local beginIp=ngx.var.begin_ip;
local endIp=ngx.var.end_ip;
local beginNum=getIpNumber(beginIp);
local endNum=getIpNumber(endIp);
local realIp=ngx.var.real_ip;
local realIpNum=getIpNumber(realIp);
if realIpNum < beginNum or realIpNum > endNum then
ngx.exit(ngx.HTTP_FORBIDDEN);
end
end
allow()
... ...
local modname= ...
local M={}
_G[modname]=M
package.loaded[modname]=M
local h2b = {
["0"] = "0000",
["1"] = "0001",
["2"] = "0010",
["3"] = "0011",
["4"] = "0100",
["5"] = "0101",
["6"] = "0110",
["7"] = "0111",
["8"] = "1000",
["9"] = "1001",
["A"] = "1010",
["B"] = "1011",
["C"] = "1100",
["D"] = "1101",
["E"] = "1110",
["F"] = "1111"
}
function M:split_ip(ip)
if not ip or type(ip) ~="string" then
return nil
end
local t={}
for w in string.gmatch(ip,"([^'.']+)") do
table.insert(t,tonumber(w))
end
return t
end
function M:ip_to_binary_str(ip)
if not ip or type(ip)~="string" then
return nil
end
local ip_table=self:split_ip(ip)
if not ip_table then
return nil
end
local ip_table_length=#ip_table
if ip_table_length~=4 then
return nil
end
local ip_str=""
for i=1,ip_table_length do
local x=string.upper(string.format("%x",ip_table[i]))
local len=string.len(x)
if len==1 then
x="0" .. x
elseif len>2 then
return nil
end
local f=string.sub(x,1,1)
local s=string.sub(x,2,2)
ip_str=ip_str .. h2b[f] .. h2b[s]
end
return ip_str
end
function M:check_ip_in_ipblock(ip,ipblock)
if (not ip) or (not ipblock) then
return false
end
local f,t=string.find(ipblock,"/")
if f then
local ipblock_head=string.sub(ipblock,0,f-1)
local ipblock_tail=string.sub(ipblock,f+1)
local ipblock_head_b=self:ip_to_binary_str(ipblock_head)
local ip_b=self:ip_to_binary_str(ip)
local mask_len=tonumber(ipblock_tail)
if (not mask_len)or mask_len> 32 or mask_len<0 or (not ip_b) or (not ipblock_head_b) then
return false
end
if string.sub(ipblock_head_b,0,mask_len) == string.sub(ip_b,0,mask_len) then
return true
end
else
if ipblock==ip then
return true
end
end
return false
end
function M:pcall_check_ip_in_ipblock(ip,ipblock,default)
local flag,res=pcall(self.check_ip_in_ipblock,self,ip,ipblock)
if flag then
return res
end
return default
end
... ...
local rateLimit = require "limit_common_flow"
local cjson = require "cjson"
local open_limit_flow=true
local api_default_max_per_sencond=60
local service_default_max_per_sencond=60
local default_err_code=9999991
local default_err_msg="系统正忙,请稍后重试!"
function get_req_param(req_param)
local method = ngx.var.request_method
if "GET" == method then
return ngx.req.get_uri_args()[req_param]
else
ngx.req.read_body()
return ngx.req.get_post_args()[req_param]
end
end
function get_req_uri()
local limit_config=lua_context.configs["api_rate_limit_conf"]
local req_service_url = ngx.var.uri
if (not limit_config.is_open) or (not req_service_url) then
return nil
end
local beginIndex, endIndex = string.find(req_service_url, "apigateway")
if endIndex == nil then
return nil
end
local real_url=string.sub(req_service_url,endIndex+1,string.len(req_service_url))
return real_url
end
function extract_limit_method()
local uri_method=get_req_uri()
if uri_method and uri_method ~= "/" and uri_method ~= "" then
return uri_method
end
local method= get_req_param("method")
return method
end
function rate_limit()
local common_config=lua_context.configs["common_conf"]
if not common_config.lua_golbal_switch then
return
end
local limit_config=lua_context.configs["api_rate_limit_conf"]
local api_rate_limit=limit_config.api_rate_limit
local req_uri_method = extract_limit_method()
--ngx.log(ngx.INFO,"=================>>" .. cjson.encode(api_rate_limit[req_uri_method]))
if (not limit_config.is_open) or (not req_uri_method) then
return
end
local max_per_sencond=api_default_max_per_sencond
if api_rate_limit[req_uri_method] and api_rate_limit[req_uri_method][1] then
max_per_sencond=api_rate_limit[req_uri_method][1]
end
local err_code=default_err_code
if api_rate_limit[req_uri_method] and api_rate_limit[req_uri_method][2] then
err_code=api_rate_limit[req_uri_method][2]
end
local err_msg=default_err_msg
if api_rate_limit[req_uri_method] and api_rate_limit[req_uri_method][3] then
err_msg=api_rate_limit[req_uri_method][3]
end
local flag=rateLimit.limit_flow("yh:nginx:limitflow:" .. req_uri_method,max_per_sencond)
if not flag then
ngx.log(ngx.ERR,"The request is limited :" .. req_uri_method)
ngx.header["Content-Type"]="application/json;charset=utf-8"
local msg='{"code":' .. err_code .. ',"message":"'.. err_msg .. '"}'
ngx.say(msg)
ngx.exit(ngx.HTTP_OK)
end
end
rate_limit()
... ...
local rateLimit ={}
function rateLimit.limit_flow(limit_key,max_limit,seconds)
if not limit_key or not max_limit then
return true
end
local redises=lua_context["redises"]
local redis_util=redises[math.random(#redises)]
local redis=redis_util:getConnect()
if not redis then
return true
end
local limit=0
if seconds then
limit=redis:eval("local current;current=redis.call('incr',KEYS[1]);if tonumber(current) == 1 then redis.call('expire',KEYS[1],KEYS[2]) end return current",2,limit_key,seconds)
else
limit=redis:eval("local current;current=redis.call('incr',KEYS[1]);if tonumber(current) == 1 then redis.call('expire',KEYS[1],2) end return current",1,limit_key)
end
redis_util:close(redis)
if limit then
if tonumber(limit)>max_limit then
return false
else
return true
end
end
return true
end
return rateLimit
... ...
local modname= ...
local M={}
_G[modname]=M
package.loaded[modname]=M
local cjson=require "cjson"
local rate_limit = require "limit_common_flow"
local iptool=require "iptool"
local lrucache = require "resty.lrucache"
local cache=lua_context.mal_ip_cache
local redis_limit_ip=lua_context["redis_limit_ip"]
function M.in_array(value,arr)
if not value then
return false
end
if not arr then
return false
end
for k, v in ipairs(arr) do
if value==v then
return true
end
end
return false
end
function M:get_req_param(req_param)
local method = ngx.var.request_method
if "GET" == method then
return ngx.req.get_uri_args()[req_param]
else
ngx.req.read_body()
return ngx.req.get_post_args()[req_param]
end
end
--1:api 2:service
function M:limit_ip_access()
local limit_ip_config=lua_context.configs["limit_ip_access"]
local common_config=lua_context.configs["common_conf"]
-- global switch control
if not common_config.lua_golbal_switch then
return
end
-- is open ip limit access control
if not limit_ip_config.is_open then
return
end
local ip=ngx.var.real_ip
if not ip then
return
end
--check is in white ip list
local white_ips_length=#limit_ip_config.white_ips
if white_ips_length >0 then
for i=1,white_ips_length do
local is_in_white_ips=iptool:pcall_check_ip_in_ipblock(ip,limit_ip_config.white_ips[i],false)
if is_in_white_ips then
return
end
end
end
-- check ip is in limit ip list
local is_limit=self.in_array(ip,limit_ip_config.limit_ip_config)
local limit_type="0"
-- check ip access is arrive max access
local ip_qps_limit=limit_ip_config.ip_qps_limit
if (not is_limit) and ip_qps_limit and ip_qps_limit[1] and ip_qps_limit[2] then
if not rate_limit.limit_flow("yh:limit:ip:" .. ip,limit_ip_config.ip_qps_limit[1],limit_ip_config.ip_qps_limit[2]) then
is_limit=true
limit_type="1"
end
end
-- check method or uri limit
local req_uri_method
local method_limit_conf
if(not is_limit) and limit_ip_config.interface_ip_qps_limit then
req_uri_method = self:get_req_param("method")
if not req_uri_method then
req_uri_method = ngx.var.uri
end
method_limit_conf=limit_ip_config.interface_ip_qps_limit[req_uri_method]
end
if (not is_limit) and req_uri_method and method_limit_conf then
if not rate_limit.limit_flow("yh:limit:ip:" .. ip .. ":" .. req_uri_method,method_limit_conf[1],method_limit_conf[2]) then
is_limit=true
limit_type="2"
end
end
local is_white_method=self.in_array(req_uri_method,limit_ip_config.white_method)
if is_white_method then
return
end
-- check redis config
if not is_limit then
local res=cache:get("yh:mip:" .. ip)
if res then
is_limit=true
limit_type="3"
end
end
if is_limit then
ngx.log(ngx.ERR, "[LimitIPAccess:ip]:" .. ip .. ",type:" .. limit_type)
ngx.header["Content-type"]="application/json;charset=utf-8"
if limit_type=="3" then
ngx.header["x-yoho-malicode"]="10011"
end
ngx.exit(ngx.HTTP_OK)
end
end
-- malicious ip
function M:mal_ip()
local method=self:get_req_param("method")
local ips=self:get_req_param("ips")
local expire=self:get_req_param("expire")
ngx.header["Content-type"]="application/json;charset=utf-8"
if not method or not ips then
ngx.say('{"code": 400, "msg": "params error!"}')
ngx.exit(ngx.HTTP_OK)
end
if method == 'publish' then
local t={}
t.ips=ips
t.expire=expire
redis_limit_ip:cmd("publish","mal_ips",cjson.encode(t))
end
local exists={}
for ip in string.gmatch(ips,"[^',']+") do
if method == 'add' then
local expire= (not expire) and 86400 or expire
cache:set("yh:mip:" .. ip,"1",expire)
elseif method == 'del' then
cache:delete("yh:mip:" .. ip)
elseif method == 'exists' then
local res=cache:get("yh:mip:" .. ip)
res= res and true or false
exists[#exists+1]=tostring(res)
end
end
local body=table.concat(exists,",")
ngx.say('{"code": 200, "msg": "'.. body ..'"}')
ngx.exit(ngx.HTTP_OK)
end
... ...
#! /usr/bin/env lua
--[[
redis操作库
--]]
local resty_redis = require("resty.redis")
Redis ={}
function Redis:new(datasource)
local o={}
o.datasource = datasource or {
host = "127.0.0.1",
port = "6379",
auth = nil,
timeout = 3000,
max_idle_timeout = 60000,
pool_size = 1000
}
setmetatable(o,self)
self.__index=self
ngx.log(ngx.DEBUG, "[Redis:init] datasource ")
return o
end
function Redis:getConnect()
local connect, err = resty_redis:new()
if not connect then
ngx.log(ngx.ERR, "[Redis:getConnect] failed to create redis : " .. self.datasource.host, err)
return nil
end
connect:set_timeout(self.datasource.timeout)
local ok, err = connect:connect(self.datasource.host, self.datasource.port)
if not ok then
ngx.log(ngx.ERR, "[Redis:getConnect] failed to connect redis : " .. self.datasource.host, err)
return nil
end
if self.datasource.auth then
local res, err = connect:auth(self.datasource.auth)
end
return connect
end
function Redis:cmd(method, ...)
local connect = self:getConnect()
if not connect then
return nil
end
ngx.log(ngx.DEBUG, "[Redis:cmd] connected to redis ok.")
-- exec cmd
local res, err = connect[method](connect, ...)
-- close
self:close(connect)
return res, err
end
function Redis:close(connect)
if not connect then
return
end
if self.datasource.pool_size <= 0 then
connect:close()
return
end
-- put it into the connection pool of size 100,
-- with 10 seconds max idle timeout
local ok, err = connect:set_keepalive(self.datasource.max_idle_timeout, self.datasource.pool_size)
if not ok then
ngx.log(ngx.ERR, "[Redis:close] set keepalive failed : ", err)
else
ngx.log(ngx.DEBUG, "[Redis:close] set keepalive ok.")
end
end
return Redis
... ...
local limitIpAccess= require 'limit_ip_access'
local redirectPath = "/apigateway" ;
--- modify content-type, add utf-8 charset if absent
if ngx.req.get_headers()["content-type"] ~= nil and ngx.req.get_headers()["content-type"] == 'application/x-www-form-urlencoded' then
ngx.req.set_header("Content-Type", "application/x-www-form-urlencoded;charset=utf-8");
end
--- if url match the service url, just redirect
if ngx.var.service_matched == "1" then
limitIpAccess:limit_ip_access()
local uri = ngx.var.uri
if nil == ngx.req.get_uri_args() then
return ngx.exec(redirectPath ..uri);
else
return ngx.exec(redirectPath ..uri, ngx.req.get_uri_args());
end
end
-- method ==9 is not supported , ngx will return directly with reponse message: "please update your app "
local VERSION_ONLINE_NEW=0;
local METHOD_NEW={
["app.passport.signin"]=9,
["app.drawline.getQueueList"]=4,
["app.drawline.addQueue"]=4,
["app.drawline.getUserActivityInfo"]=4,
["app.drawline.getActivityInfo"]=4,
["app.drawline.getLuckyUserList"]=4,
["app.drawline.changeTaskValue"]=4,
["wap.activity.getActivityInfo"]=4,
["app.activity.getInfoOfOrderShare"]=4,
["wap.order.drawOrderShareCoupon"]=4,
["wap.order.registerAndSendCoupon"]=4,
["wechat.user.follow"]=5,
["wechat.user.unfollow"]=5,
["wechat.token.getmini"]=5,
["wechat.token.get"]=5
};
--decode request method
local request_method = ngx.var.request_method;
--if header "content_type" is multipart/form-data, send to IDC
--multipart/form-data maybe used to upload image, as user header image
local request_headers = ngx.req.get_headers();
if request_headers["content_type"] ~= nil and string.sub(request_headers["content_type"],1,19) == "multipart/form-data" then
ngx.log(ngx.INFO,"redirect don't support multipart/form-data, method is " .. request_method .. ", send to api.open.yohobuy.com");
if "GET" == request_method then
ngx.exec(redirectPath,ngx.req.get_uri_args());
else
ngx.exec(redirectPath);
end
else
--if header "content_type" is x-www-form-urlencoded, decode arguments
local args = nil ;
if "GET" == request_method then
args = ngx.req.get_uri_args()
elseif "POST"== request_method then
ngx.req.read_body()
args = ngx.req.get_post_args()
end
--decode arguments "method" and "v" in request data
local method = args["method"];
local version = tonumber(args["v"]);
if version == nil then
version=1
end
if method == nil then
method="not_support_method"
end
--ngx.log(0,method);
--ngx.log(0,version);
--ngx.log(0,METHOD_NEW[method]);
---- make sure method must be present -------
if method == nil then
ngx.log(ngx.INFO, "method not allowed")
ngx.exit(ngx.HTTP_NOT_ALLOWED )
end
if METHOD_NEW[method] == 9 then
ngx.header["Content-type"]="application/json;charset=utf-8"
local rsp = '{"alg": "SALT_MD5","code": 112232323,"data": {},"md5": "f4a7a490bb6666b005008d795ed14e5d", "message": "请升级客户端版本!"}'
ngx.say(rsp)
ngx.exit(ngx.HTTP_OK)
end
if METHOD_NEW[method] == 4 then
redirectPath = "/activityApi";
ngx.log(ngx.INFO,"request method is " .. request_method .. ", version is " .. version .. ",method is " .. method .. ", send to apigateway" .. redirectPath);
if "GET" == request_method then
ngx.exec(redirectPath,ngx.req.get_uri_args());
elseif "POST"== request_method then
ngx.exec(redirectPath);
end
elseif METHOD_NEW[method] == 5 then
ngx.log(ngx.INFO,"request method is " .. request_method .. ", version is " .. version .. ",method is " .. method .. ", send to activityApi");
if "GET" == request_method then
ngx.exec("/wechatApi",ngx.req.get_uri_args());
elseif "POST"== request_method then
ngx.exec("/wechatApi");
end
else
limitIpAccess:limit_ip_access()
if "GET" == request_method then
ngx.exec(redirectPath,ngx.req.get_uri_args());
elseif "POST"== request_method then
ngx.exec(redirectPath);
end
end
end
... ...
----- 内部域名如果http头中带有 X-YOHO-IP，则是PC/h5过来的请求，直接用这个作为real ip
----- 经过动态CDN,如果x-forward-for 只有1个IP地址，则取第一个;>=2个IP地址，例如 ip1,ip2,ip3,ip4， 则取倒数第二个，即ip3
----- 请求不经过动态CDN,取 x-forward-for 中最后一个IP地址，例如 ip1,ip2,ip3,ip4，取ip4
local realIp = ngx.req.get_headers()["X-YOHO-IP"]
if realIp and (string.find(ngx.var.http_host,"yohoops%.org") or string.find(ngx.var.http_host,"yohops%.com")) then
return realIp
end
local xForwardFor = ngx.var.http_x_forwarded_for;
if xForwardFor then
local ips={}
string.gsub(xForwardFor,'[^,]+',function(w) table.insert(ips, w) end )
if string.find(ngx.var.http_host,"api%-dc",1) == 1 then
if #ips >= 2 then
realIp=ips[#ips-1]
elseif #ips<=1 then
realIp=ips[1]
end
else
realIp=ips[#ips]
end
end
-- if not exist, then setup remote_addr
if realIp == nil then
realIp = ngx.var.remote_addr;
end
return realIp;
... ...
server {
listen 80 default_server backlog=8192;
## for proxy
proxy_http_version 1.1;
proxy_set_header Connection "";
server_name api.yoho.cn api.yoho.yohoops.org apigray.yoho.cn eshop.yoho.cn service.yoho.cn service.yoho.yohoops.org api-dc1-qq.yoho.cn api-dc1-ali.yoho.cn api-dc2-qq.yoho.cn api-dc2-ali.yoho.cn;
access_log /Data/logs/nginx/api.yoho.cn_access.log fenxi;
error_log /Data/logs/nginx/api.yoho.cn_error.log;
set $real_ip "";
set_by_lua_file $real_ip conf/setup.lua;
access_by_lua_file 'conf/lua/limit_api_flow.lua';
#################### reject http HEAD & status check / .ico quest #########
# HEAD REQUEST RETURN 405
if ( $request_method = HEAD ) {
return 200;
}
# nginx status check
location = /status_check {
default_type text/html;
return 200 'server is ok!';
}
# cdn check
location = /test.html {
return 200;
}
location = /favicon.ico {
log_not_found off;
access_log off;
}
#################### begin match service.yoho.cn #########################
location /shops/service/v1/ {
set $service_matched 1;
rewrite_by_lua_file conf/redirect.lua;
}
location /operations/{
set $service_matched 1;
rewrite_by_lua_file conf/redirect.lua;
}
location /suggest/ {
set $service_matched 1;
rewrite_by_lua_file conf/redirect.lua;
}
location /payment/ {
set $service_matched 1;
rewrite_by_lua_file conf/redirect.lua;
}
location /guang/ {
set $service_matched 1;
rewrite_by_lua_file conf/redirect.lua;
}
location /passport/ {
set $service_matched 1;
rewrite_by_lua_file conf/redirect.lua;
}
#smart...
location /smart/ {
set $service_matched 1;
rewrite_by_lua_file conf/redirect.lua;
}
# union
location ^~ /union/ {
proxy_pass http://union_pools;
proxy_set_header X-Real-IP $real_ip;
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
proxy_set_header Accept-Encoding "gzip";
proxy_http_version 1.1;
proxy_set_header Connection "";
proxy_set_header Host $host;
}
# activity #
location ^~ /activity/ {
proxy_redirect off;
proxy_pass http://activityApi/activity/;
proxy_set_header X-Real-IP $real_ip;
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
proxy_set_header Accept-Encoding "gzip";
proxy_http_version 1.1;
proxy_set_header Connection "";
proxy_set_header Host $host;
}
# wechat #
location ^~ /wechat/ {
proxy_redirect off;
proxy_pass http://apiWechat/wechat/;
proxy_set_header X-Real-IP $real_ip;
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
proxy_set_header Accept-Encoding "gzip";
proxy_http_version 1.1;
proxy_set_header Connection "";
proxy_set_header Host $host;
}
# ali_pay_old #
location ^~ /pay/notice/alimobilenoticemini {
proxy_redirect off;
proxy_pass http://apigateway/gateway/payment/alipay_notify;
proxy_set_header X-Real-IP $real_ip;
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
proxy_set_header Accept-Encoding "gzip";
proxy_http_version 1.1;
proxy_set_header Connection "";
proxy_set_header Host $host;
}
# favourite, all request starts with /favourite
location ^~ /favorite {
proxy_redirect off;
proxy_pass http://brower/brower;
proxy_set_header X-Real-IP $real_ip;
proxy_set_header X-Forwarded-For $remote_addr;
proxy_http_version 1.1;
proxy_set_header Connection "";
}
#################### end match service.yoho.cn #########################
#################### begin match api.yoho.cn #########################
location / {
rewrite_by_lua_file conf/redirect.lua;
}
#################### end match api.yoho.cn #########################
location /apigateway{
internal;
proxy_redirect off;
proxy_pass http://apigateway/gateway;
proxy_set_header X-Real-IP $real_ip;
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
proxy_set_header Accept-Encoding "gzip";
proxy_http_version 1.1;
proxy_set_header Connection "";
proxy_set_header Host $host;
}
location = /activityApi{
internal;
proxy_redirect off;
proxy_pass http://activityApi/activity;
proxy_set_header X-Real-IP $real_ip;
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
proxy_set_header Accept-Encoding "gzip";
proxy_http_version 1.1;
proxy_set_header Connection "";
proxy_set_header Host $host;
}
location = /wechatApi{
internal;
proxy_redirect off;
proxy_pass http://apiWechat/wechat;
proxy_set_header X-Real-IP $real_ip;
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
proxy_set_header Accept-Encoding "gzip";
proxy_http_version 1.1;
proxy_set_header Connection "";
proxy_set_header Host $host;
}
}
... ...
## single http request, only forward to one data center (qcloud, e.g)
server {
listen 80 ;
server_name api_single.yoho.cn single.yoho.cn ;
access_log /Data/logs/nginx/api.single.yoho.cn_access.log fenxi;
error_log /Data/logs/nginx/api.single.yoho.cn_error.log;
set $real_ip "";
set_by_lua_file $real_ip conf/setup.lua;
if ( $request_method = HEAD ) {
return 200;
}
# favourite
location ^~ /favorite {
proxy_redirect off;
proxy_pass http://brower/brower;
proxy_set_header X-Real-IP $real_ip;
proxy_set_header X-Forwarded-For $remote_addr;
proxy_http_version 1.1;
proxy_set_header Connection "";
}
# brower service
location = / {
proxy_redirect off;
proxy_pass http://apigateway/gateway;
proxy_set_header X-Real-IP $real_ip;
proxy_set_header X-Forwarded-For $remote_addr;
proxy_http_version 1.1;
proxy_set_header Connection "";
}
}
\ No newline at end of file
... ...
# Interface FOR YOHO ERP SYSTEM
server {
listen 80;
server_name erp.yoho.yohoops.org;
proxy_http_version 1.1;
proxy_set_header Connection "";
access_log /Data/logs/nginx/erp.yoho.yohoops.org_access.log fenxi;
error_log /Data/logs/nginx/erp.yoho.yohoops.org_error.log;
if ( $request_method = HEAD ) {
return 200;
}
location /erp/ {
proxy_pass http://erpgateway/erp-gateway/erp/;
proxy_set_header X-Real-IP $real_ip;
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
proxy_set_header Accept-Encoding "gzip";
proxy_http_version 1.1;
proxy_set_header Connection "";
proxy_set_header Host $host;
}
## malicious ip management
location = /malIp {
content_by_lua '
local limit_ip_access = require "limit_ip_access";
limit_ip_access:mal_ip();
';
}
}
\ No newline at end of file
... ...
## dual centor, single services.
server {
listen 80 ;
server_name single.yohoops.org;
access_log /Data/logs/nginx/single.yohoops.org.log fenxi;
error_log /Data/logs/nginx/single.yohoops.org_error.log;
set $real_ip "";
set_by_lua_file $real_ip conf/setup.lua;
if ( $request_method = HEAD ) {
return 200;
}
# single
location / {
proxy_redirect off;
proxy_pass http://apigateway/gateway;
proxy_set_header X-Real-IP $real_ip;
proxy_set_header X-Forwarded-For $remote_addr;
proxy_http_version 1.1;
proxy_set_header Connection "";
}
location ^~ /activity/ {
proxy_redirect off;
proxy_pass http://activityApi/activity/;
proxy_set_header X-Real-IP $real_ip;
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
proxy_set_header Accept-Encoding "gzip";
proxy_http_version 1.1;
proxy_set_header Connection "";
proxy_set_header Host $host;
}
location ^~ /union/ {
proxy_pass http://union_pools;
proxy_set_header X-Real-IP $real_ip;
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
proxy_set_header Accept-Encoding "gzip";
proxy_http_version 1.1;
proxy_set_header Connection "";
proxy_set_header Host $host;
}
}
... ...
server{
listen 80;
server_name union.yoho.cn webunion.yohobuy.com;
access_log /Data/logs/nginx/access_union.log fenxi;
location ^~ /admin {
if ( $proxy_add_x_forwarded_for !~* "(218.94.77.166|218.94.75.50|218.94.75.58|106.39.86.*|106.38.38.*)" ) {
return 401;
break;
}
proxy_redirect off;
proxy_pass http://union_pools/union/admin;
proxy_set_header Host $host;
proxy_set_header X-Real-IP $http_x_forwarded_for;
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
proxy_set_header Accept-Encoding "gzip";
}
location ^~ /union/admin {
if ( $proxy_add_x_forwarded_for !~* "(218.94.75.50|218.94.75.58|218.94.77.166|106.39.86.*|106.38.38.*)" ) {
return 401;
break;
}
proxy_redirect off;
proxy_pass http://union_pools/union/admin;
proxy_set_header Host $host;
proxy_set_header X-Real-IP $http_x_forwarded_for;
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
proxy_set_header Accept-Encoding "gzip";
}
location = /apple-app-site-association {
root /usr/local/nginx/html;
}
location /.well-known/apple-app-site-association {
proxy_pass http://activity.yoho.cn/.well-known/apple-app-site-association;
}
location / {
proxy_redirect off;
proxy_pass http://union_pools;
proxy_set_header Host $host;
proxy_set_header X-Real-IP $http_x_forwarded_for;
proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
proxy_set_header Accept-Encoding "gzip";
}
}
... ...
---
# handlers file for openresty
\ No newline at end of file
... ...
galaxy_info:
author: chunhua.zhang@yoho.cn
description: update java openresty config
company: yohobuy.com
# If the issue tracker for your role is not on github, uncomment the
# next line and provide a value
# issue_tracker_url: http://example.com/issue/tracker
# Some suggested licenses:
# - BSD (default)
# - MIT
# - GPLv2
# - GPLv3
# - Apache
# - CC-BY
license: license (GPLv2, CC-BY, etc)
min_ansible_version: 1.2
# If this a Container Enabled role, provide the minimum Ansible Container version.
# min_ansible_container_version:
# Optionally specify the branch Galaxy will use when accessing the GitHub
# repo for this role. During role install, if no tags are available,
# Galaxy will use this branch. During import Galaxy will access files on
# this branch. If Travis integration is configured, only notifications for this
# branch will be accepted. Otherwise, in all cases, the repo's default branch
# (usually master) will be used.
#github_branch:
#
# platforms is a list of platforms, and each platform has a name and a list of versions.
#
# platforms:
# - name: Fedora
# versions:
# - all
# - 25
# - name: SomePlatform
# versions:
# - all
# - 1.0
# - 7
# - 99.99
galaxy_tags: []
# List tags for your role here, one per line. A tag is a keyword that describes
# and categorizes the role. Users find roles by searching for tags. Be sure to
# remove the '[]' above, if you add tags to this list.
#
# NOTE: A tag is limited to a single word comprised of alphanumeric characters.
# Maximum 20 tags per role.
dependencies: []
# List your role dependencies here, one per line. Be sure to remove the '[]' above,
# if you add dependencies to this list.
\ No newline at end of file
... ...
---
# tasks file for openresty
# Configure resolv.conf
- name: copy lualib files to openresty
copy:
src: lualib/resty/
dest: {{ path }}/lualib/resty/
- name: copy yoho lua files to openresty
copy:
src: nginx/conf/
dest: {{ path }}/nginx/conf/
- name: update nginx upstream for file - [nginx.conf]
template:
src: nginx.conf.j2
dest: {{ path }}/nginx/conf
\ No newline at end of file
... ...
user www www;
worker_processes 8;
worker_cpu_affinity 00000001 00000010 00000100 00001000 00010000 00100000 01000000 10000000;
pid logs/nginx.pid;
# [ debug | info | notice | warn | error | crit ]
#error_log /Data/logs/nginx/nginx_error.log info;
error_log /Data/logs/nginx/nginx_error.log info;
#Specifies the value for maximum file descriptors that can be opened by this process.
worker_rlimit_nofile 51200;
events
{
use epoll;
#maxclient = worker_processes * worker_connections / cpu_number
worker_connections 51200;
}
http
{
include mime.types;
default_type application/octet-stream;
#charset gb2312,utf-8;
charset utf-8;
log_format fenxi '$remote_addr|$http_x_forwarded_for|[$time_local]|$http_host|$request|'
'$status|$body_bytes_sent|$request_time|$upstream_response_time|$upstream_cache_status|$http_referer|'
'$http_user_agent|$upstream_addr';
#General Options
server_names_hash_bucket_size 128;
client_header_buffer_size 32k;
large_client_header_buffers 4 32k;
client_body_buffer_size 8m; #256k
#
#server_tokens off;
ignore_invalid_headers on;
recursive_error_pages on;
server_name_in_redirect off;
sendfile on;
#timeouts
keepalive_timeout 75s;
keepalive_requests 10000;
#test
#client_body_timeout 3m;
#client_header_timeout 3m;
#send_timeout 3m;
#TCP Options
tcp_nopush on;
tcp_nodelay on;
#fastcgi options
fastcgi_connect_timeout 300;
fastcgi_send_timeout 300;
fastcgi_read_timeout 300;
fastcgi_buffer_size 64k;
fastcgi_buffers 4 64k;
fastcgi_busy_buffers_size 128k;
fastcgi_temp_file_write_size 128k;
#hiden php version
fastcgi_hide_header X-Powered-By;
#size limits
client_max_body_size 50m;
gzip on;
gzip_min_length 1k;
gzip_buffers 4 16k;
#gzip_http_version 1.0;
gzip_comp_level 2;
gzip_types text/plain application/x-javascript text/css application/xml application/json;
gzip_vary on;
add_header x-yh-nginx-dc {{ dc }};
fastcgi_temp_path /dev/shm/fastcgi_temp;
client_body_temp_path /dev/shm/client_body_temp;
# where the lua package exists
lua_package_path "/Data/local/openresty-1.9.15.1/nginx/conf/lua/?.lua;;";
init_by_lua_file "conf/lua/init_lua.lua";
lua_shared_dict luacachedb 500m;
lua_shared_dict malips 10m;
lua_shared_dict ngxconf 20m;
init_worker_by_lua_file "conf/lua/init_config_worker.lua";
#brower service
upstream brower {
{% for i in upstream_brower %}
server {{ i }} ;
{% endfor %}
keepalive 100;
}
#apigatewaystart
upstream apigateway {
{% for i in upstream_gateway %}
server {{ i }} max_fails=5 fail_timeout=3s;
{% endfor %}
keepalive 32;
}
#apigatewayend
# activity
upstream activityApi{
{% for i in upstream_activity %}
server {{ i }} ;
{% endfor %}
keepalive 100;
}
# wechat
upstream apiWechat {
{% for i in upstream_wechat %}
server {{ i }} ;
{% endfor %}
keepalive 100;
}
# erp gateway
upstream erpgateway {
{% for i in upstream_erpgateway %}
server {{ i }} ;
{% endfor %}
keepalive 100;
}
# union servers
upstream union_pools {
{% for i in upstream_union %}
server {{ i }} ;
{% endfor %}
}
#upstream
fastcgi_next_upstream error timeout invalid_header http_500;
#limit_zone limit $binary_remote_addr 1m;
#fastcgi cache
#fastcgi_cache_path /nginxcache levels=1:2 keys_zone=two:10m inactive=1d max_size=3000m;
#for example just for study! have fun!
include vhosts/api.yoho.cn.conf;
include vhosts/api_single.yoho.cn.conf;
include vhosts/erp.yoho.yohoops.org.conf;
include vhosts/single.conf;
include vhosts/union.yoho.cn.conf;
}
... ...
---
- hosts: localhost
remote_user: root
roles:
- openresty
\ No newline at end of file
... ...
---
# vars file for openresty
\ No newline at end of file
... ...