ops / monitor-service · Commits

Go to a project

Authored by Xu
Commit c1d210e7dd53a90d597f053b8a5786f7450579e7 1 parent 585b6597

uic同时接收恶意Ip

Inline Side-by-side
Showing 10 changed files with 233 additions and 49 deletions
... ... @@ -13,6 +13,7 @@ import org.springframework.beans.factory.annotation.Autowired;
import com.alibaba.fastjson.JSON;
import com.monitor.other.maliciousip.MaliciousIpBL;
import com.monitor.other.maliciousip.service.impl.MaliciousIpToUic;
import com.monitor.other.maliciousip.util.IpWhiteListUtil;
import com.monitor.other.maliciousip.util.RedisReadUtil;
import com.monitor.other.maliciousip.util.RedisWriteUtil;
... ... @@ -70,23 +71,28 @@ public class MaliciousIpJob {
}
List<MaliciousIpBL> listMaliciousIp;
List<MaliciousIpBL> listBlackListIp;
String yoho_ip = YOHO_INTERNAL_IP;
listMaliciousIp = (List<MaliciousIpBL>) JSON.parseArray(ips_json, MaliciousIpBL.class);
if(CollectionUtils.isNotEmpty(listMaliciousIp)){
listBlackListIp = new ArrayList<>();
for (MaliciousIpBL maliciousIp : listMaliciousIp) {
//过滤白名单
if(StringUtils.isNotBlank(yoho_ip) && StringUtils.isNoneBlank(maliciousIp.getIp())){
if(IpWhiteListUtil.getIpWhiteList().contains(maliciousIp.getIp())){
continue;
}
//过滤公司内部ip段
String []arrIp = StringUtils.split(maliciousIp.getIp(),'.');
String inIp = arrIp[0] + "." + arrIp[1];
if(yoho_ip.indexOf(inIp) >= 0){
continue;
if(IpWhiteListUtil.getIpWhiteList().contains(maliciousIp.getIp())){
continue;
}
//过滤公司内部ip段
String []arrIp = StringUtils.split(maliciousIp.getIp(),'.');
String inIp = arrIp[0] + "." + arrIp[1];
boolean isIn = false;
String []whiteIp = StringUtils.split(YOHO_INTERNAL_IP,'|');
for (String white : whiteIp) {
if(white.equals(inIp)){
isIn = true;
break;
}
}
if(isIn){
continue;
}
//敏感接口比例大于等于99%，且敏感接口QPS大于等于100的值，暂时默认放到黑名单
if((maliciousIp.getImpApiPrecent() >= PERCENT_MAX) && (maliciousIp.getImpCount() >= COUNT_MAX)){
... ...
... ... @@ -82,7 +82,6 @@ public class MaliciousIpServiceImpl implements MaliciousIpService {
if(StringUtils.isBlank(ips_json)){
return response;
}
String yoho_ip = YOHO_INTERNAL_IP;
listMaliciousIp = (List<MaliciousIp>) JSON.parseArray(ips_json, MaliciousIp.class);
List<MaliciousIp> newlistMaliciousIp = new ArrayList<MaliciousIp>();
if(CollectionUtils.isNotEmpty(listMaliciousIp)){
... ... @@ -90,17 +89,23 @@ public class MaliciousIpServiceImpl implements MaliciousIpService {
Date date;
for (MaliciousIp maliciousIp : listMaliciousIp) {
//过滤白名单
if(StringUtils.isNotBlank(yoho_ip) && StringUtils.isNoneBlank(maliciousIp.getIp())){
if(IpWhiteListUtil.getIpWhiteList().contains(maliciousIp.getIp())){
continue;
}
//过滤公司内部ip段
String []arrIp = StringUtils.split(maliciousIp.getIp(),'.');
String inIp = arrIp[0] + "." + arrIp[1];
if(yoho_ip.indexOf(inIp) >= 0){
continue;
if(IpWhiteListUtil.getIpWhiteList().contains(maliciousIp.getIp())){
continue;
}
//过滤公司内部ip段
String []arrIp = StringUtils.split(maliciousIp.getIp(),'.');
String inIp = arrIp[0] + "." + arrIp[1];
boolean isIn = false;
String []whiteIp = StringUtils.split(YOHO_INTERNAL_IP,'|');
for (String white : whiteIp) {
if(white.equals(inIp)){
isIn = true;
break;
}
}
if(isIn){
continue;
}
if(StringUtils.isBlank(maliciousIp.getTimestamp())){
continue;
}
... ... @@ -295,20 +300,31 @@ public class MaliciousIpServiceImpl implements MaliciousIpService {
if(StringUtils.isBlank(ips_json)){
return response;
}
String yoho_ip = configReader.getString(YOHO_WHITE_LIST_IP, YOHO_INTERNAL_IP);
listMaliciousIp = (List<MaliciousIpBL>) JSON.parseArray(ips_json, MaliciousIpBL.class);
List<MaliciousIpBL> newlistMaliciousIp = new ArrayList<>();
if(CollectionUtils.isNotEmpty(listMaliciousIp)){
SimpleDateFormat simpleDateFormat = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss");
Date date;
for (MaliciousIpBL maliciousIp : listMaliciousIp) {
if(StringUtils.isNotBlank(yoho_ip) && StringUtils.isNoneBlank(maliciousIp.getIp())){
String []arrIp = StringUtils.split(maliciousIp.getIp(),'.');
String inIp = arrIp[0] + "." + arrIp[1];
if(yoho_ip.indexOf(inIp) >= 0){
continue;
//过滤白名单
if(IpWhiteListUtil.getIpWhiteList().contains(maliciousIp.getIp())){
continue;
}
//过滤公司内部ip段
String []arrIp = StringUtils.split(maliciousIp.getIp(),'.');
String inIp = arrIp[0] + "." + arrIp[1];
boolean isIn = false;
String []whiteIp = StringUtils.split(YOHO_INTERNAL_IP,'|');
for (String white : whiteIp) {
if(white.equals(inIp)){
isIn = true;
break;
}
}
if(isIn){
continue;
}
if(StringUtils.isBlank(maliciousIp.getTimestamp())){
continue;
}
... ... @@ -342,20 +358,30 @@ public class MaliciousIpServiceImpl implements MaliciousIpService {
if(StringUtils.isBlank(ips_json)){
return response;
}
String yoho_ip = configReader.getString(YOHO_WHITE_LIST_IP, YOHO_INTERNAL_IP);
listMaliciousIp = (List<MaliciousIpCS>) JSON.parseArray(ips_json, MaliciousIpCS.class);
List<MaliciousIpCS> newlistMaliciousIp = new ArrayList<>();
if(CollectionUtils.isNotEmpty(listMaliciousIp)){
SimpleDateFormat simpleDateFormat = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss");
Date date;
for (MaliciousIpCS maliciousIp : listMaliciousIp) {
if(StringUtils.isNotBlank(yoho_ip) && StringUtils.isNoneBlank(maliciousIp.getIp())){
String []arrIp = StringUtils.split(maliciousIp.getIp(),'.');
String inIp = arrIp[0] + "." + arrIp[1];
if(yoho_ip.indexOf(inIp) >= 0){
continue;
//过滤白名单
if(IpWhiteListUtil.getIpWhiteList().contains(maliciousIp.getIp())){
continue;
}
//过滤公司内部ip段
String []arrIp = StringUtils.split(maliciousIp.getIp(),'.');
String inIp = arrIp[0] + "." + arrIp[1];
boolean isIn = false;
String []whiteIp = StringUtils.split(YOHO_INTERNAL_IP,'|');
for (String white : whiteIp) {
if(white.equals(inIp)){
isIn = true;
break;
}
}
if(isIn){
continue;
}
if(StringUtils.isBlank(maliciousIp.getTimestamp())){
continue;
}
... ...
package com.monitor.other.maliciousip.service.impl;
import org.apache.commons.lang3.StringUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.beans.factory.annotation.Value;
import org.springframework.stereotype.Service;
import org.springframework.web.client.RestTemplate;
/**
* 恶意ip to Uic
*
* @author hui.xu
*
*/
@Service
public class MaliciousIpToUic {
/**
* 日志接口
*/
private Logger logger = LoggerFactory.getLogger(getClass());
/**
* 黑名单失效时间24小时
*/
private static final int MALICIOUS_IP_EXPIRE = 24 * 60 * 60;
/**
* 添加恶意ip
*/
@Value("${add.MaliciousIp}")
private String UIC_URL_ADD;
/**
* 溢出恶意ip
*/
@Value("${remove.MaliciousIp}")
private String UIC_URL_REMOVE;
@Autowired
private RestTemplate restTemplate;
/**
* 添加恶意ip
*
* @param ip
*/
public void addMaliciousIp(String ip) {
try {
if(StringUtils.isBlank(ip)){
return;
}
String url = String.format(UIC_URL_ADD + "?ip=%s&expiretime=%s", ip, MALICIOUS_IP_EXPIRE);
restTemplate.getForEntity(url, null);
} catch (Exception e) {
logger.error(String.format(" - MaliciousIpToUic - addMaliciousIp:{} - error", ip), e);
}
}
/**
* 溢出恶意
*
* @param ip
*/
public void removeMaliciousIp(String ip) {
try {
if(StringUtils.isBlank(ip)){
return;
}
String url = String.format(UIC_URL_REMOVE + "?ip=%s", ip);
restTemplate.getForEntity(url, null);
} catch (Exception e) {
logger.error(String.format(" - MaliciousIpToUic - removeMaliciousIp:{} - error", ip), e);
}
}
}
\ No newline at end of file
... ...
... ... @@ -17,16 +17,10 @@ public class IpWhiteListUtil {
Ip_WhiteList.add("106.38.38.144");
Ip_WhiteList.add("106.38.38.28");
Ip_WhiteList.add("218.94.75.50");
Ip_WhiteList.add("218.94.75.32");
Ip_WhiteList.add("218.94.75.58");
Ip_WhiteList.add("218.94.75.32");
Ip_WhiteList.add("54.223.94.23");
Ip_WhiteList.add("54.222.135.182");
Ip_WhiteList.add("54.222.146.59");
Ip_WhiteList.add("123.206.21.19");
Ip_WhiteList.add("123.206.73.107");
Ip_WhiteList.add("123.206.64.25");
Ip_WhiteList.add("123.206.55.43");
Ip_WhiteList.add("139.199.35.21");
Ip_WhiteList.add("139.199.29.44");
Ip_WhiteList.add("54.222.135.182");
... ...
... ... @@ -2,6 +2,8 @@ package com.monitor.other.maliciousip.util;
import java.util.ArrayList;
import java.util.Collection;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
import java.util.concurrent.TimeUnit;
import org.slf4j.Logger;
... ... @@ -10,6 +12,8 @@ import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.beans.factory.annotation.Qualifier;
import org.springframework.data.redis.core.RedisTemplate;
import com.monitor.other.maliciousip.service.impl.MaliciousIpToUic;
/**
* Redis 读 基本操作类
* @author hui.xu
... ... @@ -21,6 +25,9 @@ public class RedisWriteUtil {
*/
private Logger logger = LoggerFactory.getLogger(getClass());
@Autowired
private MaliciousIpToUic maliciousIpToUic;
/**
* Redis String 类型
*/
... ... @@ -47,8 +54,10 @@ public class RedisWriteUtil {
@Qualifier("qq2RedisWriteTemplate")
private RedisTemplate<String, String> qq2Template;
private ExecutorService executorService = Executors.newFixedThreadPool(50);
/**
* 获取所有恶意ip集合
* 写入恶意ip
* @return
*/
public void setIps(String []ips) {
... ... @@ -57,17 +66,29 @@ public class RedisWriteUtil {
return;
}
//此处用于拦截
for (String ip : ips) {
for (String ip : ips) {
setInRedisApi(aws1Template, MALICIOUS_IP + ip);
setInRedisApi(aws2Template, MALICIOUS_IP + ip);
setInRedisApi(qq1Template, MALICIOUS_IP + ip);
setInRedisApi(qq2Template, MALICIOUS_IP + ip);
executorService.execute(new Runnable() {
@Override
public void run() {
//向uic中添加恶意ip
maliciousIpToUic.addMaliciousIp(MALICIOUS_IP + ip);
}
});
}
}catch(Exception e){
logger.error(" - RedisWriteUtil - setIps - error", e);
}
}
/**
* 移除恶意ip
* @param ips
*/
public void delIp(String[] ips){
try{
if((null == ips) || (0 == ips.length)){
... ... @@ -77,6 +98,14 @@ public class RedisWriteUtil {
Collection<String> keys = new ArrayList<String>();
for (int index = 0; index < ips.length; index ++){
keys.add(MALICIOUS_IP + ips[index]);
String ip = ips[index];
executorService.execute(new Runnable() {
@Override
public void run() {
//向uic中添加恶意ip
maliciousIpToUic.removeMaliciousIp(MALICIOUS_IP + ip);
}
});
}
//此处用于拦截
delInRedisApi(aws1Template, keys);
... ...
<?xml version="1.0" encoding="UTF-8"?>
<beans xmlns="http://www.springframework.org/schema/beans"
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xmlns:p="http://www.springframework.org/schema/p"
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xmlns:p="http://www.springframework.org/schema/p"
xmlns:context="http://www.springframework.org/schema/context"
xsi:schemaLocation="http://www.springframework.org/schema/beans
http://www.springframework.org/schema/beans/spring-beans-3.1.xsd">
http://www.springframework.org/schema/beans/spring-beans-3.1.xsd
http://www.springframework.org/schema/context
http://www.springframework.org/schema/context/spring-context-3.1.xsd">
<bean id="dnsConfig" class="com.monitor.other.dns.common.DNSConfig">
<constructor-arg name="url" value="${dns.config.url}" />
<constructor-arg name="dnsApiUrl" value="${dns.config.api.url}" />
... ... @@ -14,7 +17,7 @@
</bean>
<bean id="maliciousIpJob" class="com.monitor.other.maliciousip.job.MaliciousIpJob"
init-method="init">
>
</bean>
<bean id="redisReadUtil" class="com.monitor.other.maliciousip.util.RedisReadUtil" />
... ... @@ -106,4 +109,37 @@
p:keySerializer-ref="stringRedisSerializer" p:valueSerializer-ref="stringRedisSerializer"
p:hashKeySerializer-ref="stringRedisSerializer"
p:hashValueSerializer-ref="stringRedisSerializer" />
<!-- 配置RestTemplate -->
<!--Http client Factory-->
<bean id="httpClientFactory" class="org.springframework.http.client.SimpleClientHttpRequestFactory">
<property name="connectTimeout" value="${connectTimeout}" />
<property name="readTimeout" value="${readTimeout}" />
</bean>
<!--RestTemplate-->
<bean id="restTemplate" class="org.springframework.web.client.RestTemplate">
<constructor-arg ref="httpClientFactory" />
<property name="messageConverters">
<list>
<bean class="org.springframework.http.converter.ByteArrayHttpMessageConverter" />
<bean class="org.springframework.http.converter.StringHttpMessageConverter">
<property name="supportedMediaTypes">
<list>
<value>text/plain;charset=UTF-8</value>
<value>text/html;charset=UTF-8</value>
<value>text/xml;charset=GBK</value>
<value>application/json;charset=UTF-8</value>
</list>
</property>
</bean>
<bean class="org.springframework.http.converter.ResourceHttpMessageConverter" />
<bean class="org.springframework.http.converter.FormHttpMessageConverter" />
<bean class="com.alibaba.fastjson.support.spring.FastJsonHttpMessageConverter">
<property name="supportedMediaTypes" value="application/json;charset=UTF-8" />
</bean>
</list>
</property>
</bean>
<!-- 配置RestTemplate -->
</beans>
\ No newline at end of file
... ...
... ... @@ -26,4 +26,12 @@ malicious.ip.redis.write.qq.host1=10.66.0.3
malicious.ip.redis.write.qq.host2=10.66.0.2
malicious.ip.redis.write.aws.host1=172.31.20.188
malicious.ip.redis.write.aws.host2=172.31.20.187
malicious.ip.redis.write.port=6379
\ No newline at end of file
malicious.ip.redis.write.port=6379
#restTemplate
connectTimeout=5000
readTimeout=10000
#uic url
add.MaliciousIp=http://uic.yohoops.org/uic/addMaliciousIp
remove.MaliciousIp=http://uic.yohoops.org/uic/removeMaliciousIp
\ No newline at end of file
... ...
... ... @@ -28,4 +28,12 @@ malicious.ip.redis.write.qq.host1=192.168.102.22
malicious.ip.redis.write.qq.host2=192.168.102.22
malicious.ip.redis.write.aws.host1=192.168.102.22
malicious.ip.redis.write.aws.host2=192.168.102.22
malicious.ip.redis.write.port=6379
\ No newline at end of file
malicious.ip.redis.write.port=6379
#restTemplate
connectTimeout=5000
readTimeout=10000
#uic url
add.MaliciousIp=http://172.16.6.237:8096/uic/addMaliciousIp
remove.MaliciousIp=http://172.16.6.237:8096/uic/removeMaliciousIp
\ No newline at end of file
... ...
datasources:
yh_ops:
servers:
- 172.16.6.243:3306
- 172.16.6.243:3306
- 172.16.6.61:3306
- 172.16.6.61:3306
username: root
password: t5/oMgwUCmO/GeMHBAQ2Cg==
... ...
#---------jdbc config----------
local.jdbc.url=jdbc:mysql://172.16.6.243:3306/yh_ops?characterEncoding=utf-8&autoReconnect=true&zeroDateTimeBehavior=convertToNull
local.jdbc.url=jdbc:mysql://172.16.6.61:3306/yh_ops?characterEncoding=utf-8&autoReconnect=true&zeroDateTimeBehavior=convertToNull
local.jdbc.user=root
local.jdbc.password=123456
#---------jdbc config----------
\ No newline at end of file
... ...