ops / monitor-service · Commits

Go to a project

Authored by qinchao
Commit a17aa8d01a3ebb9b84fecc7dec3b40f9f404b3e2 1 parent 0abc0c93

切换增加提示

Inline Side-by-side
Showing 1 changed file with 268 additions and 100 deletions
... ... @@ -40,10 +40,12 @@ public class NginxSwitchCtrl {
private Logger logger = LoggerFactory.getLogger("switchLogger");
private long sleepTime=10*1000L;
private long sleepTime_5s=5*1000L;
private AtomicInteger valueCount=new AtomicInteger(1);
private Map<String,BaseResponse> switchLbResponseMap=new ConcurrentHashMap<>();
private Map<String,List<String>> switchLbLogMap=new ConcurrentHashMap<>();
@Value("${system.envi}")
private String env;
... ... @@ -59,13 +61,40 @@ public class NginxSwitchCtrl {
public BaseResponse getLbSwitchResult(String rtnID) {
Set<String> nowIDset=switchLbResponseMap.keySet();
List<String> ids=Lists.newArrayList(rtnID.split(","));
Map<String,String> map=new HashMap<>();
if(nowIDset.containsAll(ids)){
/*for(String id:ids){
switchLbResponseMap.remove(id);
}*/
return new BaseResponse();
boolean allSuccess=true;
for(String id:ids){
BaseResponse resultBaseResponse=switchLbResponseMap.get(id);
if(200!=resultBaseResponse.getCode()){
allSuccess = false;
}
String str=resultBaseResponse.getMessage();
if(StringUtils.isBlank(str)){
str = StringUtils.join(switchLbLogMap.get(id),"<br/>");
}
map.put(id,str);
}
BaseResponse rtn= new BaseResponse();
rtn.setData(map);
if(!allSuccess){
rtn.setMessage("lb switch error");
}
return rtn;
}else{
for(String id:ids){
map.put(id,StringUtils.join(switchLbLogMap.get(id),"<br/>"));
}
BaseResponse rtn= new BaseResponse(201,"not finished");
rtn.setData(map);
return rtn;
}
return new BaseResponse(201,"not finished");
}
/**
* 切换lb的权重
... ... @@ -97,13 +126,15 @@ public class NginxSwitchCtrl {
String ts=String.valueOf(new Date().getTime());
StringBuilder rtnIDsb=new StringBuilder();
for(LbModel lb:lbList){
String rtnID=ts+"-"+valueCount.getAndIncrement();
String tmpID=ts+"-"+valueCount.getAndIncrement();
//beginSwitchLb(rtnID,targetMap,lb,innerListenerMap,outerListenerMap);
startRunAsySwitchLb(tmpID,targetMap,lb,innerListenerMap,outerListenerMap);
if(rtnIDsb.length()>0){
rtnIDsb.append(",");
}
rtnIDsb.append(rtnID);
//beginSwitchLb(rtnID,targetMap,lb,innerListenerMap,outerListenerMap);
startRunAsySwitchLb(rtnID,targetMap,lb,innerListenerMap,outerListenerMap);
rtnIDsb.append(tmpID);
}
BaseResponse response= new BaseResponse();
response.setData(rtnIDsb.toString());
... ... @@ -112,16 +143,17 @@ public class NginxSwitchCtrl {
private BaseResponse beginSwitchLb(String rtnID,Map<String,Integer> targetMap,LbModel lb,Map<String,List<String>> innerListenerMap,Map<String,List<String>> outerListenerMap){
logger.info(" beginSwitchLb lb {} ,rtnID {}",lb,rtnID);
switchLbLogMap.put(rtnID,new ArrayList<>());
BaseResponse rtnBasResponse=null;
try{
if(lb.isClbFlag()){
rtnBasResponse= switchClb(targetMap,lb);
rtnBasResponse= switchClb( rtnID,targetMap,lb);
}else {
Map<String, List<String>> listenerMap = outerListenerMap;
if (lb.isInnerFlag()) {
listenerMap = innerListenerMap;
}
rtnBasResponse= switchAlb(listenerMap,targetMap,lb);
rtnBasResponse= switchAlb(rtnID,listenerMap,targetMap,lb);
}
}catch (Exception e){
logger.error("beginSwitchLb error {}",e);
... ... @@ -149,133 +181,209 @@ public class NginxSwitchCtrl {
*
* 切换alb
*/
private BaseResponse switchAlb(Map<String, List<String>> listenerMap,Map<String,Integer> targetMap,LbModel lb){
QcloudForwardLbModel forwardLbModel = getAlbFromQcloud(lb);
if (forwardLbModel == null) {
return new BaseResponse(201, "获取alb信息出现异常，切换失败:" + lb.getKey());
}
List<AlbModelClass> dealAlb = new ArrayList<>();
for (String protoTypeAndPort : listenerMap.keySet()) {
AlbModelClass albModelClass = new AlbModelClass();
String[] protoAndPortArray = protoTypeAndPort.split(",");
String protocolType = protoAndPortArray[0];
int port = Integer.parseInt(protoAndPortArray[1]);
List<String> domainList = listenerMap.get(protoTypeAndPort);
List<String> locationIdList = new ArrayList<>();
for (QcloudForwardLbModelData data : forwardLbModel.getData()) {
if (StringUtils.equals(protocolType, data.getProtocolType()) && port == data.getLoadBalancerPort()) {
String listenerId = data.getListenerId();
albModelClass.setListenerId(listenerId);
for (QcloudForwardLbModelDataRule rule : data.getRules()) {
if (domainList.contains(rule.getDomain())) {
locationIdList.add(rule.getLocationId());
if(domainList.get(0).equals(rule.getDomain())){
List<List<QcloudLbModelBackend>> ls = getNeedChangedWei(rule.getBackends(),targetMap);
List<QcloudLbModelBackend> needAddWeight=ls.get(0);
List<QcloudLbModelBackend> needRemoveWeight=ls.get(1);
albModelClass.setNeedAddWeight(needAddWeight);
albModelClass.setNeedRemoveWeight(needRemoveWeight);
}
private BaseResponse switchAlb(String rtnID,Map<String, List<String>> listenerMap,Map<String,Integer> targetMap,LbModel lb){
QcloudForwardLbModel forwardLbModel = getAlbFromQcloud(lb);
if (forwardLbModel == null) {
return new BaseResponse(201, "获取alb信息出现异常，切换失败:" + lb.getKey());
}
}
List<AlbModelClass> dealAlb = new ArrayList<>();
for (String protoTypeAndPort : listenerMap.keySet()) {
AlbModelClass albModelClass = new AlbModelClass();
String[] protoAndPortArray = protoTypeAndPort.split(",");
String protocolType = protoAndPortArray[0];
int port = Integer.parseInt(protoAndPortArray[1]);
List<String> domainList = listenerMap.get(protoTypeAndPort);
List<String> locationIdList = new ArrayList<>();
List<String> domainList_operate = new ArrayList<>();
for (QcloudForwardLbModelData data : forwardLbModel.getData()) {
if (StringUtils.equals(protocolType, data.getProtocolType()) && port == data.getLoadBalancerPort()) {
String listenerId = data.getListenerId();
albModelClass.setListenerId(listenerId);
for (QcloudForwardLbModelDataRule rule : data.getRules()) {
if (domainList.contains(rule.getDomain())) {
locationIdList.add(rule.getLocationId());
domainList_operate.add(rule.getDomain());
if(domainList.get(0).equals(rule.getDomain())){
List<List<QcloudLbModelBackend>> ls = getNeedChangedWei(rule.getBackends(),targetMap);
List<QcloudLbModelBackend> needAddWeight=ls.get(0);
List<QcloudLbModelBackend> needRemoveWeight=ls.get(1);
albModelClass.setNeedAddWeight(needAddWeight);
albModelClass.setNeedRemoveWeight(needRemoveWeight);
}
albModelClass.setLocationIdList(locationIdList);
dealAlb.add(albModelClass);
}
}
albModelClass.setLocationIdList(locationIdList);
albModelClass.setDomainList(domainList_operate);
dealAlb.add(albModelClass);
}
}
}
logger.info(" switchAlb with {}",dealAlb);
dealLbLog(rtnID,"开始处理应用型lb "+lb.getKey());
dealLbLog(rtnID,"增加权重处理 ");
for (AlbModelClass albModelClass : dealAlb) {
//操作weight，
List<QcloudLbModelBackend> needAddWeight = albModelClass.getNeedAddWeight();
dealLbLog(rtnID,"**********************************");
dealLbLog(rtnID,"处理 domain "+StringUtils.join(albModelClass.getDomainList(),","));
dealLbLog(rtnID,"需要增加权重的机器数量 "+needAddWeight.size());
if (needAddWeight.size() > 0) {
List<String> unInstanceIdList = new ArrayList<>();
List<Integer> weightList = new ArrayList<>();
List<Integer> portList = new ArrayList<>();
for (QcloudLbModelBackend backend : needAddWeight) {
dealLbLog(rtnID,"需要增加权重的机器 "+backend.getInstanceName()+",uid="+backend.getUnInstanceId()+",new weight is "+backend.getUpdateWeight() +",port is "+backend.getPort());
unInstanceIdList.add(backend.getUnInstanceId());
weightList.add(backend.getUpdateWeight());
portList.add(backend.getPort());
}
logger.info(" switchAlb with {}",dealAlb);
for (AlbModelClass albModelClass : dealAlb) {
//操作weight，
List<QcloudLbModelBackend> needAddWeight = albModelClass.getNeedAddWeight();
if (needAddWeight.size() > 0) {
List<String> unInstanceIdList = new ArrayList<>();
List<Integer> weightList = new ArrayList<>();
List<Integer> portList = new ArrayList<>();
for (QcloudLbModelBackend backend : needAddWeight) {
unInstanceIdList.add(backend.getUnInstanceId());
weightList.add(backend.getUpdateWeight());
portList.add(backend.getPort());
}
//alb 切换权重
QcloudSdkUtil.modifyForwardSeventhBackends(lb.getLbId(), albModelClass.getListenerId(), albModelClass.getLocationIdList(), unInstanceIdList, portList, weightList);
//alb 切换权重 加权重
boolean result= false;
//调用三次，直到第一次成功
for(int j=0;j<3;j++){
result = QcloudSdkUtil.modifyForwardSeventhBackends(lb.getLbId(), albModelClass.getListenerId(), albModelClass.getLocationIdList(), unInstanceIdList, portList, weightList);
if(result){
break;
}else{
try {
// 括号内的参数是毫秒值,线程休眠
Thread.sleep(sleepTime);//10s
Thread.sleep(sleepTime_5s);
} catch (InterruptedException e) {
e.printStackTrace();
}
}
}
boolean recheckFlag = false;
for (AlbModelClass albModelClass : dealAlb) {
List<QcloudLbModelBackend> needRemoveWeight = albModelClass.getNeedRemoveWeight();
if (needRemoveWeight.size() > 0) {
recheckFlag = true;
break;
}
if(result){
dealLbLog(rtnID,"增加权重 调用完成，等待腾讯云处理");
}else{
dealLbLog(rtnID,"增加权重 调用腾讯云接口失败，切换终止！");
return new BaseResponse(201,"");
}
if (recheckFlag) {
//切掉流量之前需要校验
QcloudForwardLbModel checkBeforeMoveBackends = getAlbFromQcloud(lb);
boolean checkPass = false;
for (QcloudForwardLbModelData flbModel : checkBeforeMoveBackends.getData()) {
for (AlbModelClass albModelClass : dealAlb) {
if (StringUtils.equals(flbModel.getListenerId(), albModelClass.getListenerId())) {
List<String> unInstanceIdList = new ArrayList<>();
List<Integer> weightList = new ArrayList<>();
List<Integer> portList = new ArrayList<>();
for (QcloudLbModelBackend backend : albModelClass.getNeedRemoveWeight()) {
unInstanceIdList.add(backend.getUnInstanceId());
weightList.add(0);
portList.add(backend.getPort());
}
try {
// 括号内的参数是毫秒值,线程休眠
Thread.sleep(sleepTime);//10s
} catch (InterruptedException e) {
e.printStackTrace();
}
}
dealLbLog(rtnID,"本次domain增加权重完成 ");
}
dealLbLog(rtnID,"增加权重处理成功 ");
boolean recheckFlag = false;
for (AlbModelClass albModelClass : dealAlb) {
List<QcloudLbModelBackend> needRemoveWeight = albModelClass.getNeedRemoveWeight();
if (needRemoveWeight.size() > 0) {
recheckFlag = true;
break;
}
}
dealLbLog(rtnID,"减少权重处理 ");
if (recheckFlag) {
//切掉流量之前需要校验
QcloudForwardLbModel checkBeforeMoveBackends = getAlbFromQcloud(lb);
if(checkBeforeMoveBackends==null){
dealLbLog(rtnID,"校验lb除了减少权重机器外，至少存在一个nginx失败，获取lb错误，切换终止！");
return new BaseResponse(201,"");
}
boolean checkPass = false;
for (QcloudForwardLbModelData flbModel : checkBeforeMoveBackends.getData()) {
for (QcloudForwardLbModelDataRule rule : flbModel.getRules()) {
if (albModelClass.getLocationIdList().contains(rule.getLocationId())) {
for (QcloudLbModelBackend backend : rule.getBackends()) {
if (backend.getWeight() > 0 && !unInstanceIdList.contains(backend.getUnInstanceId())) {
checkPass = true;
break;
}
}
for (AlbModelClass albModelClass : dealAlb) {
if (StringUtils.equals(flbModel.getListenerId(), albModelClass.getListenerId())) {
List<String> instanceNameList = new ArrayList<>();
List<String> unInstanceIdList = new ArrayList<>();
List<Integer> weightList = new ArrayList<>();
List<Integer> portList = new ArrayList<>();
for (QcloudLbModelBackend backend : albModelClass.getNeedRemoveWeight()) {
instanceNameList.add(backend.getInstanceName());
unInstanceIdList.add(backend.getUnInstanceId());
weightList.add(0);
portList.add(backend.getPort());
}
dealLbLog(rtnID,"**********************************");
dealLbLog(rtnID,"处理 domain "+StringUtils.join(albModelClass.getDomainList(),","));
dealLbLog(rtnID,"需要减少权重 "+StringUtils.join(instanceNameList,","));
for (QcloudForwardLbModelDataRule rule : flbModel.getRules()) {
if (albModelClass.getLocationIdList().contains(rule.getLocationId())) {
for (QcloudLbModelBackend backend : rule.getBackends()) {
if (backend.getWeight() > 0 && !unInstanceIdList.contains(backend.getUnInstanceId())) {
checkPass = true;
break;
}
}
if (checkPass) {
//alb 切换权重
QcloudSdkUtil.modifyForwardSeventhBackends(lb.getLbId(), albModelClass.getListenerId(), albModelClass.getLocationIdList(), unInstanceIdList, portList, weightList);
}
}
if (checkPass) {
//alb 切换权重
boolean result=false;
//调用三次，直到第一次成功
for(int j=0;j<4;j++){
result = QcloudSdkUtil.modifyForwardSeventhBackends(lb.getLbId(), albModelClass.getListenerId(), albModelClass.getLocationIdList(), unInstanceIdList, portList, weightList);
if(result){
break;
}else{
try {
// 括号内的参数是毫秒值,线程休眠
Thread.sleep(sleepTime*2);//10s
Thread.sleep(sleepTime_5s);
} catch (InterruptedException e) {
e.printStackTrace();
}
}
}
}
if(result){
dealLbLog(rtnID,"减少权重 调用完成，等待腾讯云处理");
}else{
dealLbLog(rtnID,"减少权重 调用腾讯云接口失败，切换终止！");
return new BaseResponse(201,"");
}
try {
// 括号内的参数是毫秒值,线程休眠
Thread.sleep(sleepTime*2);//10s
} catch (InterruptedException e) {
e.printStackTrace();
}
}else{
dealLbLog(rtnID,"校验lb除了减少权重机器外，至少存在一个nginx失败，找不到nginx，切换终止！");
return new BaseResponse(201,"");
}
}
}
}
}
dealLbLog(rtnID,"减少权重处理完成 ");
dealLbLog(rtnID,"切换成功："+lb.getKey());
return new BaseResponse();
}
private BaseResponse switchClb(Map<String,Integer> targetMap,LbModel lb){
private void dealLbLog(String rtnID,String msg){
switchLbLogMap.get(rtnID).add(msg);
}
private BaseResponse switchClb(String rtnID,Map<String,Integer> targetMap,LbModel lb){
QcloudCLBModel clbModel=getCLbFromQcloud(lb);
if(clbModel==null){
... ... @@ -287,16 +395,40 @@ public class NginxSwitchCtrl {
List<QcloudLbModelBackend> needRemoveWeight=ls.get(1);
//操作weight，
dealLbLog(rtnID,"开始处理lb "+lb.getKey());
dealLbLog(rtnID,"需要增加权重的机器数量是 "+needAddWeight.size());
if(needAddWeight.size()>0){
List<String> unInstanceIdList=new ArrayList<>();
List<Integer> weightList=new ArrayList<>();
for(QcloudLbModelBackend backend:needAddWeight){
dealLbLog(rtnID,"需要增加权重机器信息 "+backend.getInstanceName()+",uninstanceid="+backend.getUnInstanceId()+",new weight is "+backend.getUpdateWeight());
unInstanceIdList.add(backend.getUnInstanceId());
weightList.add(backend.getUpdateWeight());
}
//clb 切换权重
QcloudSdkUtil.modifyLoadBalancerBackends(lb.getLbId(),unInstanceIdList,weightList);
boolean result=false;
for(int j=0;j<3;j++){
result = QcloudSdkUtil.modifyLoadBalancerBackends(lb.getLbId(),unInstanceIdList,weightList);
if(result){
break;
}else{
try {
// 括号内的参数是毫秒值,线程休眠
Thread.sleep(sleepTime_5s);
} catch (InterruptedException e) {
e.printStackTrace();
}
}
}
if(result){
dealLbLog(rtnID,"增加权重 调用完成，等待腾讯云处理");
}else{
dealLbLog(rtnID,"增加权重 调用腾讯云接口失败，切换终止！");
return new BaseResponse(201,"");
}
try {
// 括号内的参数是毫秒值,线程休眠
... ... @@ -305,26 +437,56 @@ public class NginxSwitchCtrl {
e.printStackTrace();
}
}
dealLbLog(rtnID,"增加权重调用完成");
//移除之前一定检测一次,保证移除后，至少还有另外一个
dealLbLog(rtnID,"需要减少权重的机器数量是 "+needRemoveWeight.size());
if(needRemoveWeight.size()>0){
List<String> unInstanceIdList=new ArrayList<>();
List<Integer> weightList=new ArrayList<>();
for(QcloudLbModelBackend backend:needRemoveWeight){
dealLbLog(rtnID,"需要减少权重机器信息 "+backend.getInstanceName()+",uninstanceid="+backend.getUnInstanceId()+",new weight is "+backend.getUpdateWeight());
unInstanceIdList.add(backend.getUnInstanceId());
weightList.add(0);
}
QcloudCLBModel checkBeforeMoveBackends=getCLbFromQcloud(lb);
if(checkBeforeMoveBackends==null){
dealLbLog(rtnID,"校验lb除了减少权重机器外，至少存在一个nginx失败，获取lb错误，切换终止！");
return new BaseResponse(201,"");
}
boolean checkPass=false;
for(QcloudLbModelBackend backend:checkBeforeMoveBackends.getBackendSet()){
if(backend.getWeight()>0&&!unInstanceIdList.contains(backend.getUnInstanceId())){
dealLbLog(rtnID,"校验lb除了减少权重机器外，至少存在一个nginx通过："+backend.getInstanceName());
checkPass=true;
break;
}
}
if(checkPass){
//clb 切换权重
QcloudSdkUtil.modifyLoadBalancerBackends(lb.getLbId(),unInstanceIdList,weightList);
boolean result=false;
for(int j=0;j<4;j++){
result = QcloudSdkUtil.modifyLoadBalancerBackends(lb.getLbId(),unInstanceIdList,weightList);
if(result){
break;
}else{
try {
// 括号内的参数是毫秒值,线程休眠
Thread.sleep(sleepTime_5s);
} catch (InterruptedException e) {
e.printStackTrace();
}
}
}
if(result){
dealLbLog(rtnID,"减少权重 调用完成，等待腾讯云处理");
}else{
dealLbLog(rtnID,"减少权重 调用腾讯云接口失败，切换终止！");
return new BaseResponse(201,"");
}
try {
// 括号内的参数是毫秒值,线程休眠
... ... @@ -332,8 +494,13 @@ public class NginxSwitchCtrl {
} catch (InterruptedException e) {
e.printStackTrace();
}
}else{
dealLbLog(rtnID,"校验lb除了减少权重机器外，至少存在一个nginx失败，无法找到另外nginx ，切换终止！");
return new BaseResponse(201,"");
}
}
dealLbLog(rtnID,"减少权重调用完成");
dealLbLog(rtnID,"切换成功："+lb.getKey());
return new BaseResponse();
}
... ... @@ -376,6 +543,7 @@ public class NginxSwitchCtrl {
private class AlbModelClass{
String listenerId;
List<String> locationIdList;
List<String> domainList;
List<QcloudLbModelBackend> needAddWeight;
List<QcloudLbModelBackend> needRemoveWeight;
}
... ...