AutoScalingCtrl.java 30.4 KB
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 189 190 191 192 193 194 195 196 197 198 199 200 201 202 203 204 205 206 207 208 209 210 211 212 213 214 215 216 217 218 219 220 221 222 223 224 225 226 227 228 229 230 231 232 233 234 235 236 237 238 239 240 241 242 243 244 245 246 247 248 249 250 251 252 253 254 255 256 257 258 259 260 261 262 263 264 265 266 267 268 269 270 271 272 273 274 275 276 277 278 279 280 281 282 283 284 285 286 287 288 289 290 291 292 293 294 295 296 297 298 299 300 301 302 303 304 305 306 307 308 309 310 311 312 313 314 315 316 317 318 319 320 321 322 323 324 325 326 327 328 329 330 331 332 333 334 335 336 337 338 339 340 341 342 343 344 345 346 347 348 349 350 351 352 353 354 355 356 357 358 359 360 361 362 363 364 365 366 367 368 369 370 371 372 373 374 375 376 377 378 379 380 381 382 383 384 385 386 387 388 389 390 391 392 393 394 395 396 397 398 399 400 401 402 403 404 405 406 407 408 409 410 411 412 413 414 415 416 417 418 419 420 421 422 423 424 425 426 427 428 429 430 431 432 433 434 435 436 437 438 439 440 441 442 443 444 445 446 447 448 449 450 451 452 453 454 455 456 457 458 459 460 461 462 463 464 465 466 467 468 469 470 471 472 473 474 475 476 477 478 479 480 481 482 483 484 485 486 487 488 489 490 491 492 493 494 495 496 497 498 499 500 501 502 503 504 505 506 507 508 509 510 511 512 513 514 515 516 517 518 519 520 521 522 523 524 525 526 527 528 529 530 531 532 533 534 535 536 537 538 539 540 541 542 543 544 545 546 547 548 549 550 551 552 553 554 555 556 557 558 559 560 561 562 563 564 565 566 567 568 569 570 571 572 573 574 575 576 577 578 579 580 581 582 583 584 585 586 587 588 589 590 591 592 593 594 595 596 597 598 599 600 601 602 603 604 605 606 607 608 609 610 611 612 613 614 615 616 617 618 619 620 621 622 623 624 625 626 627 628 629 630 631 632 633 634 635 636 637 638 639 640 641 642 643 644 645 646 647 648 649 650 651 652 653 654 655 656 657 658 659 660 661 662 663 664 665 666 667 668 669 670 671 672 673 674 675 676 677 678 679 680 681 682 683 684 685 686 687 688 689 690 691 692 693 694 695 696 697 698 699 700 701 702 703 704 705 706 707 708 709 710 711 712 713 714 715 716 717 718 719 720 721 722 723 724 725 726 727 728 729 730 731 732 733 734 735 736 737
package com.ui.ctrl;

import com.alibaba.fastjson.JSON;
import com.alibaba.fastjson.JSONObject;
import com.ui.cloud.AwsModifyScalingGroupReq;
import com.ui.cloud.CommAutoModifyScalingGroupReq;
import com.ui.cloud.DeployProjectCloud;
import com.ui.cloud.QcloudModifyScalingGroupReq;
import com.ui.cloud.model.AutoScalingGroup;
import com.ui.cloud.task.AutoScalingFactory;
import com.ui.contants.HttpUriContants;
import com.ui.http.HttpRestClient;
import com.ui.model.AutoScalingInfoReq;
import com.ui.model.BaseResponse;
import com.ui.model.domain.BuildMessage;
import com.ui.model.domain.HostInfo;
import com.ui.model.domain.RestartModel;
import com.ui.model.req.BuildRequest;
import com.ui.model.req.User;
import com.ui.project.ProjectEnvironment;
import org.apache.commons.lang.StringUtils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.core.ParameterizedTypeReference;
import org.springframework.stereotype.Controller;
import org.springframework.ui.Model;
import org.springframework.web.bind.annotation.RequestMapping;
import org.springframework.web.bind.annotation.ResponseBody;
import org.springframework.web.servlet.ModelAndView;

import javax.servlet.http.HttpSession;
import java.util.*;

/**
 * 亚马逊云 腾讯云 伸缩组操作
 * Created by xh on 2017/5/9.
 */
@Controller
@RequestMapping("/autoScalingTool")
public class AutoScalingCtrl {
    public static final Logger logger = LoggerFactory.getLogger(AutoScalingCtrl.class);
    @Autowired
    private HttpRestClient httpClient;

    @Autowired
    private AutoScalingFactory autoScalingFactory;

    private String DEPLOY_URL = "http://172.31.16.167:8883/";
    private String NODE_DEPLOY_URL = "http://172.31.16.167:8883/node/";

    private Map<String, BaseResponse> msgMap = new HashMap<>();

    /**
     * 跳转到页面
     * @param model
     * @return
     */
    @RequestMapping("/toautoscalingpage")
    public ModelAndView toAutoScalingPage(Model model) {
        return new ModelAndView("manager/autoscaling");
    }

    /**
     * 显示配置信息
     * @param cloudType 云类型 1:AWS 2:QCLOUD
     * @return
     */
    @RequestMapping("/getAutoScalingGroups")
    @ResponseBody
    public BaseResponse<?> getAutoScalingGroups(int cloudType) {
       return autoScalingFactory.getAutoScalingGroups(cloudType);
    }


    /**
     * 显示配置信息
     * @param cloudType 云类型 1:AWS 2:QCLOUD
     * @return
     */
   /* @RequestMapping("/autoscalingconfig")
    @ResponseBody
    public BaseResponse<?> autoScalingConfig(int cloudType) {
        BaseResponse<?> response = httpClient.defaultPost(
                HttpUriContants.GET_CLOUD_AUTOSCALING_CONFIG + "?cloudType="+cloudType, null,
                BaseResponse.class);
        return response;
    }*/

    /**
     * 显示配置信息
     * @param qCloudReq 修改的相关参数,暂时只用了腾讯云的,日后考虑亚马逊的
     * @return
     */
    @RequestMapping("/modifyScalingGroup")
    @ResponseBody
    public BaseResponse<?> modifyScalingGroup(QcloudModifyScalingGroupReq qCloudReq, int cloudType) {
        CommAutoModifyScalingGroupReq req = new CommAutoModifyScalingGroupReq();
        req.setCloudType(cloudType);
        if(1==cloudType){
            AwsModifyScalingGroupReq awsReq=new AwsModifyScalingGroupReq();
            awsReq.setScalingGroupName(qCloudReq.getAwsScalingGroupName());
            awsReq.setMaxSize(qCloudReq.getMaxSize());
            awsReq.setMinSize(qCloudReq.getMinSize());
            awsReq.setDesiredCapacity(qCloudReq.getDesiredCapacity());
            req.setAwsModifyScalingGroupReq(awsReq);
        }else{
            req.setQCloudParms(qCloudReq);
        }
        BaseResponse<?> response = httpClient.defaultPost(
                HttpUriContants.GET_AUTO_SCALING_MODIFY_SCALING_GROUP, req, BaseResponse.class);
        return response;
    }

    @RequestMapping("/checkHostReachableTest")
    @ResponseBody
    public BaseResponse checkHostReachableTest(String changedIps) {
        String backData=httpClient.post(DEPLOY_URL + "hostDetect/checkSshPortTest?changedIps="+changedIps,null , String.class);
        BaseResponse response=new BaseResponse();
        response.setData(backData);
        return response;
    }

    @RequestMapping("/checkHostReachable")
    @ResponseBody
    public BaseResponse checkHostReachable(String changedIps) {
        String backData=httpClient.post(DEPLOY_URL + "hostDetect/checkSshPort?changedIps="+changedIps, null, String.class);
        BaseResponse response=new BaseResponse();
        response.setData(backData);
        return response;
    }

    /**
     * 停止java类型的项目
     * 如果是缩容,并且如果是java项目的缩容,需要先停止非保护机器(kill相关机器的java项目进程,从zk中移除)
     * 再执行缩容,再拉起
     */
    @RequestMapping("/stopNoneProtectedJavaProcess")
    @ResponseBody
    public BaseResponse<?> stopNoneProtectedJavaProcess(int cloudType ,String groupName,String groupId,String projects) {
        String url = String.format(HttpUriContants.GET_AUTO_SCALING_INSTANCES_IP_PROTECTED_AND_NONE_BY_SCALINGGROUPID+"?cloudType=%d&groupName=%s&groupId=%s",cloudType,groupName,groupId);
        BaseResponse<?> response = httpClient.defaultPost(url, null, BaseResponse.class);
        if(response==null||response.getCode()!=200||response.getData()==null){
            return new BaseResponse(201);
        }
        Map<String,List<String>> map=(Map<String,List<String>>)response.getData();
        List<String> protected_ips=map.get("protected");
        List<String> none_protected_ips=map.get("noneProtected");
        String stopJavaFlag="0";
        StringBuilder rtnProcMsgId=new StringBuilder();
        if(protected_ips!=null&&protected_ips.size()>0){
            //存在受保护的机器,才继续停止非受保护的
            if(none_protected_ips!=null&&none_protected_ips.size()>0){
                for(String stopIp:none_protected_ips){
                    for(String project:projects.split(",")){
                        if(StringUtils.isNotBlank(project)){
                            RestartModel restartModel=new RestartModel();
                            restartModel.setCloud("single");
                            restartModel.setExe("stop");
                            restartModel.setIp(stopIp);
                            restartModel.setProject(project);
                            stopJavaFlag="1";
                            BaseResponse<List<RestartModel>> stopJavaResponse = httpClient.exchangeForpost(HttpUriContants.JAVA_STOP_RESTART, new ParameterizedTypeReference<BaseResponse<List<RestartModel>>>() {
                            }, restartModel);

                            if(stopJavaResponse!=null&&stopJavaResponse.getCode()==200&&stopJavaResponse.getData()!=null){
                                List<RestartModel> restartModelResponseData = stopJavaResponse.getData();
                                if(restartModelResponseData.size()>0){
                                    if(rtnProcMsgId.length()>0){
                                        rtnProcMsgId.append(",");
                                    }
                                    rtnProcMsgId.append(""+restartModelResponseData.get(0).getId());
                                }
                            }
                        }
                    }
                }
            }
        }
        BaseResponse rntBaseResponse= new BaseResponse();
        Map<String,String> rntMap=new HashMap<>();
        rntMap.put("protectedIps",StringUtils.join(protected_ips.toArray(),","));
        rntMap.put("noneProtectedIps",StringUtils.join(none_protected_ips.toArray(),","));
        rntMap.put("rtnProcMsgId",rtnProcMsgId.toString());
        rntMap.put("stopJavaFlag",stopJavaFlag);//java stop flag
        rntBaseResponse.setData(rntMap);
       return rntBaseResponse;
    }


    /**
     * 腾讯云:根据实例id获取内网ip
     * 亚马逊云待定:不清楚流程
     * @param groupId 伸缩组id
     * @return
     */
    @RequestMapping("/getIpsByScalingGroupId")
    @ResponseBody
    public BaseResponse<?> getIpsByScalingGroupId(int cloudType ,String groupName,String groupId){
        String url = String.format(HttpUriContants.GET_AUTO_SCALING_INSTANCES_IP_BY_SCALINGGROUPID+"?cloudType=%d&groupName=%s&groupId=%s",cloudType,groupName,groupId);
        BaseResponse<?> response = httpClient.defaultPost(url, null, BaseResponse.class);
        return response;
    }


    /**
     * 系统发布的group下的所有项目
     */
    @RequestMapping("/getDeployProjectNames")
    @ResponseBody
    public BaseResponse<?> getDeployProjectNames(String environment,String scalingGroupName )
    {
        BaseResponse t=new BaseResponse();
        t.setData(AutoScalingGroupProjectConstant.getProjectNames(environment, scalingGroupName));
        return t;
    }

    /**
     * 系统发布
     * @param session
     * @param projectName 项目名称
     * @param ips 所需发布的ip集合 ,分割
     * @param branch 分支名称
     * @param environment 发布环境 此为qcloud
     * @return
     */
    @RequestMapping("/deployCloudProject")
    @ResponseBody
    public BaseResponse<?> deployCloudProject(HttpSession session, String projectName, String ips, String branch, String environment){
        return deployProject(session, projectName, ips, branch, environment,ProjectEnvironment.getUrl(environment));
    }

    /**
     * node系统发布
     * @param session
     * @param projectName 项目名称
     * @param ips 所需发布的ip集合 ,分割
     * @param branch 分支名称
     * @param environment 发布环境 此为qcloud
     * @return
     */
    @RequestMapping("/deployCloudProjectForNode")
    @ResponseBody
    public BaseResponse<?> deployCloudProjectForNode(HttpSession session, String projectName, String ips, String branch, String environment){
        return deployProject(session, projectName, ips, branch, environment,NODE_DEPLOY_URL);
    }

    private BaseResponse<?> deployProject(HttpSession session, String projectName, String ips, String branch, String environment,String url){
        BaseResponse response = null;

        try{
            User user = (User) session.getAttribute("user");
            List<BuildMessage> list = new ArrayList<>();
            BuildRequest buildRequest = new BuildRequest();
            //分支名称
            buildRequest.setBranch(branch);
            //登陆用户
            buildRequest.setUser(user.getName());
            //设置需要发布的ip
            buildRequest.setHost(ips);
            //发布环境
            buildRequest.setEnvironment(environment);
            //不存在回滚文件
            //操作类型
            buildRequest.setOperate("Deploy");
            //关联工单
            buildRequest.setWorkid("release0");
            //项目名称
            buildRequest.setProject(AutoScalingGroupProjectConstant.getProjectNames(environment, projectName));

            String messageids = "";
            BuildMessage buildMessage = httpClient.post(url + "build", buildRequest, BuildMessage.class);
            if (buildMessage != null) {
                list.add(buildMessage);
                messageids =  buildMessage.getId() + ",";
            }
            DeployProjectCloud deploy = new DeployProjectCloud();
            deploy.setMessageids(messageids);
            deploy.setMessageList(list);
            response = new BaseResponse();
            response.setData(deploy);
        }catch (Exception e){
            response.setCode(300);
            response.setMessage(e.getMessage());
        }

        return response;
    }


    /**
     * 获取执行结果
     *
     * @param messageid  qcloud_1233 或者 aws_1234
     * @return
     */
    @RequestMapping(value = "/getBuildMsgAfterDeploy")
    @ResponseBody
    public BaseResponse getBuildMsgAfterDeploy(String messageid, String project) {
        return getBuildMsg(messageid,project,ProjectEnvironment.getUrl(messageid.split("_")[0]));
    }

    @RequestMapping(value = "/getBuildMsgAfterDeployForNode")
    @ResponseBody
    public BaseResponse getBuildMsgAfterDeployForNode(String messageid, String project) {
        return getBuildMsg(messageid,project,NODE_DEPLOY_URL);
    }

    private BaseResponse getBuildMsg(String messageid, String project,String url) {
        try {
            BaseResponse response=new BaseResponse();
            String cloudType=messageid.split("_")[0];
            String[] projectArray= AutoScalingGroupProjectConstant.getProjectNames(cloudType,project).split(",");
            StringBuilder rtnMsg=new StringBuilder();
            Boolean allOK=true;
            String desc="";
            for(String pro:projectArray){
                Map<String, String> map = new HashMap<>();
                map.put("messageid", messageid);
                map.put("project", pro);

                //rtnMsg.append("project is "+pro+"<br>");
                String buildInfoJsonString=httpClient.get(url + "getbuildmsg", String.class, map);
                if(StringUtils.isNotBlank(buildInfoJsonString)){
                    JSONObject jo=JSON.parseObject(buildInfoJsonString);
                    logger.debug("getBuildMsgAfterDeploy return jo is "+buildInfoJsonString);
                    Integer code=(Integer)jo.get("code");
                    //2 任务执行正常结束 ;3 任务已经被取消 ; 4 任务执行出错,剩余任务被取消
                    if(code==2|| code == 3 || code == 4){
                        if(desc.length()>0){
                            desc+=";";
                        }
                        desc += (pro+" 状态 -->"+code+":"+jo.get("message"));
                    }else{
                        if(desc.length()>0){
                            desc+=";";
                        }
                        desc += (pro+" 状态 -->"+code);
                        allOK=false;
                    }
                }else{
                    allOK=false;
                    desc += (pro+" getBuildMsg is null ");
                    logger.debug("getBuildMsgAfterDeploy return jo null .... ");
                }
            }
            response.setMessage(desc);
            response.setData(allOK);

            return response;
        } catch (Exception ex) {
            logger.debug("getBuildMsgAfterDeploy exception ",ex);
            return new BaseResponse(201,"发生异常"+ex.getMessage());
        }
    }

    /**
     * node获取执行结果
     *
     * @param messageid  qcloud_1233_项目名称1 或者 aws_1234_项目名称2
     * @return
     */
    /*@RequestMapping(value = "/getBuildMsgAfterDeployForNode")
    @ResponseBody
    public BaseResponse getBuildMsgAfterDeployForNode(String messageid) {
        try {
            BaseResponse response=new BaseResponse();
            String desc="";
            Boolean allOK=true;
            for(String msgidAndProj:messageid.split(";")){
                String msgid=msgidAndProj.split(",")[0];
                String proj=msgidAndProj.split(",")[1];

                Map<String, String> map = new HashMap<>();
                map.put("messageid", messageid);
                map.put("project", proj);

                //rtnMsg.append("project is "+pro+"<br>");
                String buildInfoJsonString=httpClient.get(NODE_DEPLOY_URL + "getbuildmsg", String.class, map);
                if(StringUtils.isNotBlank(buildInfoJsonString)){
                    JSONObject jo=JSON.parseObject(buildInfoJsonString);
                    logger.debug("getBuildMsgAfterDeployForNode return jo is "+buildInfoJsonString);
                    Integer code=(Integer)jo.get("code");
                    //node getbuildmsg 返回的结果没有3和4 ;2 任务执行正常结束 ;3 任务已经被取消 ; 4 任务执行出错,剩余任务被取消
                    if(code==2){
                        if(desc.length()>0){
                            desc+=";";
                        }
                        desc += (proj+" 状态 -->"+jo.get("message"));
                    }else{
                        if(desc.length()>0){
                            desc+=";";
                        }
                        desc += (proj+" 状态 -->"+code);
                        allOK=false;
                    }
                }else{
                    allOK=false;
                    desc += (proj+" getBuildMsgAfterDeployForNode is null ");
                    logger.debug("getBuildMsgAfterDeployForNode return jo null .... ");
                }
            }
            response.setMessage(desc);
            response.setData(allOK);
            return response;
        } catch (Exception ex) {
            logger.debug("getBuildMsgAfterDeployForNode exception ",ex);
            return new BaseResponse(201,"发生异常"+ex.getMessage());
        }
    }*/

    /**
     * 项目发布后集成测试
     * @param cloudType 云类型 1:AWS 2:QCLOUD
     * @param projectName 项目名称
     * @param ips 部署在哪台服务器上,内网ip集合, ,分割
     * @return
     */
    @RequestMapping("/integrationTest")
    @ResponseBody
    public BaseResponse<?> integrationTest(int cloudType, String projectName, String ips){
        String cloudTypeStr="";
        if(1==cloudType){
            cloudTypeStr="aws";
        }else if(2==cloudType){
            cloudTypeStr="qcloud";
        }
        String url = String.format(HttpUriContants.GET_CLOUD_INTEGRATION_TEST+"?cloudType=%s&projectName=%s&ips=%s",cloudType, AutoScalingGroupProjectConstant.getProjectNames(cloudTypeStr,projectName), ips);
        BaseResponse<?> response = httpClient.defaultPost(url, null, BaseResponse.class);
        return response;
    }


    /**
     * 自动伸缩后,修改ip,ip从云管理中心重新获取
     * http://git.yoho.cn/yohoops/auto_deploy.git
     * @param cloudType 云类型 1:AWS 2:QCLOUD
     * @param projectName 项目名称
     * @param oldInstanceIps 伸缩之前的实例ip
     * @param upDownFlag   伸缩标志 :up  down
     * @return
     */
    @RequestMapping("/updateIpToGit")
    @ResponseBody
    public BaseResponse<?> updateIpToGit(int cloudType, String projectName, String scalingGroupId,String oldInstanceIps,String newInstanceIps,String upDownFlag){
        String cloudTypeStr="";
        if(1==cloudType){
            cloudTypeStr="aws";
        }else if(2==cloudType){
            cloudTypeStr="qcloud";
        }

        //String projectNameArray[]= AutoScalingGroupProjectConstant.getProjectNames(cloudTypeStr,projectName).split(",");
        String url = String.format(HttpUriContants.GET_CLOUD_UPDATE_IP_TO_GIT+"?cloudType=%d&projectName=%s&newInstanceIps=%s",cloudType,projectName, newInstanceIps);
        BaseResponse<?> responseTmp = httpClient.defaultPost(url, null, BaseResponse.class);
        return responseTmp;
    }


    /**
     * es项目伸缩之后,要调用脚本,更新es的配置文件
     * 由于调用脚本的机器权限,需要通过16.167机器
     * @return
     */
    @RequestMapping("/updateEsConfig")
    @ResponseBody
    public BaseResponse<?> updateEsConfig(String allIps){
        String backData=httpClient.post(DEPLOY_URL + "esmachine/updateConfig?allIps="+allIps,null , String.class);
        BaseResponse response=new BaseResponse();
        response.setData(backData);
        return response;
    }

    /**
     * 如果伸缩组是gateway项目,则需要更新upstream详细
     * @param cloudType
     * @param scalingGroupName
     * @param scalingGroupId
     * @param oldInstanceIps
     * @param newInstanceIps
     * @param upDownFlag
     * @return
     */
    @RequestMapping("/updateGatewayUpstream")
    @ResponseBody
    public BaseResponse<?> updateGatewayUpstream(int cloudType, String scalingGroupName, String scalingGroupId,String oldInstanceIps,String newInstanceIps,String upDownFlag){
        String cloudTypeStr="";
        if(1==cloudType){
            cloudTypeStr="aws";
        }else if(2==cloudType){
            cloudTypeStr="qcloud";
        }

        StringBuilder sb=new StringBuilder();
        if("gateway".equals(scalingGroupName)){
            //List<String> newInstanceIps=(List<String>)newInstanceIpsResponse.getData();
            List<String> newInstanceIpList=null;
            if(StringUtils.isNotBlank(newInstanceIps)){
                newInstanceIpList=Arrays.asList(newInstanceIps.split(","));
            }else{
                newInstanceIpList=new ArrayList<String>();
            }
            int timesMoreThanOne=0;
            int lastTimesMark=0;
            if("up".equals(upDownFlag)){
                List<String> oldIps= Arrays.asList(oldInstanceIps.split(","));
                for(String strIP:newInstanceIpList){
                    if(!oldIps.contains(strIP)&&StringUtils.isNotBlank(strIP)){
                        if(timesMoreThanOne!=lastTimesMark){
                            try{
                                lastTimesMark=timesMoreThanOne;
                                Thread.sleep(7000);//monitor-service提交git到服务器需要时间
                            }catch (Exception e){
                                logger.error("updateGatewayUpstream add sleep error",e);
                            }
                        }
                        timesMoreThanOne++;
                        String url = String.format(HttpUriContants.GET_CLOUD_UPDATE_GATEWAY_CHANGE_UPSTREAM+"?operate=%s&cloudTypeStr=%s&ip=%s","add",cloudTypeStr, strIP);
                        BaseResponse<?> responseTmp = httpClient.defaultPost(url, null, BaseResponse.class);
                        sb.append("gateway upstream add "+strIP);
                    }
                }

            }else if("down".equals(upDownFlag)){
                List<String> oldIps= Arrays.asList(oldInstanceIps.split(","));
                for(String strIP:oldIps){
                    if(!newInstanceIpList.contains(strIP)){
                        if(timesMoreThanOne!=lastTimesMark){
                            try{
                                lastTimesMark=timesMoreThanOne;
                                Thread.sleep(7000);//monitor-service提交git到服务器需要时间
                            }catch (Exception e){
                                logger.error("updateGatewayUpstream del sleep error",e);
                            }
                        }
                        timesMoreThanOne++;
                        String url = String.format(HttpUriContants.GET_CLOUD_UPDATE_GATEWAY_CHANGE_UPSTREAM+"?operate=%s&cloudTypeStr=%s&ip=%s","del",cloudTypeStr, strIP);
                        BaseResponse<?> responseTmp = httpClient.defaultPost(url, null, BaseResponse.class);
                        sb.append("gateway upstream del "+strIP);
                    }
                }
            }
        }
        BaseResponse<?> response=new BaseResponse();
        response.setMessage(sb.toString());
        return response;
    }


    /**
     * 如果伸缩组是node项目,则需要更新auto_deploy配置
     * @param cloudType
     * @param scalingGroupName
     * @param scalingGroupId
     * @param oldInstanceIps
     * @param newInstanceIps
     * @param upDownFlag
     * @return
     */
    @RequestMapping("/updateAnsibleDeployForNode")
    @ResponseBody
    public BaseResponse<?> updateAnsibleDeployForNode(int cloudType, String scalingGroupName, String scalingGroupId,String oldInstanceIps,String newInstanceIps,String upDownFlag,String updateType){
        String cloudTypeStr="";
        if(1==cloudType){
            cloudTypeStr="aws";
        }else if(2==cloudType){
            cloudTypeStr="qcloud";
        }

        StringBuilder sb=new StringBuilder();
        List<String> newInstanceIpList=null;
        if(StringUtils.isNotBlank(newInstanceIps)){
            newInstanceIpList=Arrays.asList(newInstanceIps.split(","));
        }else{
            newInstanceIpList=new ArrayList<String>();
        }
        int timesMoreThanOne=0;
        int lastTimesMark=0;
        if("up".equals(upDownFlag)){
            List<String> oldIps= Arrays.asList(oldInstanceIps.split(","));
            String changeIps="";
            for(String strIP:newInstanceIpList){
                if(!oldIps.contains(strIP)&&StringUtils.isNotBlank(strIP)){
                    if(changeIps.length()>0){
                        changeIps += ",";
                    }
                    changeIps += strIP;
                }
            }
            String operate="xxx";
            if("deploy".equals(updateType)){
                operate="add_deploy";
            }else if("nginx".equals(updateType)){
                operate="add_nginx";
            }
            ///String url = String.format(HttpUriContants.GET_CLOUD_UPDATE_ANSIBLE_DEPLOY_FOR_NODE+"?operate=%s&cloudTypeStr=%s&ip=%s",operate,cloudTypeStr, changeIps);
            AutoScalingInfoReq req=new AutoScalingInfoReq();
            req.setOperate(operate);
            req.setCloudTypeStr(cloudTypeStr);
            req.setChangeIps(changeIps);
            BaseResponse<?> responseTmp = httpClient.defaultPost(HttpUriContants.GET_CLOUD_UPDATE_ANSIBLE_DEPLOY_FOR_NODE, req, BaseResponse.class);
            sb.append("node git add "+changeIps);

        }else if("down".equals(upDownFlag)){
            List<String> oldIps= Arrays.asList(oldInstanceIps.split(","));
            String changeIps="";
            for(String strIP:oldIps){
                if(!newInstanceIpList.contains(strIP)){
                    if(changeIps.length()>0){
                        changeIps += ",";
                    }
                    changeIps += strIP;
                }
            }
            String operate="xxx";
            if("deploy".equals(updateType)){
                operate="del_deploy";
            }else if("nginx".equals(updateType)){
                operate="del_nginx";
            }
            //String url = String.format(HttpUriContants.GET_CLOUD_UPDATE_ANSIBLE_DEPLOY_FOR_NODE+"?operate=%s&cloudTypeStr=%s&ip=%s",operate,cloudTypeStr, changeIps);
            AutoScalingInfoReq req=new AutoScalingInfoReq();
            req.setOperate(operate);
            req.setCloudTypeStr(cloudTypeStr);
            req.setChangeIps(changeIps);
            BaseResponse<?> responseTmp = httpClient.defaultPost(HttpUriContants.GET_CLOUD_UPDATE_ANSIBLE_DEPLOY_FOR_NODE, req, BaseResponse.class);
            sb.append("node git del "+changeIps);
        }
        //刷新node配置git
        //httpClient.get(NODE_DEPLOY_URL + "refreshGit", String.class, null);

        BaseResponse<?> response=new BaseResponse();
        response.setMessage(sb.toString());
        return response;
    }


    /**
     * the last step of 伸缩:根据伸缩ips更新Host_info表数据
     * @param cloudType 云类型
     * @param scalingGroupName 组名称
     * @param oldInstanceIps 伸缩之前的ip集合
     * @param upDownFlag 扩容up 缩减down
     * @return
     */
    @RequestMapping("/updateToHost")
    @ResponseBody
    public BaseResponse updateToHost(int cloudType, String scalingGroupName, String newInstanceIps,String oldInstanceIps,String upDownFlag){
        BaseResponse resp = null;
        List<String> oldIps = Arrays.asList(oldInstanceIps.split(","));
        List<String> newIpLists = Arrays.asList(newInstanceIps.split(","));


        if("up".equals(upDownFlag)){//扩容,则向host_info中添加数据
            List<String> newIps =  new ArrayList<>();
            for(String strIp : newIpLists){
                if(!oldIps.contains(strIp) && StringUtils.isNotBlank(strIp))
                    newIps.add(strIp);//新增ip集合
            }
            if(newIps.size() > 0){
                List<HostInfo> hostInfos = new ArrayList<>();
                for(String ip: newIps){
                    HostInfo h =  new HostInfo();
                    h.setHostIp(ip);
                    h.setTags(AutoScalingGroupProjectConstant.getTags(cloudType == 1 ? "aws":"qcloud",scalingGroupName));
                    if(scalingGroupName.contains("java")){
                        h.setAlias(String.format("%s-%s-%s",(cloudType == 1 ? "AWS":"Qcloud"),  scalingGroupName,ip));
                    }else{
                        h.setAlias(String.format("%s-%s-%s",(cloudType == 1 ? "AWS":"Qcloud"), "java-"+ scalingGroupName,ip));
                    }
                    h.setCloudType(cloudType);
                    hostInfos.add(h);
                }
                resp = httpClient.defaultPost(HttpUriContants.SAVE_HOST_LIST_INFOS ,hostInfos,BaseResponse.class);
            }

        }else if("down".equals(upDownFlag)){//缩减--更新host_info tags字段为空
            List<String> removeIps = new ArrayList<>();
            for(String strIp : oldIps){
                if(!newIpLists.contains(strIp) ){
                    removeIps.add(strIp);//被缩减的ip集合
                }
            }
            resp =  httpClient.defaultPost(HttpUriContants.REMOVE_HOST_INFO_BY_IPS  , removeIps,BaseResponse.class);
        }
        return resp;
    }


    /**
     * memcache更新:内部dns
     */
    @RequestMapping("/updateMemcacheInfoForInternalDomain")
    @ResponseBody
    public BaseResponse updateMemcacheInfoForInternalDomain(AutoScalingInfoReq infoReq){
        BaseResponse resp = null;
        resp = httpClient.defaultPost(HttpUriContants.GET_CLOUD_UPDATE_MEMCACHE_DEPLOY_FOR_INTERNAL_DOMAIN ,infoReq,BaseResponse.class);
        return resp;
    }

    /**
     *  更新:监控对象
     */
    @RequestMapping("/updateMobjectInfo")
    @ResponseBody
    public BaseResponse updateMobjectInfo(AutoScalingInfoReq infoReq){
        BaseResponse resp=new BaseResponse();
        if(StringUtils.isBlank(infoReq.getCloudTypeStr())){
            if(1==infoReq.getCloudType()){
                infoReq.setCloudTypeStr("aws");
            }else if(2==infoReq.getCloudType()){
                infoReq.setCloudTypeStr("qcloud");
            }
        }

        String mobjectPaths=AutoScalingGroupProjectConstant.getMobjectPaths(infoReq.getCloudTypeStr(),infoReq.getScalingGroupName());
        if(StringUtils.isNotBlank(mobjectPaths)){
            infoReq.setMobjectPaths(mobjectPaths);
            resp = httpClient.defaultPost(HttpUriContants.GET_CLOUD_UPDATE_MOBJECT ,infoReq,BaseResponse.class);
        }else{
            resp.setData("mobjectPaths路径没有配置,跳过更新mobject");
        }
        return resp;
    }


    /**
     *  nigix类型的伸缩:伸缩后执行脚本
     */
    @RequestMapping("/updateNginxConfig")
    @ResponseBody
    public BaseResponse updateNginxConfig(AutoScalingInfoReq infoReq){
        BaseResponse resp = null;
        resp = httpClient.defaultPost(HttpUriContants.GET_CLOUD_UPDATE_NGINX_CONFIG ,infoReq,BaseResponse.class);
        return resp;
    }
}