MainSparkExecutoKill.sh
3.13 KB
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
#!/bin//bash
source /etc/profile
i21=`yarn application -list|grep -w trace-analyzer|awk '{print $6}'`
j21=`yarn application -list|grep -w trace-analyzer|awk '{print $1}'`
echo $i21
echo $j21
if [ ! $i21 ]; then
hdfs dfs -rm -r /spark/checkpoint/trace-analyzer
nohup spark-submit \
--name trace-analyzer \
--master yarn-cluster \
--class com.yoho.trace.online.TraceExecutor \
--executor-memory 2g \
--driver-memory 2g \
--executor-cores 2 \
--queue hadoop \
--files /usr/local/service/spark/conf/log4j-container.properties \
--conf spark.executor.extraJavaOptions="-Dlog4j.configuration=file:log4j-container.properties -Dlog4j.debug=true" \
--conf spark.dynamicAllocation.enabled=true \
--conf spark.shuffle.service.enabled=true \
--conf spark.dynamicAllocation.executorIdleTimeout=65 \
--conf spark.dynamicAllocation.cachedExecutorIdleTimeout=65 \
--conf spark.dynamicAllocation.schedulerBacklogTimeout=2 \
--conf spark.dynamicAllocation.sustainedSchedulerBacklogTimeout=2 \
--conf spark.dynamicAllocation.initialExecutors=2 \
--conf spark.dynamicAllocation.maxExecutors=4 \
--conf spark.dynamicAllocation.minExecutors=4 \
--conf spark.default.parallelism=6 \
--conf spark.memory.fraction=0.5 \
--conf spark.memory.storageFraction=0.3 \
--conf spark.executor.extraJavaOptions="-XX:+UseParNewGC -XX:+UseConcMarkSweepGC -XX:+CMSParallelRemarkEnabled -XX:+ParallelRefProcEnabled -XX:+CMSClassUnloadingEnabled -XX:+UseCMSCompactAtFullCollection -XX:+UseCMSInitiatingOccupancyOnly -XX:CMSInitiatingOccupancyFraction=75 -Dlog4j.configuration=file:log4j-container.properties" \
/home/hadoop/run/trace-analyzer/yoho-trace-analyner-1.0.0-SNAPSHOT-jar-with-dependencies.jar >>/home/hadoop/trace/trace.log 2>&1 &
#/usr/bin/curl -l -H "Content-type: application/json" -X POST -d '{"host":"10.67.1.139","service":"TraceExecutor","info":"spark任务异常"}' http://172.31.50.139:8880/monitor/recvMonitAlarm
echo "start TraceExecutor ok!!!"
else
if [ $i21 != "RUNNING" ]
then
echo "restart TraceExecutor!!!"
hdfs dfs -rm -r /spark/checkpoint/trace-analyzer
/usr/bin/yarn -kill $j21 &&
nohup spark-submit \
--name trace-analyzer \
--master yarn-cluster \
--class com.yoho.trace.online.TraceExecutor \
--executor-memory 2g \
--driver-memory 2g \
--num-executors 8 \
--executor-cores 2 \
--queue hadoop \
--conf spark.default.parallelism=6 \
--conf spark.memory.fraction=0.5 \
--conf spark.memory.storageFraction=0.3 \
--conf spark.executor.extraJavaOptions="-XX:+UseParNewGC -XX:+UseConcMarkSweepGC -XX:+CMSParallelRemarkEnabled -XX:+ParallelRefProcEnabled -XX:+CMSClassUnloadingEnabled -XX:+UseCMSCompactAtFullCollection
-XX:+UseCMSInitiatingOccupancyOnly -XX:CMSInitiatingOccupancyFraction=75" \
/home/hadoop/run/trace-analyzer/yoho-trace-analyner-1.0.0-SNAPSHOT-jar-with-dependencies.jar >>/home/hadoop/trace/trace.log 2>&1 &
/usr/bin/curl -l -H "Content-type: application/json" -X POST -d '{"host":"10.67.1.139","service":"TraceExecutor","info":"spark任务异常"}' http://172.31.50.139:8880/monitor/recvMonitAlarm
fi
fi