ops / yoho-trace-analyzer · Commits

Go to a project

Authored by wangning
Commit c8bc70670180ca86fa036d02c7211ac2c3ec8f6a 1 parent 1a89dce6

批量入库

Inline Side-by-side
Showing 2 changed files with 47 additions and 45 deletions
... ... @@ -464,15 +464,11 @@ public class TraceAnalyzeHandler implements TraceHandler, Serializable {
apiResultTraceDStream.foreachRDD(new VoidFunction<JavaPairRDD<String, ApiTraceResult>>() {
@Override
public void call(JavaPairRDD<String, ApiTraceResult> stringSortedTraceJavaPairRDD) throws Exception {
System.out.println("handlerAllTrace begin1");
logger.info("handlerAllTrace begin1");
stringSortedTraceJavaPairRDD.foreachPartition(new VoidFunction<Iterator<Tuple2<String, ApiTraceResult>>>() {
@Override
public void call(Iterator<Tuple2<String, ApiTraceResult>> tuple2Iterator) throws Exception {
System.out.println("handlerAllTrace begin2");
logger.info("handlerAllTrace begin2");
HTable resultTable = null;
try {
if(tuple2Iterator == null){
... ...
... ... @@ -7,15 +7,15 @@ import org.apache.commons.lang3.StringUtils;
import org.apache.hadoop.hbase.TableName;
import org.apache.hadoop.hbase.client.HTable;
import org.apache.hadoop.hbase.client.Put;
import org.apache.hadoop.hbase.client.Table;
import org.apache.hadoop.hbase.util.Bytes;
import org.apache.spark.api.java.JavaPairRDD;
import org.apache.spark.api.java.function.VoidFunction;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import properties.PropertiesFactory;
import scala.Tuple2;
import java.util.ArrayList;
import java.util.Iterator;
import java.util.List;
/**
... ... @@ -68,49 +68,55 @@ public class ApiStatisticsResultStore {
);
}
//insert data
apiResultRdd.foreach(
new VoidFunction<Tuple2<String, ApiTraceResult>>() {
@Override
public void call(Tuple2<String, ApiTraceResult> tuple2) throws Exception {
ApiTraceResult result = tuple2._2();
HTable resultTable = null;
try {
if (resultTable == null) {
resultTable = (HTable) HBasePool.getConnection().getTable(TableName.valueOf(tableName));
}
String rowkey = result.getApiName() + ":" + result.getTraceMd5();
Put put = new Put(Bytes.toBytes(rowkey));
put.addColumn(Bytes.toBytes("trace"), Bytes.toBytes("api"), Bytes.toBytes(result.getApiName()));
String[] md5Tags = StringUtils.split(result.getTraceMd5(), '.');
if (null == md5Tags || 2 != md5Tags.length) {
put.addColumn(Bytes.toBytes("trace"), Bytes.toBytes("traceMd5"), Bytes.toBytes(StringUtils.EMPTY));
} else {
put.addColumn(Bytes.toBytes("trace"), Bytes.toBytes("traceMd5"), Bytes.toBytes(md5Tags[1]));
}
put.addColumn(Bytes.toBytes("trace"), Bytes.toBytes("duration"), Bytes.toBytes(result.getDuration()));
put.addColumn(Bytes.toBytes("trace"), Bytes.toBytes("times"), Bytes.toBytes(result.getCallTimes()));
put.addColumn(Bytes.toBytes("trace"), Bytes.toBytes("maxLatency"), Bytes.toBytes(result.getMaxLatency()));
put.addColumn(Bytes.toBytes("trace"), Bytes.toBytes("maxLatencyTrace"), Bytes.toBytes(result.getMaxLatencyTrace()));
put.addColumn(Bytes.toBytes("trace"), Bytes.toBytes("minLatency"), Bytes.toBytes(result.getMinLatency()));
put.addColumn(Bytes.toBytes("trace"), Bytes.toBytes("minLatencyTrace"), Bytes.toBytes(result.getMinLatencyTrace()));
put.addColumn(Bytes.toBytes("trace"), Bytes.toBytes("spans"), Bytes.toBytes(JSONObject.toJSONString(result.getSpans())));
logger.info("put data to hbase, {}", put);
resultTable.put(put);
} catch (Exception e) {
logger.error("store ApiTraceResult failed, result {} ", result, e);
} finally {
if (resultTable != null)
resultTable.close();
apiResultRdd.foreachPartition(new VoidFunction<Iterator<Tuple2<String, ApiTraceResult>>>() {
@Override
public void call(Iterator<Tuple2<String, ApiTraceResult>> tuple2Iterator) throws Exception {
HTable resultTable = null;
try {
if (resultTable == null) {
resultTable = (HTable) HBasePool.getConnection().getTable(TableName.valueOf(tableName));
}
if(tuple2Iterator == null){
return;
}
List<Put> putList = new ArrayList<>();
while(tuple2Iterator.hasNext()){
Tuple2<String, ApiTraceResult> next = tuple2Iterator.next();
ApiTraceResult apiTraceResult = next._2;
String rowkey = apiTraceResult.getApiName() + ":" + apiTraceResult.getTraceMd5();
Put put = new Put(Bytes.toBytes(rowkey));
put.addColumn(Bytes.toBytes("trace"), Bytes.toBytes("api"), Bytes.toBytes(apiTraceResult.getApiName()));
String[] md5Tags = StringUtils.split(apiTraceResult.getTraceMd5(), '.');
if (null == md5Tags || 2 != md5Tags.length) {
put.addColumn(Bytes.toBytes("trace"), Bytes.toBytes("traceMd5"), Bytes.toBytes(StringUtils.EMPTY));
} else {
put.addColumn(Bytes.toBytes("trace"), Bytes.toBytes("traceMd5"), Bytes.toBytes(md5Tags[1]));
}
put.addColumn(Bytes.toBytes("trace"), Bytes.toBytes("duration"), Bytes.toBytes(apiTraceResult.getDuration()));
put.addColumn(Bytes.toBytes("trace"), Bytes.toBytes("times"), Bytes.toBytes(apiTraceResult.getCallTimes()));
put.addColumn(Bytes.toBytes("trace"), Bytes.toBytes("maxLatency"), Bytes.toBytes(apiTraceResult.getMaxLatency()));
put.addColumn(Bytes.toBytes("trace"), Bytes.toBytes("maxLatencyTrace"), Bytes.toBytes(apiTraceResult.getMaxLatencyTrace()));
put.addColumn(Bytes.toBytes("trace"), Bytes.toBytes("minLatency"), Bytes.toBytes(apiTraceResult.getMinLatency()));
put.addColumn(Bytes.toBytes("trace"), Bytes.toBytes("minLatencyTrace"), Bytes.toBytes(apiTraceResult.getMinLatencyTrace()));
put.addColumn(Bytes.toBytes("trace"), Bytes.toBytes("spans"), Bytes.toBytes(JSONObject.toJSONString(apiTraceResult.getSpans())));
putList.add(put);
}
resultTable.put(putList);
} catch (Exception e) {
logger.error(e.getMessage(),e);
} finally {
if (resultTable != null)
resultTable.close();
}
);
}
});
}
}
... ...