ops / yoho-trace-analyzer · Commits

Go to a project

Authored by 谢勇
Commit fc1389c5b4207b6215b86263e82c1e7f90dca59c 1 parent 337ecebe

性能优化

Inline Side-by-side
Showing 3 changed files with 111 additions and 88 deletions
... ... @@ -8,6 +8,7 @@ import java.util.List;
import org.apache.commons.collections.CollectionUtils;
import org.apache.commons.lang3.StringUtils;
import org.apache.hadoop.hbase.TableName;
import org.apache.hadoop.hbase.client.Durability;
import org.apache.hadoop.hbase.client.HTable;
import org.apache.hadoop.hbase.client.Put;
import org.apache.hadoop.hbase.util.Bytes;
... ... @@ -342,6 +343,8 @@ public class TraceAnalyzeHandler implements TraceHandler, Serializable {
SpanIpResult spanIpResult = next._2;
Put put = new Put(Bytes.toBytes(key));
//其实不推荐关闭WAL，不过关了的确可以提升性能...因为HBase在写数据前会先写WAL，以保证在异常情况下，HBase可以按照WAL的记录来恢复还未持久化的数据。
put.setDurability(Durability.SKIP_WAL);
put.addColumn(Bytes.toBytes("data"), Bytes.toBytes("avg_duration"), Bytes.toBytes(spanIpResult.getAvgDuration()));
put.addColumn(Bytes.toBytes("data"), Bytes.toBytes("times"), Bytes.toBytes(spanIpResult.getTimes()));
put.addColumn(Bytes.toBytes("data"), Bytes.toBytes("ip"), Bytes.toBytes(spanIpResult.getIp()));
... ... @@ -418,12 +421,16 @@ public class TraceAnalyzeHandler implements TraceHandler, Serializable {
ApiTraceResult apiTraceResult = next._2;
Put put = new Put(Bytes.toBytes(apiTraceResult.getTraceStartTime()/1000 + ":" + apiTraceResult.getTraceId()));
//其实不推荐关闭WAL，不过关了的确可以提升性能...因为HBase在写数据前会先写WAL，以保证在异常情况下，HBase可以按照WAL的记录来恢复还未持久化的数据。
put.setDurability(Durability.SKIP_WAL);
put.addColumn(Bytes.toBytes("data"), Bytes.toBytes("spans"), Bytes.toBytes(JSONObject.toJSONString(apiTraceResult.getSpans())));
put.addColumn(Bytes.toBytes("data"), Bytes.toBytes("traceid"), Bytes.toBytes(apiTraceResult.getTraceId()));
put.addColumn(Bytes.toBytes("data"), Bytes.toBytes("starttime"), Bytes.toBytes(apiTraceResult.getTraceStartTime()/1000));
Put put2 = new Put(Bytes.toBytes(apiTraceResult.getApiName() + ":" + apiTraceResult.getTraceStartTime()/1000 + ":" + apiTraceResult.getTraceId()));
//其实不推荐关闭WAL，不过关了的确可以提升性能...因为HBase在写数据前会先写WAL，以保证在异常情况下，HBase可以按照WAL的记录来恢复还未持久化的数据。
put2.setDurability(Durability.SKIP_WAL);
put2.addColumn(Bytes.toBytes("data"), Bytes.toBytes("spans"), Bytes.toBytes(JSONObject.toJSONString(apiTraceResult.getSpans())));
put2.addColumn(Bytes.toBytes("data"), Bytes.toBytes("traceid"), Bytes.toBytes(apiTraceResult.getTraceId()));
put2.addColumn(Bytes.toBytes("data"), Bytes.toBytes("starttime"), Bytes.toBytes(apiTraceResult.getTraceStartTime()/1000));
... ... @@ -474,6 +481,8 @@ public class TraceAnalyzeHandler implements TraceHandler, Serializable {
String[] md5Tags = StringUtils.split(apiTraceResult.getTraceMd5(), '.');
Put put1 = new Put(Bytes.toBytes( md5Tags[1] + ":" + apiTraceResult.getTraceStartTime()/1000 + ":" + apiTraceResult.getTraceId()));
//其实不推荐关闭WAL，不过关了的确可以提升性能...因为HBase在写数据前会先写WAL，以保证在异常情况下，HBase可以按照WAL的记录来恢复还未持久化的数据。
put1.setDurability(Durability.SKIP_WAL);
put1.addColumn(Bytes.toBytes("data"), Bytes.toBytes("spans"), Bytes.toBytes(JSONObject.toJSONString(apiTraceResult.getSpans())));
put1.addColumn(Bytes.toBytes("data"), Bytes.toBytes("traceid"), Bytes.toBytes(apiTraceResult.getTraceId()));
put1.addColumn(Bytes.toBytes("data"), Bytes.toBytes("starttime"), Bytes.toBytes(apiTraceResult.getTraceStartTime()/1000));
... ... @@ -482,6 +491,8 @@ public class TraceAnalyzeHandler implements TraceHandler, Serializable {
Put put2 = new Put(Bytes.toBytes( apiTraceResult.getApiName() + ":" + apiTraceResult.getTraceStartTime()/1000 + ":" + apiTraceResult.getTraceId()));
//其实不推荐关闭WAL，不过关了的确可以提升性能...因为HBase在写数据前会先写WAL，以保证在异常情况下，HBase可以按照WAL的记录来恢复还未持久化的数据。
put2.setDurability(Durability.SKIP_WAL);
put2.addColumn(Bytes.toBytes("data"), Bytes.toBytes("spans"), Bytes.toBytes(JSONObject.toJSONString(apiTraceResult.getSpans())));
put2.addColumn(Bytes.toBytes("data"), Bytes.toBytes("traceid"), Bytes.toBytes(apiTraceResult.getTraceId()));
put2.addColumn(Bytes.toBytes("data"), Bytes.toBytes("starttime"), Bytes.toBytes(apiTraceResult.getTraceStartTime()/1000));
... ...
... ... @@ -19,42 +19,48 @@ import com.yoho.trace.store.TraceSpanStore;
/**
* Created by markeloff on 2017/7/26.
*/
public class TraceHbaseHandler implements TraceHandler , Serializable {
Logger logger = LoggerFactory.getLogger(TraceHbaseHandler.class) ;
@Override
public void handle(final JavaDStream<Spans> kafkaMsgDStream) {
kafkaMsgDStream.foreachRDD(new VoidFunction<JavaRDD<Spans>>() {
@Override
public void call(JavaRDD<Spans> spansJavaRDD) throws Exception {
spansJavaRDD.foreachPartition(new VoidFunction<Iterator<Spans>>() {
@Override
public void call(Iterator<Spans> spansIterator) throws Exception {
HTable traceTable = null ;
long begin = System.currentTimeMillis() ;
try {
traceTable = (HTable) HBasePool.getConnection().getTable(TableName.valueOf("trace"));
//traceTable.setWriteBufferSize(1024 * 1024 * 6);
//traceTable.setAutoFlush(false, true);
int count = TraceSpanStore.store(spansIterator, traceTable) ;
//traceTable.flushCommits();
logger.info("flush spans to hbase, count {}, elapse {}", count, System.currentTimeMillis()-begin );
}finally {
try {
if(traceTable != null ) traceTable.close();
} catch (IOException e) {
e.printStackTrace();
}
}
}
});
}
});
}
public class TraceHbaseHandler implements TraceHandler, Serializable {
Logger logger = LoggerFactory.getLogger(TraceHbaseHandler.class);
@Override
public void handle(final JavaDStream<Spans> kafkaMsgDStream) {
kafkaMsgDStream.foreachRDD(new VoidFunction<JavaRDD<Spans>>() {
@Override
public void call(JavaRDD<Spans> spansJavaRDD) throws Exception {
spansJavaRDD.foreachPartition(new VoidFunction<Iterator<Spans>>() {
@Override
public void call(Iterator<Spans> spansIterator) throws Exception {
// HTable traceTable = null ;
HTable[] tables = new HTable[3];
int count = 0;
long begin = System.currentTimeMillis();
try {
for (int i = 0; i < 3; i++) {
tables[i] = (HTable) HBasePool.getConnection().getTable(TableName.valueOf("trace"));
tables[i].setWriteBufferSize(1024 * 1024 * 6);
tables[i].setAutoFlush(false, true);
while (spansIterator.hasNext()) {
count = count + TraceSpanStore.store(spansIterator.next(), tables[i]);
}
tables[i].flushCommits();
logger.info("flush spans to hbase, count {}, elapse {}", count,System.currentTimeMillis() - begin);
}
} finally {
try {
for (HTable hTable : tables) {
if (hTable != null)
hTable.close();
}
} catch (IOException e) {
e.printStackTrace();
}
}
}
});
}
});
}
}
... ...
package com.yoho.trace.store;
import java.io.Serializable;
import java.util.ArrayList;
import java.util.Iterator;
import java.util.List;
import org.apache.commons.collections.CollectionUtils;
import org.apache.hadoop.hbase.client.Durability;
import org.apache.hadoop.hbase.client.HTable;
import org.apache.hadoop.hbase.client.Put;
import org.apache.hadoop.hbase.util.Bytes;
... ... @@ -21,51 +20,58 @@ import com.yoho.trace.sleuth.Spans;
*/
public class TraceSpanStore implements Serializable {
private static final Logger logger = LoggerFactory.getLogger(TraceSpanStore.class);
public static int store(Iterator<Spans> spansIterator, HTable traceTable) {
List<Put> spanPutList = new ArrayList<>(10000);
try {
while (spansIterator.hasNext()) {
Spans spans = spansIterator.next();
if (spans != null && CollectionUtils.isNotEmpty(spans.getSpans())) {
List<Span> spanList = spans.getSpans();
for (int i = 0; i < spanList.size(); i++) {
Span span = spanList.get(i);
StringBuffer sb = new StringBuffer(128);
String logEvent = "none";
if (CollectionUtils.isNotEmpty(span.logs())) {
logEvent = span.logs().get(0).getEvent();
}
String rowkey = sb.append(span.idToHex(span.getTraceId()) + ":" + (span.getBegin() / 1000))
.append(":").append(span.idToHex(span.getSpanId())).append(":").append(logEvent)
.toString();
Put put = new Put(Bytes.toBytes(rowkey));
put.addColumn(Bytes.toBytes("span"), Bytes.toBytes("service"),Bytes.toBytes(spans.getHost().getServiceName()));
put.addColumn(Bytes.toBytes("span"), Bytes.toBytes("event"), Bytes.toBytes(logEvent));
put.addColumn(Bytes.toBytes("span"), Bytes.toBytes("ip"),Bytes.toBytes(spans.getHost().getAddress()));
put.addColumn(Bytes.toBytes("span"), Bytes.toBytes("name"), Bytes.toBytes(span.getName()));
put.addColumn(Bytes.toBytes("span"), Bytes.toBytes("traceid"),Bytes.toBytes(Span.idToHex(span.getTraceId())));
put.addColumn(Bytes.toBytes("span"), Bytes.toBytes("spanid"),Bytes.toBytes(Span.idToHex(span.getSpanId())));
put.addColumn(Bytes.toBytes("span"), Bytes.toBytes("begin"), Bytes.toBytes(span.getBegin()));
put.addColumn(Bytes.toBytes("span"), Bytes.toBytes("end"), Bytes.toBytes(span.getEnd()));
if (CollectionUtils.isNotEmpty(span.getParents())) {
put.addColumn(Bytes.toBytes("span"), Bytes.toBytes("parent"),Bytes.toBytes(Span.idToHex(span.getParents().get(span.getParents().size() - 1))));
}
put.addColumn(Bytes.toBytes("span"), Bytes.toBytes("logs"),Bytes.toBytes(JSONObject.toJSONString(span.logs())));
put.addColumn(Bytes.toBytes("span"), Bytes.toBytes("tags"),Bytes.toBytes(JSONObject.toJSONString(span.tags())));
spanPutList.add(put);
}
}
}
traceTable.put(spanPutList);
} catch (Exception e) {
}
return spanPutList.size();
}
}
\ No newline at end of file
private static final Logger logger = LoggerFactory.getLogger(TraceSpanStore.class) ;
public static int store(Spans spans, HTable traceTable) {
int count = 0 ;
if (spans != null && CollectionUtils.isNotEmpty(spans.getSpans())) {
try {
List<Span> spanList = spans.getSpans() ;
for(int i=0; i<spanList.size(); i++){
Span span = spanList.get(i);
StringBuffer sb = new StringBuffer(128);
String logEvent = "none" ;
if( CollectionUtils.isNotEmpty(span.logs()) ){
logEvent = span.logs().get(0).getEvent() ;
}
String rowkey = sb.append( span.idToHex(span.getTraceId()) + ":" + (span.getBegin()/1000) ).append(":").append(span.idToHex(span.getSpanId()))
.append(":").append(logEvent).toString() ;
Put put = new Put(Bytes.toBytes(rowkey)) ;
//其实不推荐关闭WAL，不过关了的确可以提升性能...因为HBase在写数据前会先写WAL，以保证在异常情况下，HBase可以按照WAL的记录来恢复还未持久化的数据。
put.setDurability(Durability.SKIP_WAL);
put.addColumn(Bytes.toBytes("span"),Bytes.toBytes("service"),Bytes.toBytes( spans.getHost().getServiceName() )) ;
put.addColumn(Bytes.toBytes("span"),Bytes.toBytes("event"),Bytes.toBytes( logEvent )) ;
put.addColumn(Bytes.toBytes("span"),Bytes.toBytes("ip"),Bytes.toBytes( spans.getHost().getAddress() )) ;
put.addColumn(Bytes.toBytes("span"),Bytes.toBytes("name"),Bytes.toBytes(span.getName())) ;
put.addColumn(Bytes.toBytes("span"),Bytes.toBytes("traceid"),Bytes.toBytes( Span.idToHex(span.getTraceId()) )) ;
put.addColumn(Bytes.toBytes("span"),Bytes.toBytes("spanid"),Bytes.toBytes( Span.idToHex(span.getSpanId()) )) ;
put.addColumn(Bytes.toBytes("span"),Bytes.toBytes("begin"),Bytes.toBytes( span.getBegin() )) ;
put.addColumn(Bytes.toBytes("span"),Bytes.toBytes("end"),Bytes.toBytes( span.getEnd() )) ;
if( CollectionUtils.isNotEmpty( span.getParents())) {
put.addColumn(Bytes.toBytes("span"), Bytes.toBytes("parent"), Bytes.toBytes(Span.idToHex(span.getParents().get(span.getParents().size() - 1)))) ;
}
put.addColumn(Bytes.toBytes("span"),Bytes.toBytes("logs"),Bytes.toBytes(JSONObject.toJSONString(span.logs()))) ;
put.addColumn(Bytes.toBytes("span"),Bytes.toBytes("tags"),Bytes.toBytes(JSONObject.toJSONString(span.tags()))) ;
traceTable.put(put);
count++ ;
}
}catch(Exception e){
logger.error("store to hbase failed, spand {} ", JSONObject.toJSONString(spans),e);
}
}
return count ;
}
}
... ...