ops / yoho-trace-analyzer · Commits

Go to a project

Authored by 谢勇
Commit 337ecebe45e9b0fb7256550185acc15455ee3acb 1 parent ce4f44ef

性能优化

Inline Side-by-side
Showing 3 changed files with 16 additions and 19 deletions
... ... @@ -38,7 +38,7 @@ public class TraceHbaseHandler implements TraceHandler , Serializable {
long begin = System.currentTimeMillis() ;
try {
traceTable = (HTable) HBasePool.getConnection().getTable(TableName.valueOf("trace"));
traceTable.setWriteBufferSize(1024 * 1024 * 6);
//traceTable.setWriteBufferSize(1024 * 1024 * 6);
//traceTable.setAutoFlush(false, true);
int count = TraceSpanStore.store(spansIterator, traceTable) ;
//traceTable.flushCommits();
... ...
package com.yoho.trace.store;
import com.alibaba.fastjson.JSONObject;
import com.yoho.trace.anaylzer.model.ApiTraceResult;
import com.yoho.trace.anaylzer.model.SpanResult;
import java.util.ArrayList;
import java.util.Iterator;
import java.util.List;
import org.apache.commons.lang3.StringUtils;
import org.apache.hadoop.hbase.TableName;
import org.apache.hadoop.hbase.client.HTable;
... ... @@ -12,11 +13,11 @@ import org.apache.spark.api.java.JavaPairRDD;
import org.apache.spark.api.java.function.VoidFunction;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import scala.Tuple2;
import java.util.ArrayList;
import java.util.Iterator;
import java.util.List;
import com.alibaba.fastjson.JSONObject;
import com.yoho.trace.anaylzer.model.ApiTraceResult;
import scala.Tuple2;
/**
* Created by markeloff on 2017/7/26.
... ... @@ -116,7 +117,6 @@ public class ApiStatisticsResultStore {
}
resultTable.put(putList);
} catch (Exception e) {
logger.error(e.getMessage(),e);
} finally {
... ... @@ -125,7 +125,6 @@ public class ApiStatisticsResultStore {
}
}
});
}
}
... ...
... ... @@ -25,11 +25,10 @@ public class TraceSpanStore implements Serializable {
public static int store(Iterator<Spans> spansIterator, HTable traceTable) {
List<Put> spanPutList = new ArrayList<>(10000);
while (spansIterator.hasNext()) {
Spans spans = spansIterator.next();
if (spans != null && CollectionUtils.isNotEmpty(spans.getSpans())) {
try {
try {
while (spansIterator.hasNext()) {
Spans spans = spansIterator.next();
if (spans != null && CollectionUtils.isNotEmpty(spans.getSpans())) {
List<Span> spanList = spans.getSpans();
for (int i = 0; i < spanList.size(); i++) {
Span span = spanList.get(i);
... ... @@ -61,12 +60,11 @@ public class TraceSpanStore implements Serializable {
put.addColumn(Bytes.toBytes("span"), Bytes.toBytes("tags"),Bytes.toBytes(JSONObject.toJSONString(span.tags())));
spanPutList.add(put);
}
traceTable.put(spanPutList);
} catch (Exception e) {
logger.error("store to hbase failed, spand {} ", JSONObject.toJSONString(spans), e);
}
}
traceTable.put(spanPutList);
} catch (Exception e) {
}
return spanPutList.size();
}
... ...