...
|
...
|
@@ -15,6 +15,113 @@ POST /android_report.do |
|
|
Host: error-report.yoho.cn
|
|
|
```
|
|
|
|
|
|
|
|
|
## 1. Write APP report error info to log files
|
|
|
|
|
|
```
|
|
|
null|null|null|android|null|180c05cb6ef78f508731690851f7e263|ULOGIN|null|null|null|null|20170505153357|50988732|null|null|null|0|6.0|CAM-TL00H|other|0|0|null|VUxPR0lO|8608630377630425317b29953e941d9|1493969638271|2017-05-05 15:33:58.271
|
|
|
```
|
|
|
|
|
|
## 2.split logs file to per-hour:
|
|
|
|
|
|
```
|
|
|
app_error_2017-05-05-11.log
|
|
|
app_error_2017-05-05-12.log
|
|
|
```
|
|
|
|
|
|
## 3.create external hive tables
|
|
|
|
|
|
```sql
|
|
|
create external table app_error_logs (
|
|
|
time string,
|
|
|
cid string,
|
|
|
body string,
|
|
|
client string,
|
|
|
ec string,
|
|
|
ei string,
|
|
|
et string,
|
|
|
id string,
|
|
|
method string,
|
|
|
response string,
|
|
|
sid string,
|
|
|
ts string,
|
|
|
uid string,
|
|
|
tec string,
|
|
|
cts string,
|
|
|
method_event string,
|
|
|
index string,
|
|
|
osv string,
|
|
|
dm string,
|
|
|
av string,
|
|
|
net string,
|
|
|
ca string,
|
|
|
st string,
|
|
|
exception_type string,
|
|
|
udid string,
|
|
|
collect_time bigint,
|
|
|
collect_time_ts timestamp
|
|
|
)
|
|
|
partitioned by (day string)
|
|
|
row format delimited
|
|
|
fields terminated by '|'
|
|
|
stored as textfile;
|
|
|
|
|
|
```
|
|
|
|
|
|
## 4. cron: put files to qcloud cos storage
|
|
|
|
|
|
```bash
|
|
|
|
|
|
#!/bin/bash
|
|
|
|
|
|
file_path="/Data/applogs"
|
|
|
file_name_prefix="app_error"
|
|
|
|
|
|
ssh_host="root@10.66.4.133"
|
|
|
remote_path="/data/applogs"
|
|
|
|
|
|
function getFileFullName () {
|
|
|
file_time=`date -d '-1 hour' +'%Y-%m-%d-%H'`
|
|
|
file_name=${file_name_prefix}"_"${file_time}".log"
|
|
|
file_full_path=${file_path}"/"${file_name}
|
|
|
}
|
|
|
|
|
|
getFileFullName
|
|
|
|
|
|
date_path=`date -d '-1 hour' +'%Y-%m-%d'`
|
|
|
if [ -e ${file_full_path} ]
|
|
|
then
|
|
|
remote_date_path=${remote_path}"/"${date_path}
|
|
|
if ssh $ssh_host test -e $remote_date_path
|
|
|
then
|
|
|
echo `date`":"${remote_date_path}" exists, does not create"
|
|
|
else
|
|
|
echo `date`":"${remote_date_path}" does not exist, ready to create"
|
|
|
ssh $ssh_host "cd ${remote_path}; mkdir ${date_path}"
|
|
|
ssh $ssh_host /usr/local/service/hadoop/bin/hadoop fs -mkdir cosn://applogs/app_error_logs/${date_path}
|
|
|
ssh $ssh_host "/usr/local/service/hive/bin/hive -e \"alter table app_error_logs add partition (day='${date_path}') location 'cosn://applogs/app_error_logs/${date_path}'\""
|
|
|
fi
|
|
|
|
|
|
remote_file=${remote_date_path}"/"${file_name}
|
|
|
if ssh $ssh_host test -e $remote_file
|
|
|
then
|
|
|
echo `date`":"${remote_file}" exists"
|
|
|
exit
|
|
|
fi
|
|
|
|
|
|
scp ${file_full_path} ${ssh_host}:${remote_date_path}
|
|
|
ssh $ssh_host /usr/local/service/hadoop/bin/hadoop fs -put ${remote_file} cosn://applogs/app_error_logs/${date_path}
|
|
|
else
|
|
|
echo `date`":the file ["${file_full_path}"] does not exist"
|
|
|
exit
|
|
|
fi
|
|
|
|
|
|
|
|
|
|
|
|
```
|
|
|
|
|
|
|
|
|
|
|
|
APP ERROR INFO:
|
|
|
http://git.yoho.cn/mobile/yohoanalyticssdk
|
|
|
|
...
|
...
|
|