APP Error Report Process

APP report error

ios

POST /ios_report.do
Host: error-report.yoho.cn

android

POST /android_report.do
Host: error-report.yoho.cn

1. Write APP report error info to log files

null|null|null|android|null|180c05cb6ef78f508731690851f7e263|ULOGIN|null|null|null|null|20170505153357|50988732|null|null|null|0|6.0|CAM-TL00H|other|0|0|null|VUxPR0lO|8608630377630425317b29953e941d9|1493969638271|2017-05-05 15:33:58.271

2.split logs file to per-hour:

app_error_2017-05-05-11.log
app_error_2017-05-05-12.log

3.create external hive tables

create external table app_error_logs (
time string,
cid string,
body string,
client string,
ec string,
ei string,
et string,
id string,
method string,
response string,
sid string,
ts string,
uid string,
tec string,
cts string,
method_event string,
index string,
osv string,
dm string,
av string,
net string,
ca string,
st string,
exception_type string,
udid string,
collect_time bigint,
collect_time_ts timestamp
)
partitioned by (day string) 
row format delimited 
fields terminated by '|' 
stored as textfile;

4. cron: put files to qcloud cos storage

#!/bin/bash

file_path="/Data/applogs"
file_name_prefix="app_error"

ssh_host="root@10.66.4.133"
remote_path="/data/applogs"

function getFileFullName () {
    file_time=`date -d '-1 hour' +'%Y-%m-%d-%H'`
    file_name=${file_name_prefix}"_"${file_time}".log"
    file_full_path=${file_path}"/"${file_name}
}

getFileFullName

date_path=`date -d '-1 hour' +'%Y-%m-%d'`
if [ -e ${file_full_path} ]
then
    remote_date_path=${remote_path}"/"${date_path}
    if ssh $ssh_host test -e $remote_date_path
    then
       echo `date`":"${remote_date_path}" exists, does not create"
    else
       echo `date`":"${remote_date_path}" does not exist, ready to create"
       ssh $ssh_host "cd ${remote_path}; mkdir ${date_path}"
       ssh $ssh_host /usr/local/service/hadoop/bin/hadoop fs -mkdir cosn://applogs/app_error_logs/${date_path}
       ssh $ssh_host "/usr/local/service/hive/bin/hive -e \"alter table app_error_logs add partition (day='${date_path}') location 'cosn://applogs/app_error_logs/${date_path}'\""
    fi

    remote_file=${remote_date_path}"/"${file_name}
    if ssh $ssh_host test -e $remote_file
    then
        echo `date`":"${remote_file}" exists"
        exit
    fi

    scp ${file_full_path} ${ssh_host}:${remote_date_path}
    ssh $ssh_host /usr/local/service/hadoop/bin/hadoop fs -put ${remote_file} cosn://applogs/app_error_logs/${date_path}
else
    echo  `date`":the file ["${file_full_path}"] does not exist"
    exit
fi

APP ERROR INFO: http://git.yoho.cn/mobile/yohoanalyticssdk