linux 超大日志数据分析 AWK

#!/bin/bash
///
./flowdata.log
2017-02-02 15:29:19,390 [views:111:ebitpost] [INFO]- ebitapi: http://218.85.118.8:8000/api/user/query, ebit response: src_ip: 110.86.101.119:63688, content: {"data":{"basic_rate_down":20480,"basic_rate_up":2048,"dial_acct":"fj::059391534153","max_linerate_down":102400,"max_linerate_up":102400},"message":"提速判断成功","result":0}
///
///
./ipdb_cn.txt
1.1.1.0  中国 广东 深圳 
1.1.2.0  中国 广东 深圳
...
233.233.2.0  中国 新疆 乌鲁木齐
///


grep -Po 'src_ip: ?\K[^:,]+|client_ip: ?\K[^:,]+' flowdata.log |awk -F. '!a[$0]++{$4=0;print}' OFS=. | awk 'FNR==NR{a[$1]=$3;b[$1]=1;next}b[$1]{s[a[$1]]++;next}{s["未知"]++}END{for(i in s) printf "%s\t%.2f%\n",i,s[i]/FNR*100}' ipdb_cn.txt - |column -t |sort -k2nr





grep -Po 'src_ip: ?|\K[^:,]+|client_ip: ?\K[^:,]+' flowdata.log |awk -F. '!a[$0]++{b[$1FS$2FS$3]++}END{for(i in b)print i FS 0,b[i]}' |awk 'FNR==NR{a[$1]=$3;next}{a[$1]!=""?s[a[$1]]+=$2:s["未知"]+=$2;n+=$2}END{for(i in s)printf "%s\t%.2f%\n",i,s[i]/n*100}' ipdb_cn.txt |sort -k2n |column -t