基于python实现日志收集

#! /usr/bin/python
# encoding:utf-8

import paramiko
import time
import os
import re
import codecs
import commands
from time import  localtime
from datetime import datetime,date

# 执行命令
def sftp_exec_command(command):
    list = []
    try:
        ssh_client = paramiko.SSHClient()
        ssh_client.set_missing_host_key_policy(paramiko.AutoAddPolicy())
        ssh_client.connect(host, 22, user, password)
        std_in, std_out, std_err = ssh_client.exec_command(command)
        for line in std_out:
            list.append(line.strip("\n"))
        ssh_client.close()
        return list
    except Exception, e:
        print e

# 上传文件
def sftp_upload_file(server_path, local_path):
    try:
        t = paramiko.Transport((host, 22))
        t.connect(username=user, password=password)
        sftp = paramiko.SFTPClient.from_transport(t)
        sftp.put(local_path, server_path)
        t.close()
    except Exception, e:
        print e

# 下载文件
def sftp_down_file(server_path, local_path):
    try:
        t = paramiko.Transport((host, 22))
        t.connect(username=user, password=password)
        sftp = paramiko.SFTPClient.from_transport(t)
        sftp.get(server_path, local_path)
        t.close()
    except Exception, e:
        print e

# 读配置文件
def read_conf(loc_conf):
    l = []
    conf = open(loc_conf, 'r');
    for line in conf:
        dic = dict()
        if  line.startswith('host'):
            lst = line.strip().split(',')
            for i in xrange(len(lst)):
                str = lst[i]
                idx = str.index('=')
                key = str[0:idx]
                value = str[idx + 1:]
                dic[key] = value
            l.append(dic)
    return l

#d 读配置文件，返回列表List
conf = read_conf('E:/applog/log.cnf')

# 指定本地保存路径
target_log_dir = "E:/applog/"
today = time.strftime('%Y%m%d')
now = time.strftime('%H%M%S')

# 创建当天目录
local_today = target_log_dir+today
isExists = os.path.exists(local_today)
# cmd_find = "find %s -mtime 0 -name '*'" %log_path
if not isExists:
    os.mkdir(local_today)
    print 'Sucessfully created today dir: %s' %local_today

# 记录日志
log_file = local_today + '/collect.log'
print log_file
collect_log = file(log_file, 'a+')

# if __name__ == '__main__':
    # sftp_exec_command("df -h")
    # sftp_upload_file("/home/oracle/test.html", "D:/doubanhtml/douban0.html")

    # sftp_down_file("/u01/app/oracle/diag/rdbms/orcl/orcl/trace/alert_orcl.log", "D:/alert_orcl.log")

if __name__ == '__main__':
    for a in xrange(len(conf)):
        # 遍历配置文件列表
        host = conf[a]['host']
        port = 22
        timeout = 30
        user = conf[a]['user']
        password = conf[a]['password']
        app_name = conf[a]['name']
        app_path = conf[a]['path']
        # password = raw_input("Input your password plz:")
        # 创建程序目录
        local_app = target_log_dir + today + '/' + app_name

        isExists = os.path.exists(local_app)
        # cmd_find = "find %s -mtime 0 -name '*.trc'" %log_path
        if not isExists:
            os.makedirs(local_app)
            print >>collect_log,'*******Sucessfully created app directory : %s*******' %local_app
            collect_log.write('\n')

        # 将远端日志路径传入app_path_list，进行遍历抓取
        app_path_list = []
        app_path_list.append(app_path)
        for log_path in app_path_list:
            print >>collect_log,'====starting get logs of app:%s->%s->%s====' %(app_name,host,log_path)
            collect_log.write('\n')
            cmd_find = "find %s -newermt %s -name '*.*'" % (log_path,today)
            lt = sftp_exec_command(cmd_find)
            print >>collect_log,'--Sucessfully get logs:'
            for each in lt:
                # 取文件名
                filename = each.split('/')[-1]
                local_app_log = local_app + '/' + filename
                # alert_path_final=alert_path.decode('unicode-escape').encode('utf-8')
                sftp_down_file(each, local_app_log)
                print >>collect_log,each
            collect_log.write('\n')

host=192.168.48.10,user=oracle,password=oracle,path=/u01/app/oracle/diag/rdbms/orcl/orcl/trace,name=ora
host=192.168.48.10,user=oracle,password=oracle,path=/home/oracle,name=ora_home

*******Sucessfully created app directory : E:/applog/20171207/ora*******

====starting get logs of app:ora->192.168.48.10->/u01/app/oracle/diag/rdbms/orcl/orcl/trace====

--Sucessfully get logs:
/u01/app/oracle/diag/rdbms/orcl/orcl/trace/test.trc

*******Sucessfully created app directory : E:/applog/20171207/ora_home*******

====starting get logs of app:ora_home->192.168.48.10->/home/oracle====

--Sucessfully get logs:
/home/oracle/201712061.trc
/home/oracle/.viminfo
/home/oracle/.bash_history
/home/oracle/oracheck/oracheck_171126.log
/home/oracle/20171206.trc