Python自动化部署

# -*- coding: utf-8 -*-
#!/bin/env python
'''
#Auth: karl
#Function: released version
#Date:2017/6/27
#Version:V1.0
'''
import  sys,re,os,time,datetime
import  paramiko
import logging
import socket
import ConfigParser
import traceback
from progressbar import *
import Auto_Mysql_release
import platform
import smtplib
import email.mime.multipart
import email.mime.text
import json
import os
import struct
import requests
receivers = "XXX@.com"
receiver = "XXX@com"
#reg 为0时正常执行命令,为1时开始检查服务启动是否正常,为3时不用再备份原文件
def ssh2(host, port, username, password, cmds,reg=0):
  #  链接远程服务器并执行命令p
    try:
        paramiko.util.log_to_file('./../log/exec_cmd_' + time_str + '.log')
        ssh = paramiko.SSHClient()
        ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())
        ssh.connect(host, int(port), username, password, timeout=5)
        # 执行命令文件中的命令
        if reg == 3:
            stdin, stdout, stderr = ssh.exec_command('ls /tmp/backup/old/ADMIN/integration.properties')
            if stdout.readline() != '':
                pass
            else:
                for cmd in cmds:
                    logging.info('running: ' + cmd)
                    print 'running: ' + cmd
                    time.sleep(2)
                    stdin, stdout, stderr = ssh.exec_command(cmd)
                    for out_msg in stdout.readlines():
                        print out_msg
        else:
            for cmd in cmds:
                print "=======>>",cmd
                logging.info('running: ' + cmd)
                print 'running: ' + cmd
                time.sleep(5)
                stdin, stdout, stderr = ssh.exec_command(cmd)
                if reg == 1:
                    out_msg=str("".join(stdout.readlines()))
                    print "000000------>>",out_msg
                    if "[main] INFO  org.eclipse.jetty.server.Server.doStart(Server.java:379)" in out_msg and "error" not in out_msg and "Failed startup" not in out_msg:
                        print  "THIS IS ok..."
                        print "------->> %s"%(host)
                    else:
                        #回滚到当前版本
#                        repair_cmd_file = conf.get(section, 'repair_cmd_file')
#                        exec_file_cmd(conf, section, repair_cmd_file)
                        copy_cmd_file(conf, section, host,1)
                        for ip_f in host_ip[1:]:
                            num_f = host_ip.index(ip_f)
                            num_f += 1
                            process_cmd_file(conf, section,ip_f, num_f)
                        for ip_s in host_ip:
                            num_s = host_ip.index(ip_s)
                            num_s += 1
                            print "+++++++ %s" % (ip_s)
                            start_server(conf, section, ip_s, num_s)
                        exit(1)
                else:
                    print "++++++++++++++",reg
                    for out_msg in stdout.readlines():
                        print "----", out_msg
            for err_msg in stderr.readlines():
                print err_msg
                exit(1)
        ssh.close()
        print 'command execute successful!'
        logging.info('command execute successful!')
    except Exception, e:
        print '%s\tError\n' % (host)
        print traceback.format_exc()
        __err_exit_show_msg(str(e))
def upload(conf, section):
    '''
    上传文件到远程服务器
    '''
    host = conf.get(section, 'host')
    port = conf.get(section, 'port')
    username = conf.get(section, 'username')
    password = conf.get(section, 'password')
    local_file = conf.get(section, 'local_file')
    remote_file = conf.get(section, 'remote_file')
    try:
        paramiko.util.log_to_file('../log/upload_' + time_str + '.log')
        logging.info('paramiko log created')
        t = paramiko.Transport((host, int(port)))
        t.connect(username=username, password=password)
        logging.info('connected host <' + host + '> successful')
        logging.info('Upload file SUCCESSFUL %s ' % datetime.datetime.now())
        sftp = paramiko.SFTPClient.from_transport(t)
        logging.info('Upload file SUCCESSFUL %s ' % datetime.datetime.now())
        print 'Beginning to upload file to %s  %s ' % (host, datetime.datetime.now())
        # 定义上传进度条样式
        widgets = ['File: ', Percentage(), ' ',
                   Bar(marker='#', left='[', right=']'),
                   ' ', ETA(), ' ', FileTransferSpeed()]
        file_size = os.path.getsize(local_file)
        pbar = ProgressBar(widgets=widgets, maxval=file_size)
        # 开始进度条
        pbar.start()
        # 使用匿名方法接收上传返回值，并且显示进度条
        progress_bar = lambda transferred, toBeTransferred: pbar.update(transferred)
        sftp.put(local_file,remote_file, callback=progress_bar)
        pbar.finish()
        logging.info('Upload file SUCCESSFUL %s ' % datetime.datetime.now())
        print 'Upload file SUCCESSFUL %s ' % datetime.datetime.now()
        t.close()
        logging.info('sftp closed!')
        cmd="tar -xvf %s  -C /home/appdeploy/version/ >/dev/null " % local_file
        if "Linux" == platform.system():
            os.system(cmd)
            report_cmd_file = __checke_conf_key_value_empty(conf, section, 'test_report')
    except Exception, e:
        logging.error('host: <' + host + '> connect error!')
        print host, 'connect error!'
        print traceback.format_exc()
        __err_exit_show_msg(str(e))
def email_send(section,version):
    evn=section
    V=version
    msg = email.mime.multipart.MIMEMultipart()
    msg['Subject'] = '版本发布通知邮件'
    msg['From'] = '18682147058@163.com'
    msg['To'] = ','.join(receivers)
    content = '''
        你好，各位同事：
                    本次%s版本(%s)发布:   发布成功,祝贺！！！
    '''%(evn,V)
    txt = email.mime.text.MIMEText(content)
    msg.attach(txt)
    smtp = smtplib.SMTP()
    smtp.connect('smtp.163.com', '25')
    smtp.login('18682147058@163.com', 'passwd')
    smtp.sendmail(msg['From'], receivers, msg.as_string())
    smtp.quit()
    print('邮件发送成功email has send out !')
def email_linux(receivers, subject=None, bodyhtml=None,attachments=None):
    '''
对接统一通知平台，发邮件样例
receivers 收件人邮箱
subject 主题
bodyhtml 邮件内容
attachment即附件路径默认为空，如有附件传入文件路径'''
    file_name = attachments.split("/")[-1]
    lis = ''
    time_str =str(time.strftime('%Y-%m-%d',time.localtime(time.time())))
    # 把附件内容转换为字符列表
    if attachments != None:
        file_name = os.path.basename(attachments)
        file = open(attachments, 'rb')
        _content = file.read()
        lis = struct.unpack('%db' % len(_content),_content)
    # 对应渠道模板中`message`中参数
    templateJson = {
        'version'  : bodyhtml,
        'time'     :time_str,
        'question' :"详情请审阅附件，谢谢！"
    }
    data = {
        'userId': receivers, #收件人邮箱,支持多收件人,分号隔离;如下面的抄送人格式
        'ccId': receiver, #抄送人,如果没有可以屏蔽该语句
        'templateCode': 'version_release_code', #业务模板code
        'templateParam': templateJson, #如果对应渠道模板中没有类{{}}格式的参数，可以屏蔽该语句
        'subject': subject, #邮件主题，默认是渠道模板名称
        'attachmentName' : file_name, #邮件附件名称， 如果没有附件可以屏蔽该语句
        'attachmentArray': lis, #邮件附件内容,如果没有附件可以屏蔽该语句
        'msgType': 'txt', #消息类型，目前只支持txt
        'accessId': '52HX1CYE', #通知平台接入Id
        'accessToken': 'ebf1dd3140cf4f0abd79872d7d237c3d' #通知平台接入Token
    }
    json_str = json.dumps(data)
    url = "http://public-int-gw.int.sfdc.com.cn:1080/unp/notice/single"
    headers = {'content-type': 'application/json; charset=UTF-8'}
    try:
        response = requests.post(url, data=json_str, headers=headers)
        print response.text
        print "-------------"
        result_json = json.loads(response.text)
        print(result_json) #打印返回内容
    except Exception as e:
        print('调用统一通知平台接口失败:',str(e))
def copy_cmd_file(conf, section, ip,reg):
    if reg == 0:
        filep="/home/appdeploy/version/Version_3.0"
    else:
        filep="/tmp/backup/old"
    host=ip
    port = conf.get(section, 'port')
    username = conf.get(section, 'username')
    password = conf.get(section, 'password')
    remote_file = conf.get(section, 'remote_file')
    print "----------------------------copy files--- %s--------------------" % (host) 
    cmd_c = [
        '\cp -vr {files}/ADMIN/*.war  {path}_ADMIN_01/deploy/webapps/'.format(files=filep, path=Filepath),
        '\cp -vr {files}/ADMIN/integration.properties {path}_ADMIN_01/deploy/resources/'.format(files=filep,path=Filepath),
        '\cp -vr {files}/TRAPP/*.war  {path}_TRAPP_01/deploy/webapps/'.format(files=filep, path=Filepath),
        '\cp -vr {files}/TRAPP/integration.properties  {path}_TRAPP_01/deploy/resources/'.format(files=filep,path=Filepath),
        '\cp -vr {files}/TRTS/*.war  {path}_TRTS_01/deploy/webapps/'.format(files=filep, path=Filepath),
        '\cp -vr {files}/TRTS/integration.properties  {path}_TRTS_01/deploy/resources/'.format(files=filep, path=Filepath),
    ]
    ssh2(host, port, username, password, cmd_c)
#针对脚本进行参数化设置
def process_cmd_file(conf, section,ip_f, num_f):
    host = ip_f
    port = conf.get(section, 'port')
    username = conf.get(section, 'username')
    password = conf.get(section, 'password')
    remote_file = conf.get(section, 'remote_file')
    print "----------------------------copy remote files--- %s--------------------" % (host)
    print "-------->>>>>",host_ip[0],port,username,password,remote_file
    cmd=[
        'scp {path}_ADMIN_01/deploy/webapps/*.war  {user}@{ip}:{path}_ADMIN_0{num}/deploy/webapps/'.format(path=Filepath,user=username,ip=host,num=num_f),
#       'scp {path}_ADMIN_01/deploy/resources/integration.properties  {user}@{ip}:{path}_ADMIN_0{num}/deploy/resources/integration.properties'.format(path=Filepath,user=username,ip=host,num=num_f),
        'scp {path}_TRTS_01/deploy/webapps/*.war  {user}@{ip}:{path}_TRTS_0{num}/deploy/webapps/'.format(path=Filepath,user=username,ip=host,num=num_f),
#       'scp {path}_TRTS_01/deploy/resources/integration.properties  {user}@{ip}:{path}_TRTS_0{num}/deploy/resources/integration.properties'.format(path=Filepath,user=username,ip=host,num=num_f),
        'scp {path}_TRAPP_01/deploy/webapps/*.war  {user}@{ip}:{path}_TRAPP_0{num}/deploy/webapps/'.format(path=Filepath,user=username,ip=host,num=num_f),
#        'scp {path}_TRAPP_01/deploy/resources/integration.properties  {user}@{ip}:{path}_TRAPP_0{num}/deploy/resources/integration.properties'.format(path=Filepath,user=username,ip=host,num=num_f)
    ]
    ssh2(host_ip[0], port, username, password, cmd)
def exec_file_cmd(conf, section, cmd_file):
    '''
    执行文件中的命令
    '''
    host = conf.get(section, 'host')
    port = conf.get(section, 'port')
    username = conf.get(section, 'username')
    password = conf.get(section, 'password')
    cmds = __get_cmds(cmd_file)
    ssh2(host, port, username, password, cmds)
def backup_ori(conf, section):
    '''
    备份远程原文件
    '''
    host = conf.get(section, 'host')
    port = conf.get(section, 'port')
    username = conf.get(section, 'username')
    password = conf.get(section, 'password')
    remote_file = conf.get(section, 'remote_file')
    remote_ori_backup_dir = conf.get(section, 'remote_ori_backup_dir')
    # 获得备份后缀
    suffix_time =  time.strftime('%Y-%m-%d',time.localtime(time.time()))
    backup_ori_cmd = [
        'mkdir -p {dir}/ADMIN {dir}/TRTS {dir}/TRAPP'.format(dir=remote_ori_backup_dir),
        'cp -vr {path}_TRAPP_01/deploy/webapps/*.war  {dir}/TRAPP'.format(path=Filepath,dir=remote_ori_backup_dir),
        'cp -vr {path}_ADMIN_01/deploy/webapps/*.war  {dir}/ADMIN'.format(path=Filepath,dir=remote_ori_backup_dir),
        'cp -vr {path}_TRTS_01/deploy/webapps/*.war  {dir}/TRTS'.format(path=Filepath,dir=remote_ori_backup_dir),
        'cp -vr {path}_TRAPP_01/deploy/resources/integration.properties  {dir}/TRAPP'.format(path=Filepath,dir=remote_ori_backup_dir),
        'cp -vr {path}_ADMIN_01/deploy/resources/integration.properties  {dir}/ADMIN'.format(path=Filepath,dir=remote_ori_backup_dir),
        'cp -vr {path}_TRTS_01/deploy/resources/integration.properties  {dir}/TRTS'.format(path=Filepath,dir=remote_ori_backup_dir)
    ]
    ssh2(host, port, username, password, backup_ori_cmd,3)
def backup_new(conf, section):
    '''
    备份远程新上传的文件
    '''
    host = conf.get(section, 'host')
    port = conf.get(section, 'port')
    username = conf.get(section, 'username')
    password = conf.get(section, 'password')
    remote_file = conf.get(section, 'remote_file')
    remote_backup_dir = conf.get(section, 'remote_backup_dir')
    # 获得备份后缀
    suffix_time = time.strftime('%Y-%m-%d_%H-%M-%S',time.localtime(time.time()))
    backup_new_cmd = [
        'mkdir -p {dir}'.format(dir=remote_backup_dir),
        'cp -vr {new_file} {dir}/{new_bak_file}_{time}'.format(new_file=remote_file,
                                                           dir=remote_backup_dir,
                                                           new_bak_file=os.path.basename(remote_file),
                                                           time=str(suffix_time))
    ]
    ssh2(host, port, username, password, backup_new_cmd)
def select_section(conf_file_name):
    '''
    选择指定读取的配置文件项
    例如：*.conf配置文件中有多个配置项 a 和 b:
      [a]
      xxxxx
      [b]
      yyyyy
    '''
    # 检测指定的配置文件是否存在
    __check_file_exists(conf_file_name)
    # 读取配置文件
    conf = ConfigParser.ConfigParser()
    conf.read(conf_file_name)
    sections = conf.sections()
    # 选择配置文件选项界面
    print 'please choose confit item:'
    for index, value in enumerate(sections):
        print  '  ', index, ':', value
    while True:
        sec_index = raw_input('please choose one item default [0]:')
        if not sec_index.isdigit() or int(sec_index) >= len(sections):
            print 'choose invalid!'
            continue
        return conf, sections[int(sec_index)]
        return conf, sections[0]
def check_config(conf, section):
    '''
    检测配置文件的正确性
    '''
    logging.info('check config starting...')
    print 'check config starting...'
    # 检测配置文件中值是否都填写
    host = __checke_conf_key_value_empty(conf, section, 'host')  # 检测配置文件中主机名
    port = __checke_conf_key_value_empty(conf, section, 'port')  # 检测配置文件中端口
    username = __checke_conf_key_value_empty(conf, section, 'username')  # 检测配置文件用户名
    password = __checke_conf_key_value_empty(conf, section, 'password')  # 检测配置文件密码
    local_file = __checke_conf_key_value_empty(conf, section, 'local_file')  # 检测配置文件本地需要上传文件
    remote_file = __checke_conf_key_value_empty(conf, section, 'remote_file')  # 检测配置文件上传到远程的文件
    remote_backup_dir = __checke_conf_key_value_empty(conf, section, 'remote_backup_dir')  # 检测配置文件远程备份目录
    remote_ori_backup_dir = __checke_conf_key_value_empty(conf, section, 'remote_ori_backup_dir')  # 检测配置文件远程临时备份目录
    start_cmd_file = __checke_conf_key_value_empty(conf, section, 'start_cmd_file')  # 检测配置文件启动服务文件
    report_cmd_file = __checke_conf_key_value_empty(conf, section, 'test_report')  # 检测配置文件停止服务文件
    # 检测配置文件中的网络是否可用
    __check_network_ping(host)
    # 检测ssh链接是否成功
    __check_ssh(host, int(port), username, password)
    # 检测本地需要上传的文件是否存在
    __check_file_exists(local_file)
    # 检测命令文件是否存在
    __check_file_exists(start_cmd_file)
    print 'check config successful!!'
    logging.info('check config successful!!')
def __valid_ip(address):
    '''
    检测IP是否合法IP
    '''
    try:
        socket.inet_aton(address)
        return True
    except:
        print traceback.format_exc()
        return False
def __check_file_exists(conf_file_name):
    '''
    检测指定的配置文件是否存在
    '''
    if not os.path.exists(conf_file_name):
        logging.error('can not find config file: ' + conf_file_name)
        __err_exit_show_msg('can not find config file: ' + conf_file_name)
    return conf_file_name
def __checke_conf_key_value_empty(conf, section, key):
    '''
    检测配置文件的key是否存在
    '''
    try:
        value = conf.get(section, key)
        # 检测配置文件中的值是否为空
        if value:
            return value
        else:
            msg = '''
      ERROR  The key:{key} value is empty in conf file
      '''.format(key=key)
            __err_exit_show_msg(msg)
    except ConfigParser.NoOptionError:
        print traceback.format_exc()
        msg = '''
      ERROR  cannot find key:{key} in conf file
    '''.format(key=key)
        __err_exit_show_msg(msg)
def __check_network_ping(host):
    if not __valid_ip(host):
        __err_exit_show_msg('host: ' + host + ' invalid')
    if "Linux" == platform.system():
        if 0 <> os.system('ping -c 3 ' + host):
            __err_exit_show_msg('host: ' + host + ' cannot ping...')
    else:
        if 0 <> os.system('ping -n 1 -w 5 ' + host):
            __err_exit_show_msg('host: ' + host + ' cannot ping...')
def __check_ssh(host, port, username, password):
    try:
        ssh = paramiko.SSHClient()
        ssh.set_missing_host_key_policy(paramiko.AutoAddPolicy())
        ssh.connect(host, port, username, password, timeout=5)
        ssh.close()
    except Exception as e:
        print traceback.format_exc()
        msg = '''
    SSH connect failure. 
    please check your host/port/username/password
    host    :  {host}
    port    :  {port}
    username:  {username}
    password:  {password}
    '''.format(host=host, port=port,
               username=username,
               password=password)
        __err_exit_show_msg(msg)
def __get_cmds(cmd_file):
    '''
    文件中获取执行命令
    '''
    with open(cmd_file, 'r') as cmd_f:
        pattern = re.compile('(^\s*#|^\s*$)')
        func = lambda x: x if not re.match(pattern, x) else None
        cmds = [cmd for cmd in cmd_f]
        return filter(func, cmds)
def check_server(conf, section,ip,num):
    host = ip
    port = conf.get(section, 'port')
    username = conf.get(section, 'username')
    password = conf.get(section, 'password')
    suffix_time = time.strftime('%Y%m%d', time.localtime(time.time()))
    print "----------------------------check result--- %s--------------------"%(ip)
    checkserver_cmd=[
    'cat {path}_ADMIN_0{num}/logs/novatar_{time}.0.log |grep  "error\|org.eclipse.jetty.server.Server.doStart(Server.java:379)\|Failed startup"'.format(path=Filepath,time=suffix_time,num=num),
    'cat {path}_TRAPP_0{num}/logs/novatar_{time}.0.log |grep  "error\|org.eclipse.jetty.server.Server.doStart(Server.java:379)\|Failed startup"'.format(path=Filepath,time=suffix_time,num=num),
    'cat {path}_TRTS_0{num}/logs/novatar_{time}.0.log |grep  "error\|org.eclipse.jetty.server.Server.doStart(Server.java:379)\|Failed startup"'.format(path=Filepath,time=suffix_time,num=num)
    ]
    ssh2(host, port, username, password, checkserver_cmd,1)
def start_server(conf, section, ip, num):
    host = ip
    port = conf.get(section, 'port')
    username = conf.get(section, 'username')
    password = conf.get(section, 'password')
    print "----------------------------restart server --- %s--------------------" % (ip)
    checkserver_cmd = [
     'sh {path}_ADMIN_0{num}_run.sh stop; sh {path}_ADMIN_0{num}_run.sh start'.format(path=Restartfile,num=num),
     'sh {path}_TRAPP_0{num}_run.sh stop; sh {path}_TRAPP_0{num}_run.sh start'.format(path=Restartfile,num=num),
     'sh {path}_TRTS_0{num}_run.sh stop; sh {path}_TRTS_0{num}_run.sh start'.format(path=Restartfile,num=num)
    ]
    ssh2(host, port, username, password, checkserver_cmd)
def __err_exit_show_msg(msg):
    '''
    发生错误的时候显示相关错误信息并且退出程序
    '''
    print 'ERROR:  ' + msg
    logging.error('ERROR:  ' + msg)
    os.system('pause')
    sys.exit()
if __name__ == '__main__':
    try:
        start = time.clock()
        # 设置日志文件
        time_str = time.strftime('%Y-%m-%d',time.localtime(time.time()))
        log_file = '../log/upload_distribution_' + str(time_str) + '.log'
        logging.basicConfig(level=logging.INFO,
                            format='%(asctime)s %(name)-12s %(levelname)-8s %(message)s',
                            filename=log_file,
                            filemode='w',
                            datefmt='%Y-%m-%d %X')
        # 定义配置文件路径
        conf_file_name = '../conf/release.conf'
        # 选择配置文件section
        conf, section = select_section(conf_file_name)
        if section == "STG":
            Filepath="/app/jetty/server/SCS_ATP_CNSZ99_JETTY_APP"
            Restartfile="/app/jetty/logs/SCS_ATP_CNSZ99_JETTY_APP" 
            host_ip=[]
        elif section == "DEVTEST":
            Filepath="/app/jetty/server/SCS_ATP_CORE_CNSZ22_JETTY_APP"
            Restartfile="/app/jetty/logs/SCS_ATP_CORE_CNSZ22_JETTY_APP"
            host_ip=[]
        else :
            Filepath="/app/jetty/server/SCS_ATP_CORE_CNSZ17_JETTY_APP"
            Restartfile="/app/jetty/logs/SCS_ATP_CNSZ17_JETTY_APP"
            host_ip=[]
        # 检测配置文件正确性
        check_config(conf, section)
        print('\033[1;35m You can view the configuration file in the version number:\"/home/appdeploy/version/Version_\033[1;32m3.0\033[1;35m.tar" \033[0m!')
        VERSION=raw_input("Please Enter Version: eg 3.0 \n")
        # 备份原文件
        backup_ori(conf, section)
        # 上传文件
        upload(conf, section)
        # 备份新上传的文件
        backup_new(conf, section)
        # 解压文件
        start_cmd_file = conf.get(section, 'start_cmd_file')
        exec_file_cmd(conf, section, start_cmd_file)
        # 上传新文件到应用目录
        copy_cmd_file(conf, section, host_ip[0],0)
        # 拷贝文件到远程服务器
        if len(host_ip) > 1:
            for ip_f in host_ip[1:]:
                num_f = host_ip.index(ip_f)
                num_f += 1
                process_cmd_file(conf, section,ip_f, num_f)
        # 执行拷贝excel 等文件,记录到数据库中
        agre=["3306", "XXX", "root",host_ip[0],VERSION]
        print "agre :",agre
        stg=Auto_Mysql_release.Mysql_connect(*agre)
        stg.File_get()
        stg.File_deal()
        #启动所有服务器的服务
        for ip_s in host_ip:
            num_s = host_ip.index(ip_s)
            num_s += 1
            start_server(conf, section,ip_s,num_s)
        # 监听服务是否启动成功
        time.sleep(90)
        for ip in host_ip:
            num=host_ip.index(ip)
            num+=1
            check_server(conf, section,ip,num)
        # 实行完毕退出
        cmd_f = ["rm -fr /tmp/backup/old/*"]
        ssh2(host_ip[0],"22","mwopr","XXX",cmd_f)
        os.system('exit')
        end = time.clock()
        print "version release SUCCESSFULL.......共耗时%f  s" %(end - start)
        # 邮件群发到相关责任人
        if section == "ONLINE":
            testReport = conf.get(section, 'test_report')
            text="%s版本发布通知邮件"% (VERSION)
            email_linux(receivers,"版本发布通知邮件",VERSION,testReport)
        elif section == "DEVTEST":
            email_send(section,VERSION)
        else:
            pass
        exit(0)
    except Exception as e:
        print traceback.format_exc()
        __err_exit_show_msg(str(e))