性能测试-mongostat监控mongoDB性能

... # 登录至mongo所在服务器
cd  $存放路径
wget https://fastdl.mongodb.org/tools/db/mongodb-database-tools-rhel70-x86_64-100.5.2.rpm
yum install -y cyrus-sasl-gssapi cyrus-sasl-plain
yum install -y  cyrus-sasl
rpm -ivh mongodb-database-tools-rhel70-x86_64-100.5.2.rpm

(base) [root@s mongo_tool]#  mongostat --host localhost:27017 
insert query update delete getmore command dirty  used flushes vsize   res qrw arw net_in net_out conn                time
    *0    *0     *0     *0       0     0|0  0.0% 63.2%       0 4.57G 2.94G 0|0 1|0   111b   43.6k   20 Jan  4 14:58:36.840
    *0    *0     *0     *0       0     1|0  0.0% 63.2%       0 4.57G 2.94G 0|0 1|0   112b   44.3k   20 Jan  4 14:58:37.833

mongostat --host localhost:27017 

mongostat -h 127.0.0.1:27017 -u root -p 123456 --authenticationDatabase admin 

mongostat --noheaders -n 60 60>> test.log

import os
import re
import numpy as np
import pandas as pd
import matplotlib.pylab as plt
from pyecharts.charts import Bar, Page
from pyecharts import options as opts
from pyecharts.globals import ThemeType

def text2list(text: str):
    temp = text.replace("|", " ").replace("*", " ").replace("G", " ").replace("M", " ").replace("b", " ").replace("k", " ")
    return re.split("\s+", temp)

def data_clean(attrs_origin: str, filename: str):
    """获取数据并进行清洗"""
    data = pd.read_table(filename)
    # list = []
    list = []
    for i in data.values:
        # 删除* , 将|转换为空格
        li = text2list(i[0])
        # 删除无用参数, 和attr对齐
        temp = li[1:20]
        temp.append(li[-1])
        list.append(temp)

    data = pd.DataFrame(list, columns=text2list(attrs_origin))
    return data

def series2list(data: pd.DataFrame ,column: str):
    # 获取1列数据
    data_str = list(data[column])
    return list(map(eval, data_str))

if __name__ == '__main__':
    # 数据清洗 : attrs_origin一般不需要动, 指定filename
    attrs_origin = "insert query update delete getmore command|command2 dirty  used flushes vsize  res qr|qw ar|aw net_in net_out conn                time"
    filename = "test_s10.log"
    data = data_clean(attrs_origin, filename)
    # 图表生成
    time_data = list(map(lambda x:x[0:5] ,data['time']))

    # 将文本转化为数字
    insert = series2list(data, "insert")
    query = series2list(data, "query")
    update = series2list(data, "update")
    delete = series2list(data, "delete")
    flushes = series2list(data, "flushes")
    res = series2list(data, "res")
    ar = series2list(data, "ar")
    aw = series2list(data, "aw")
    qr = series2list(data, "qr")
    qw = series2list(data, "qw")
    conn = series2list(data, "conn")

    # 使用pyecharts形成图表

    page = Page(layout=Page.DraggablePageLayout)

    bar1 = (
        Bar(init_opts=opts.InitOpts(theme=ThemeType.LIGHT))
        .add_xaxis(time_data)
        .add_yaxis("insert",  insert)
        .add_yaxis("query",  query)
        .add_yaxis("update",  query)
        .add_yaxis("delete",  query)
        .set_global_opts(title_opts=opts.TitleOpts(title="Mongo增删改查监控图", subtitle="此曲线图反映了Mongo执行增删改查命令的次数变化"))
    )
    bar2 = (
        Bar(init_opts=opts.InitOpts(theme=ThemeType.LIGHT))
        .add_xaxis(time_data)
        .add_yaxis("flushes",  flushes)
        .set_global_opts(title_opts=opts.TitleOpts(title="Mongo flushes监控图", subtitle="此曲线图反映了Mongo缓存数据的频率, 长期为1则可能存在性能问题"))
    )
    bar3 = (
        Bar(init_opts=opts.InitOpts(theme=ThemeType.LIGHT))
        .add_xaxis(time_data)
        .add_yaxis("account read",  ar)
        .add_yaxis("account write",  aw)
        .add_yaxis("queue read,",  qr)
        .add_yaxis("queue write",  qw)
        .set_global_opts(title_opts=opts.TitleOpts(title="Mongo读写队列监控图", subtitle="此曲线图反映了Mongo执行读写操作和等待读写操作的客户端数量, 一直在累计增大则可能存在性能问题"))
    )
    bar4 = (
        Bar(init_opts=opts.InitOpts(theme=ThemeType.LIGHT))
        .add_xaxis(time_data)
        .add_yaxis("conn",  conn)
        .set_global_opts(title_opts=opts.TitleOpts(title="Mongo conn连接监控图", subtitle="此曲线图反映了qr,qw,ar,aw连接总数,建议在5000以下"))
    )
    page.add(bar1, bar2, bar3, bar4)
    page.render("test.html")
    os.system("test.html")