私有云下的自动化故障稳定性测试

将所有用到的工具灌注到docker中制作成镜像
FROM ubuntu:16.04
RUN sed -i s/^deb-src.*// /etc/apt/sources.list
RUN apt-get update && apt-get install --yes sudo python python-pip vim git-core && \

    pip install --upgrade pip && \

    useradd -u 65500 -m rally && \

    usermod -aG sudo rally && \

    echo "rally ALL=(ALL) NOPASSWD:ALL" > /etc/sudoers.d/00-rally-user

COPY ./source /home/rally/source

COPY motd /etc/motd

WORKDIR /home/rally/source

# ensure that we have all system packages installed

RUN pip install bindep &&  apt-get install --yes $(bindep -b | tr '\n' ' ')

RUN pip install -i https://pypi.douban.com/simple . --constraint upper-constraints.txt 
--ignore-installed PyYAML  && \

    pip install -i https://pypi.douban.com/simple pymysql --ignore-installed PyYAML && \

    pip install -i https://pypi.douban.com/simple psycopg2  --ignore-installed PyYAML && \

    pip install os-faults  && \

    pip install rally-runners && \

    mkdir /etc/rally && \

    echo "[database]" > /etc/rally/rally.conf && \

    echo "connection=sqlite:////home/rally/data/rally.db" >> /etc/rally/rally.conf

RUN echo '[ ! -z "$TERM" -a -r /etc/motd ] && cat /etc/motd' >> /etc/bash.bashrc

# Cleanup pip

RUN rm -rf /root/.cache/

USER root

ENV HOME /home/rally

RUN mkdir -p /home/rally/data && rally db recreate

RUN apt-get -y  install software-properties-common  && \

    apt-add-repository ppa:ansible/ansible && \

    apt-get update && \

    apt-get install -y ansible

# Docker volumes have specific behavior that allows this construction to work.

# Data generated during the image creation is copied to volume only when it's

# attached for the first time (volume initialization)

VOLUME ["/home/rally/data"]

---
{% set repeat = repeat|default(3) %}
HttpRequests.ListInstance:
{% for iteration in range(repeat) %}
- args:
    url: http://xx.xxx.xx.xxx/api/resource/listVirtualMachine
    method: POST
    status_code: 200
    json:
      pageSize: 1
      pageNumber: 1
    headers:
      Content-Type: application/json
      charset: UTF-8
  runner:
    type: constant_for_duration
    duration: 60
    concurrency: 1
  context:
    users:
      tenants: 1
      users_per_tenant: 1
  sla:
    failure_rate:
      max: 0
  hooks:
  - name: fault_injection@openstack
    args:
      action: kill mysql service on one node
    trigger:
      name: event
      args:
        unit: iteration
        at:
        - 50
{% endfor %}

from rally.task import contextfrom rally.common import loggingfrom rally import constsfrom rally.plugins.openstack import osclients

LOG = logging.getLogger(__name__)


@context.configure(name="create_flavor", order=1000)class CreateFlavorContext(context.Context):
    """This sample creates a flavor with specified option."""

    CONFIG_SCHEMA = {
        "type": "object",
        "$schema": consts.JSON_SCHEMA,
        "additionalProperties": False,
        "properties": {
            "flavor_name": {
                "type": "string",
            },
            "ram": {
                "type": "integer",
                "minimum": 1
            },
            "vcpus": {
                "type": "integer",
                "minimum": 1
            },
            "disk": {
                "type": "integer",
                "minimum": 1
            }
        }
    }

    def setup(self):
        """This method is called before the task starts."""
        try:
            # use rally.osclients to get necessary client instance
            nova = osclients.Clients(self.context["admin"]["credential"]).nova()
            # and than do what you need with this client
            self.context["flavor"] = nova.flavors.create(
                # context settings are stored in self.config
                name=self.config.get("flavor_name", "rally_test_flavor"),
                ram=self.config.get("ram", 1),
                vcpus=self.config.get("vcpus", 1),
                disk=self.config.get("disk", 1)).to_dict()
            LOG.debug("Flavor with id '%s'" % self.context["flavor"]["id"])
        except Exception as e:
            msg = "Can't create flavor: %s" % e.message
            if logging.is_debug():
                LOG.exception(msg)
            else:
                LOG.warning(msg)

    def cleanup(self):
        """This method is called after the task finishes."""
        try:
            nova = osclients.Clients(self.context["admin"]["credential"]).nova()
            nova.flavors.delete(self.context["flavor"]["id"])
            LOG.debug("Flavor '%s' deleted" % self.context["flavor"]["id"])
        except Exception as e:
            msg = "Can't delete flavor: %s" % e.message
            if logging.is_debug():
                LOG.exception(msg)
            else:
                LOG.warning(msg)

  hooks:
  - name: fault_injection@openstack
    args:
      action: kill mysql service on one node
    trigger:
      name: event
      args:
        unit: iteration
        at:
        - 5

cloud_management:
  driver: universal

node_discover:
  driver: node_list
  args:
  - ip: 10.xxx.xx.14
    auth:
      username: root
      password: *********
      private_key_file: cloud_key
  - ip: 10.xxx.xx.15
    auth:
      username: root
      password: *********
      private_key_file: cloud_key
  - ip: 10.xxx.xx.16
    auth:
      username: root
      password: *********
      private_key_file: cloud_key

power_managements:
- driver: ipmi
  args:
    mac_to_bmc:
      EC:38:8F:7D:40:FF:
        address: 172.xx.xx.118
        username: root
        password: *********
      E8:4D:D0:B3:CC:3D:
        address: 172.xx.xx.119
        username: root
        password: *********
      E8:4D:D0:B3:CA:D0:
        address: 172.xx.xx.120
        username: root
        password: *********

services:
  nova-api:
    driver: system_service
    args:
      service_name: nova-api
      grep: nova-api
  glance-api:
    driver: system_service
    args:
      service_name: glance-api
      grep: glance-api
  identity:
    driver: system_service
    args:
      service_name: identity
      grep: identity
  memcached:
    driver: system_service
    args:
      service_name: memcached
      grep: memcached
  mysql:                 # name of the service
    driver: process      # name of the service driver
    args:                # arguments for the driver
      grep: mysqld
      port:
      - tcp
      - 3306
      restart_cmd: sudo systemctl restart mariadb
      start_cmd: sudo systemctl start mariadb
      terminate_cmd: sudo systemctl stop mariadb
  rabbitmq-server:
    driver: system_service
    args:
      service_name: rabbitmq-server
      grep: rabbitmq-server
    hosts:
    - 172.16.170.13

import random

from rally.task import runnerfrom rally import consts


@runner.configure(name="random_times")class RandomTimesScenarioRunner(runner.ScenarioRunner):
    """Sample scenario runner plugin.

    Run scenario random number of times (between min_times and max_times)    """

    CONFIG_SCHEMA = {
        "type": "object",
        "$schema": consts.JSON_SCHEMA,
        "properties": {
            "type": {
                "type": "string"
            },
            "min_times": {
                "type": "integer",
                "minimum": 1
            },
            "max_times": {
                "type": "integer",
                "minimum": 1
            }
        },
        "additionalProperties": True
    }

    def _run_scenario(self, cls, method_name, context, args):
        # runners settings are stored in self.config
        min_times = self.config.get('min_times', 1)
        max_times = self.config.get('max_times', 1)

        for i in range(random.randrange(min_times, max_times)):
            run_args = (i, cls, method_name,
                        runner._get_scenario_context(context), args)
            result = runner._run_scenario_once(run_args)
            # use self.send_result for result of each iteration
            self._send_result(result)

from rally import constsfrom rally.plugins.openstack import scenariofrom rally.task import atomicfrom rally.task import validation


@validation.add("required_services", services=[consts.Service.NOVA])@validation.add("required_platform", platform="openstack", users=True)@scenario.configure(name="ScenarioPlugin.list_flavors_useless")class ListFlavors(scenario.OpenStackScenario):
    """Sample plugin which lists flavors."""

    @atomic.action_timer("list_flavors")
    def _list_flavors(self):
        """Sample of usage clients - list flavors

        You can use self.context, self.admin_clients and self.clients        which are initialized on scenario instance creation"""
        self.clients("nova").flavors.list()

    @atomic.action_timer("list_flavors_as_admin")
    def _list_flavors_as_admin(self):
        """The same with admin clients"""
        self.admin_clients("nova").flavors.list()

    def run(self):
        """List flavors."""
        self._list_flavors()
        self._list_flavors_as_admin()

import json

from rally.verification import reporter


@reporter.configure("summary-in-json")class SummaryInJsonReporter(reporter.VerificationReporter):
    """Store summary of verification(s) in JSON format"""

    # ISO 8601
    TIME_FORMAT = "%Y-%m-%dT%H:%M:%S%z"

    @classmethod
    def validate(cls, output_destination):
        # we do not have any restrictions for destination, so nothing to
        # check
        pass

    def generate(self):
        report = {}

        for v in self.verifications:
            report[v.uuid] = {
                "started_at": v.created_at.strftime(self.TIME_FORMAT),
                "finished_at": v.updated_at.strftime(self.TIME_FORMAT),
                "status": v.status,
                "run_args": v.run_args,
                "tests_count": v.tests_count,
                "tests_duration": v.tests_duration,
                "skipped": v.skipped,
                "success": v.success,
                "expected_failures": v.expected_failures,
                "unexpected_success": v.unexpected_success,
                "failures": v.failures,
                # v.tests includes all information about launched tests,
                # but for simplification of this fake reporters, let's
                # save just names
                "launched_tests": [test["name"]
                                   for test in v.tests.values()]
            }

        raw_report = json.dumps(report, indent=4)

        if self.output_destination:
            # In case of output_destination existence report will be saved
            # to hard drive and there is nothing to print to stdout, so
            # "print" key is not used
            return {"files": {self.output_destination: raw_report},
                    "open": self.output_destination}
        else:
            # it is something that will be print at CLI layer.
            return {"print": raw_report}

import randomimport re
from rally.verification import manager
# Verification component expects that method "run" of verifier returns# object. Class Result is a simple wrapper for two expected properties.class Result(object):
    def __init__(self, totals, tests):
        self.totals = totals
        self.tests = tests


@manager.configure("fake-tool", default_repo="https://example.com")class FakeTool(manager.VerifierManager):
    """Fake Tool \o/"""

    TESTS = ["fake_tool.tests.bar.FatalityTestCase.test_one",
             "fake_tool.tests.bar.FatalityTestCase.test_two",
             "fake_tool.tests.bar.FatalityTestCase.test_three",
             "fake_tool.tests.bar.FatalityTestCase.test_four",
             "fake_tool.tests.foo.MegaTestCase.test_one",
             "fake_tool.tests.foo.MegaTestCase.test_two",
             "fake_tool.tests.foo.MegaTestCase.test_three",
             "fake_tool.tests.foo.MegaTestCase.test_four"]

    # This fake verifier doesn't launch anything, just returns random
    #  results, so let's override parent methods to avoid redundant
    #  clonning repo, checking packages and so on.

    def install(self):
        pass

    def uninstall(self, full=False):
        pass

    # Each tool, which supports configuration, has the own mechanism
    # for that task. Writing unified method is impossible. That is why
    # `VerificationManager` implements the case when the tool doesn't
    # need (doesn't support) configuration at all. Such behaviour is
    # ideal for FakeTool, since we do not need to change anything :)

    # Let's implement method `run` to return random data.
    def run(self, context):
        totals = {"tests_count": len(self.TESTS),
                  "tests_duration": 0,
                  "failures": 0,
                  "skipped": 0,
                  "success": 0,
                  "unexpected_success": 0,
                  "expected_failures": 0}
        tests = {}
        for name in self.TESTS:
            duration = random.randint(0, 10000)/100.
            totals["tests_duration"] += duration
            test = {"name": name,
                    "status": random.choice(["success", "fail"]),
                    "duration": "%s" % duration}
            if test["status"] == "fail":
                test["traceback"] = "Ooooppps"
                totals["failures"] += 1
            else:
                totals["success"] += 1
            tests[name] = test
        return Result(totals, tests=tests)

    def list_tests(self, pattern=""):
        return [name for name in self.TESTS if re.match(pattern, name)]

因为原生的脚本不匹配与当前版本了，所以我修改了部分代码，调用make_report生成rst文件    

文件内容：

pip install sphinx