random-testing

#!/usr/bin/env python

# We want to generate a pseudo-random stream of commands to the API. By
# this we mean it generates real commands with the required (or optional)
# attributes, but a mix of real and random values. So we can do something
# useful, we'll eventually move along the standard state transition
# (confirm resize or revert resize will almost always follow a resize
# eventually) but there will likely be other commands in between. There
# is the rare case we'll just "give up" and delete everything.

# We also want to end up in a predetermined end state. In our case, we
# want to end up with an instance that is deleted.

import logging
import optparse
import random
import sys
import threading
import time

import novaclient.exceptions
import novaclient.v1_1.client

import creds

log_file = './debug.log'
formatter = logging.Formatter('%(message)s %(asctime)-15s')
LOG = logging.getLogger(__name__)
file_handler = logging.FileHandler(log_file)
console_handler = logging.StreamHandler()
file_handler.setFormatter(formatter)
#console_handler.setFormatter(formatter)
LOG.addHandler(console_handler)
LOG.addHandler(file_handler)
LOG.setLevel(logging.INFO)

max_servers = 1
tenants = []


def weighted_sample(items, n):
    "Roulette wheel selection"
    results = []
    total = float(sum(w for w, v in items))
    i = 0
    w, v = items[0]
    for n in xrange(n or 1, 0, -1):
        x = total * (1 - random.random() ** (1.0 / n))
        total -= x
        while x > w:
            x -= w
            i += 1
            w, v = items[i]
        w -= x
        results.append(v)

    return results

class Instance(object):
    current_operation = None

    def __init__(self, server):
        self.server = server	# novaclient server object
        self.current_state = server.status
        self.plan = []


class Tenant(object):
    def __init__(self, name):
        self.name = name
        self.creds = creds.users[name]

        self.client = novaclient.v1_1.client.Client(
                self.name,
                self.creds['key'],
                self.creds['tenant'],
                creds.auth_url,
                insecure=True,
                **creds.auth_data)

        self.flavors = self.client.flavors.list()
        self.flavors.sort(lambda a, b: cmp(a.ram, b.ram))

        self.images = self.client.images.list()

        self.instances_by_serverid = {}
        self.instances_by_shortid = {}

    def add_instance(self, instance):
        self.instances_by_serverid[instance.server.id] = instance

        shortid = instance.server.id[:4]
        while shortid in self.instances_by_shortid:
            shortid = instance.server.id[:len(shortid) + 1]

        instance.id = shortid

        self.instances_by_shortid[shortid] = instance


names = ['homer', 'marge', 'bart', 'lisa', 'maggie', 'patty', 'selma',
         'milhouse', 'wendell',  'lewis', 'janey', 'martin', 'sherri',
         'terri', 'ralph', 'nelson', 'lenny', 'carl', 'itchy', 'scratchy',
         'duffman', 'ned', 'rod', 'todd', 'barney', 'kearney', 'krusty',
         'waylon', 'moe', 'willie']


class Operation(object):
    expected = 'state:ACTIVE'

    def __init__(self, tenant, instance=None):
        self.tenant = tenant
        self.instance = instance

    def setup(self):
        pass

    def update(self, result):
        if self.expected != result:
            return 'fail', self.expected

        return 'next', None

    def __str__(self):
        return self.__class__.__name__


class Create(Operation):

    def setup(self):
        # Randomly pick a name
        self.name = random.sample(names, 1)[0]
        # Randomly pick an image
        self.image = random.sample(self.tenant.images, 1)[0]
        # Randomly pick a flavor
        self.flavor = random.sample(self.tenant.flavors, 1)[0]

    def execute(self):
        LOG.info('Creating server %s (image %s, flavor %dMB)' % (self.name,
                self.image.name, self.flavor.ram))
        try:
            server = self.tenant.client.servers.create(image=self.image.id,
                                                       flavor=self.flavor.id,
                                                       name=self.name)
        except:
            msg = ("Unable to create server: \
                    %s (image %s, flavor %dMB)") % (self.name,
                                                    self.image,
                                                    self.flavor.ram)
            LOG.error(msg)

        self.instance = Instance(server)
        self.instance.current_operation = self
        self.tenant.add_instance(self.instance)

        LOG.info('%s: created server %s' % (self.instance.id,
                                         self.instance.server.id))

    def __str__(self):
        return 'Create name=%r, image=%r, flavor=%dMB' % (self.name,
                self.image.id, self.flavor.ram)


class SetPassword(Operation):
    def setup(self):
        self.password = 'foobar'

    def execute(self):
        self.expected = 'state:ACTIVE'
        #msg = "Unable to change password"
        #LOG.error(msg)

    def update(self, result):
        if result == 'state:PASSWORD':
           return 'wait', None

        return super(SetPassword, self).update(result)

    def __str__(self):
        return 'SetPassword password=%r' % self.password


class Resize(Operation):
    def setup(self):
        self.flavor = random.sample(self.tenant.flavors, 1)[0]

    def execute(self):
        if self.instance.server.status != 'ACTIVE':
            # Invalid state
            status = self.instance.server.status
            msg = "Unable to resize server while in %s" % status
            LOG.error(msg)
            self.expected = 'error:409'
        elif self.flavor.id == self.instance.server.flavor['id']:
            # No change in size
            LOG.info("Server is already %dMB" % self.flavor.ram)
            self.expected = 'error:400'
        else:
            self.expected = 'state:VERIFY_RESIZE'

        LOG.info('%s: resize to %dMB' % (self.instance.id, self.flavor.ram))
        self.tenant.client.servers.resize(self.instance.server.id,
                                          self.flavor.id)

    def update(self, result):
        if result == 'state:RESIZE':
           return 'wait', None

        return super(Resize, self).update(result)

    def __str__(self):
        return 'Resize flavor=%dMB' % self.flavor.ram


class ConfirmResize(Operation):
    def execute(self):
        if self.instance.server.status != 'VERIFY_RESIZE':
            self.expected = 'error:409'
            msg = "Cannot confirm resize of server" 
            LOG.error(msg)

        LOG.info('%s: confirm resize' % self.instance.id)
        self.tenant.client.servers.confirm_resize(self.instance.server.id)


class RevertResize(Operation):
    def execute(self):
        if self.instance.server.status != 'VERIFY_RESIZE':
            self.expected = 'error:409'

        LOG.info('%s: revert resize' % self.instance.id)
        self.tenant.client.servers.revert_resize(self.instance.server.id)

    def update(self, result):
        if result == 'state:REVERT_RESIZE':
           return 'wait', None

        return super(RevertResize, self).update(result)


class Rescue(Operation):
    def execute(self):
        if self.instance.server.status != 'ACTIVE':
            # Invalid state
            msg = "Cannot rescue server in %s" % self.instance.server.status
            LOG.error(msg)
            self.expected = 'error:409'
        else:
            self.expected = 'state:RESCUE'

        LOG.info('%s: rescue' % self.instance.id)
        self.tenant.client.servers.rescue(self.instance.server.id)


class Unrescue(Operation):
    def execute(self):
        if self.instance.server.status != 'RESCUE':
            # Invalid state
            msg = "Cannot unrescue instance not in RESCUE"
            LOG.error(msg)
            self.expected = 'error:409'
        else:
            self.expected = 'state:ACTIVE'

        LOG.info('%s: unrescue' % self.instance.id)
        self.tenant.client.servers.unrescue(self.instance.server.id)


class Delete(Operation):
    def execute(self):
        LOG.info('%s: deleting' % self.instance.id)
        self.tenant.client.servers.delete(self.instance.server.id)

    def update(self, result):
        # Deleting from states will move back to ACTIVE while it's being
        # deleted
        if result in ('state:ACTIVE', 'state:DELETED'):
           return 'wait', None

        return super(Delete, self).update(result)


operations = [
    (1, SetPassword),
    (1, Resize),
    (1, ConfirmResize),
    (1, RevertResize),
    (1, Rescue),
    (1, Unrescue),
    (1, Delete),
]


def clean_up(name):
# TODO: delete custom images in the account
# currently the core just deletes the instances from the account and cleans it
# up before starting the os-ran loop

    tenant = Tenant(name)

    for server in tenant.client.servers.list():
        LOG.info('%s %-14s %s' % (server.id, server.status, server.name))
        instance = Instance(server)
        tenant.add_instance(instance)

        instance.current_operation = Delete(tenant, instance)
        instance.current_operation.setup()
        try:
            instance.current_operation.execute()
        except novaclient.exceptions.ClientException as exc:
            LOG.exception(exc);

def tenant_loop(name, endtime):
    # FIXME: Implement personalities? (Blend of images, blend of flavors,
    # auto-disk config, gives up easily, etc)

    tenant = Tenant(name)

    if time.time() > endtime:
        return
    for server in tenant.client.servers.list():
        LOG.info('%s %-14s %s' % (server.id, server.status, server.name))

        instance = Instance(server)
        finish_current_operation(tenant, instance)
        tenant.add_instance(instance)

    while True:
        if time.time() > endtime:
            return

        if not tenant.instances_by_serverid:
            # We can't do anything useful without any instances
            op = Create(tenant)
            op.setup()
            op.execute()

        # Check if any instances have changed state
        workdone = False
        serverids = set(tenant.instances_by_serverid.keys())
        for server in tenant.client.servers.list():
            instance = tenant.instances_by_serverid.get(server.id)
            if not instance:
                # Someone deleting an instance between list() and get()?
                continue

            serverids.remove(server.id)

            if instance.server.status != server.status:
                # Callback to operation for it to determine if this
                # is expected or not
                if instance.current_operation:
                    op = instance.current_operation
                    action, expected = op.update('state:%s' % server.status)
                    if action == 'fail':
                        msg = '%s: UNEXPECTED state change from %s to %s,' \
                              ' expecting %s' % (instance.id,
                                                 instance.server.status,
                                                 server.status,
                                                 expected)
                        LOG.error(msg)
                        if server.status == "ERROR":
                            msg = '%s: Instance fault: %s:%s' % (instance.id,
                                                 server.fault.get('message', ''),
                                                 server.fault.get('code', ''))
                            LOG.error(msg)
                    elif action in ('wait', 'next'):
                        msg = '%s: state change from %s to %s' % \
                                (instance.id, instance.server.status,
                                 server.status)
                        LOG.info(msg)
                        if action == 'next':
                            instance.current_operation = None
                    else:
                        msg = '%s: UNKNOWN action %r' % (instance.id, action)
                        LOG.info(msg)
                        raise ValueError('Unknown state change action %r' %
                                         action)
                else:
                    msg = '%s: state change from %s to %s' % (instance.id,
                          instance.server.status, server.status)
                    LOG.info(msg)


            instance.server = server
            instance.current_state = server.status

            if not instance.current_operation:
                new_operation(tenant, instance)
                workdone = True

        for serverid in serverids:
            instance = tenant.instances_by_serverid[serverid]
            del tenant.instances_by_serverid[serverid]
            del tenant.instances_by_shortid[instance.id]
            LOG.info('%s: no longer exists' % instance.id)

        if not workdone:
            time.sleep(5)

def finish_current_operation(tenant, instance):
    """Allow an instance to finish the current operation before starting a new one"""
    if instance.current_operation:
        return
    if instance.current_state == 'BUILD':
        instance.current_operation = Create(tenant, instance)
    elif instance.current_state == 'RESIZE':
        instance.current_operation = Resize(tenant, instance)
    elif instance.current_state == 'RESCUE':
        instance.current_operation = Rescue(tenant, instance)

def new_operation(tenant, instance):
    """Pick a new operation and do it"""
    op = weighted_sample(operations, 1)[0](tenant, instance)
    op.setup()
    try:
        op.execute()
        instance.current_operation = op
    except novaclient.exceptions.ClientException as exc:
        action, expected = op.update('error:%d' % exc.code)
        if action == 'fail':
            msg = '%s: UNEXPECTED error %d, expecting %s' % \
                    (instance.id, exc.code, expected)
            LOG.exception(msg)
        elif action == 'next':
            msg = '%s: received expected error %d' % \
                    (instance.id, exc.code)
            LOG.info(msg)
        else:
            msg = '%s: UNKNOWN action %r' % (instance.id, action)
            LOG.error(msg)
            raise ValueError('Unknown error action %r' % action)

def tenant_thread(tenant, end_time):
    try:
        tenant_loop(tenant, end_time)
    except Exception as e:
        LOG.info(e)

def parse_options():
    parser = optparse.OptionParser()
    parser.add_option("-n",
                      "--duration",
                      dest="duration",
                      help="Approximate amount of time to run in seconds.",
                      default=600)
    parser.add_option("-w",
                      "--wipe",
                      action="store_true",
                      dest="cleanup_acc",
                      help="Clean up the account before starting the os-ran.")
    parser.add_option("-p",
                      "--parallel",
                      dest="multi_thread",
                      help="Number of threads to run per tenant.",
                      default=1)
    parser.add_option("-a",
                      "--add",
                      action="store_true",
                      dest="add_users",
                      help="Enable addition of multiple users on command line.")
    parser.add_option("-d",
                      "--debug",
                      action="store_true",
                      dest="debugging_info",
                      help="Enable additional debugging information.")
    return parser.parse_args()
    
def multi_thread(end_time, threads_per_tenant=1):
    threads = {}
    myiter=0

    for kee, val in creds.users.iteritems():
        for it in range(int(threads_per_tenant)):
            myiter += 1
            LOG.info("The user for thread %i is %s" % (myiter, kee))
            threads[str(myiter)] = threading.Thread(target=tenant_thread, \
            name=kee, args=(kee, end_time))
            threads[str(myiter)].start()

    for i in range(myiter):
        threads[str(i+1)].join()

def enter_users():
    ans = 'y'

    while ans=='y':
        ans = raw_input("Do you wish to enter more users/tenant? y/N  ")
        if ans=='y' or ans=='Y' or ans=='Yes' or ans=='yes':
            mykey = raw_input("Enter the user name of the tenant  ")
            creds.users[mykey] = {}
            creds.users[mykey]['key'] = raw_input("Please enter the api key for this\
     user  ")
            creds.users[mykey]['tenant'] = raw_input("Please enter the tenant id for\
     this user  ")
            ans = 'y'
        elif ans=='n' or ans=='N' or ans=='No' or ans=='no':
            pass
        else:
            pass

def main():
    options, args = parse_options()
    start_time = time.time()
    end_time = start_time + int(options.duration)

    if options.debugging_info:
        LOG.setLevel(logging.DEBUG)

    LOG.debug('Openstack random was started')

    LOG.info("The users in our db who will run os-ran are:\n")
    for kee, val in creds.users.iteritems():
        LOG.info(kee + '\n')

    if options.add_users==True:
        LOG.info("Adding users to account.")
        enter_users()
    else:
        LOG.info("Not adding users to account.")
        pass

    if options.cleanup_acc==True:
        LOG.info("Account is being wiped.")
        for kee, val in creds.users.iteritems():
            if time.time() > end_time:
                return 0
            clean_up(kee)
    else:
        LOG.info("Account is not being wiped.")
    
    LOG.info("Running %d threads per tenant." % int(options.multi_thread))
    multi_thread(end_time, int(options.multi_thread))


if __name__ == "__main__":
    try:
        sys.exit(main())
    except Exception as e:
        LOG.exception(e);
        LOG.info("os-ran exited")
        sys.exit(1)