diff --git a/NodeInstance.py b/NodeInstance.py index cfd66dc0..43998660 100644 --- a/NodeInstance.py +++ b/NodeInstance.py @@ -22,6 +22,17 @@ import os, sys, socket, time, libvirt, kazoo.client, threading, fencenode +# ANSII colours for output +class bcolours: + PURPLE = '\033[95m' + BLUE = '\033[94m' + GREEN = '\033[92m' + YELLOW = '\033[93m' + RED = '\033[91m' + ENDC = '\033[0m' + BOLD = '\033[1m' + UNDERLINE = '\033[4m' + class NodeInstance(): def __init__(self, name, t_node, s_domain, zk): # Passed-in variables on creation @@ -87,7 +98,7 @@ class NodeInstance(): # Flush all VMs on the host def flush(self): - print('>>> Flushing node {} of running VMs.'.format(self.name)) + print(bcolours.BLUE + '>>> ' + bcolours.ENDC + 'Flushing node {} of running VMs.'.format(self.name)) for dom_uuid in self.domain_list: most_memfree = 0 target_hypervisor = None @@ -104,12 +115,12 @@ class NodeInstance(): target_hypervisor = hypervisor if target_hypervisor == None: - print('>>> Failed to find migration target for VM "{}"; shutting down.'.format(dom_uuid)) + print(bcolours.RED + '>>> ' + bcolours.ENDC + 'Failed to find migration target for VM "{}"; shutting down.'.format(dom_uuid)) transaction = self.zk.transaction() transaction.set_data('/domains/{}/state'.format(dom_uuid), 'shutdown'.encode('ascii')) transaction.commit() else: - print('>>> Migrating VM "{}" to hypervisor "{}".'.format(dom_uuid, target_hypervisor)) + print(bcolours.BLUE + '>>> ' + bcolours.ENDC + 'Migrating VM "{}" to hypervisor "{}".'.format(dom_uuid, target_hypervisor)) transaction = self.zk.transaction() transaction.set_data('/domains/{}/state'.format(dom_uuid), 'migrate'.encode('ascii')) transaction.set_data('/domains/{}/hypervisor'.format(dom_uuid), target_hypervisor.encode('ascii')) @@ -120,14 +131,14 @@ class NodeInstance(): time.sleep(1) def unflush(self): - print('>>> Restoring node {} to active service.'.format(self.name)) + print(bcolours.BLUE + '>>> ' + bcolours.ENDC + 'Restoring node {} to active service.'.format(self.name)) self.zk.set('/nodes/{}/state'.format(self.name), 'start'.encode('ascii')) for dom_uuid in self.s_domain: last_hypervisor = self.zk.get('/domains/{}/lasthypervisor'.format(dom_uuid))[0].decode('ascii') if last_hypervisor != self.name: continue - print('>>> Setting unmigration for VM "{}".'.format(dom_uuid)) + print(bcolours.BLUE + '>>> ' + bcolours.ENDC + 'Setting unmigration for VM "{}".'.format(dom_uuid)) transaction = self.zk.transaction() transaction.set_data('/domains/{}/state'.format(dom_uuid), 'migrate'.encode('ascii')) transaction.set_data('/domains/{}/hypervisor'.format(dom_uuid), self.name.encode('ascii')) @@ -142,7 +153,7 @@ class NodeInstance(): libvirt_name = "qemu:///system" conn = libvirt.open(libvirt_name) if conn == None: - print('>>> Failed to open connection to {}'.format(libvirt_name)) + print(bcolours.RED + '>>> ' + bcolours.ENDC + 'Failed to open connection to {}'.format(libvirt_name)) return # Get past state and update if needed @@ -193,7 +204,7 @@ class NodeInstance(): conn.close() # Display node information to the terminal - print('>>> {} - {} keepalive'.format(time.strftime('%d/%m/%Y %H:%M:%S'), self.name)) + print(bcolours.PURPLE + '>>> ' + bcolours.ENDC + '{} - {} keepalive'.format(time.strftime('%d/%m/%Y %H:%M:%S'), self.name)) print(' CPUs: {} | Free memory: {} | Load: {}'.format(self.cpucount, self.memfree, self.cpuload)) print(' Active domains: {}'.format(' '.join(self.domain_list))) @@ -210,7 +221,7 @@ class NodeInstance(): # (A node is considered dead when its keepalive timer is >30s out-of-date while in 'start' state) node_deadtime = int(time.time()) - 30 if node_keepalive < node_deadtime and node_state == 'start': - print('>>> Node {} is dead! Performing fence operation in 3 seconds.'.format(node_name)) + print(bcolours.RED + '>>> ' + bcolours.ENDC + 'Node {} is dead! Performing fence operation in 3 seconds.'.format(node_name)) self.zk.set('/nodes/{}/state'.format(node_name), 'dead'.encode('ascii')) fence_thread = threading.Thread(target=fencenode.fence, args=(node_name, self.zk), kwargs={}) fence_thread.start() @@ -248,7 +259,7 @@ class NodeInstance(): pass # Display cluster information to the terminal - print('>>> {} - Cluster status'.format(time.strftime('%d/%m/%Y %H:%M:%S'))) + print(bcolours.PURPLE + '>>> ' + bcolours.ENDC + '{} - Cluster status'.format(time.strftime('%d/%m/%Y %H:%M:%S'))) print(' Active nodes: {}'.format(' '.join(self.active_node_list))) print(' Flushed nodes: {}'.format(' '.join(self.flushed_node_list))) print(' Inactive nodes: {}'.format(' '.join(self.inactive_node_list))) diff --git a/VMInstance.py b/VMInstance.py index e2ab7973..a03b228d 100644 --- a/VMInstance.py +++ b/VMInstance.py @@ -22,6 +22,17 @@ import os, sys, uuid, socket, time, threading, libvirt, kazoo.client +# ANSII colours for output +class bcolours: + PURPLE = '\033[95m' + BLUE = '\033[94m' + GREEN = '\033[92m' + YELLOW = '\033[93m' + RED = '\033[91m' + ENDC = '\033[0m' + BOLD = '\033[1m' + UNDERLINE = '\033[4m' + class VMInstance: def __init__(self, domuuid, zk, thishypervisor): # Passed-in variables on creation @@ -67,52 +78,54 @@ class VMInstance: # Start up the VM def start_vm(self, xmlconfig): - print(">>> {} - Starting VM.".format(self.domuuid)) + print(bcolours.BLUE + '>>> ' + bcolours.ENDC + '{} - Starting VM.'.format(self.domuuid)) self.instart = True # Start up a new Libvirt connection libvirt_name = "qemu:///system" conn = libvirt.open(libvirt_name) if conn == None: - print('>>> {} - Failed to open local libvirt connection.'.format(self.domuuid)) + print(bcolours.RED + '>>> ' + bcolours.ENDC + '{} - Failed to open local libvirt connection.'.format(self.domuuid)) self.instart = False return try: dom = conn.createXML(xmlconfig, 0) except libvirt.libvirtError as e: - print('>>> {} - Failed to create VM.'.format(self.domuuid)) + print(bcolours.RED + '>>> ' + bcolours.ENDC + '{} - Failed to create VM.'.format(self.domuuid)) self.zk.set('/domains/{}/state'.format(self.domuuid), 'stop'.encode('ascii')) if not self.domuuid in self.thishypervisor.domain_list: self.thishypervisor.domain_list.append(self.domuuid) conn.close() + print(bcolours.GREEN + '>>> ' + bcolours.ENDC + '{} - Successfully started VM.'.format(self.domuuid)) self.dom = dom self.instart = False # Stop the VM forcibly without updating state def terminate_vm(self): - print(">>> {} - Terminating VM.".format(self.domuuid)) + print(bcolours.BLUE + '>>> ' + bcolours.ENDC + '{} - Terminating VM.'.format(self.domuuid)) self.instop = True try: self.dom.destroy() except AttributeError: - pass + print(bcolours.RED + '>>> ' + bcolours.ENDC + '{} - Failed to terminate VM.'.format(self.domuuid)) if self.domuuid in self.thishypervisor.domain_list: try: self.thishypervisor.domain_list.remove(self.domuuid) except ValueError: pass + print(bcolours.GREEN + '>>> ' + bcolours.ENDC + '{} - Successfully terminated VM.'.format(self.domuuid)) # Stop the VM forcibly def stop_vm(self): - print(">>> {} - Forcibly stopping VM.".format(self.domuuid)) + print(bcolours.BLUE + '>>> ' + bcolours.ENDC + '{} - Forcibly stopping VM.'.format(self.domuuid)) self.instop = True try: self.dom.destroy() except AttributeError: - pass + print(bcolours.RED + '>>> ' + bcolours.ENDC + '{} - Failed to stop VM.'.format(self.domuuid)) if self.domuuid in self.thishypervisor.domain_list: try: self.thishypervisor.domain_list.remove(self.domuuid) @@ -120,12 +133,13 @@ class VMInstance: pass self.zk.set('/domains/{}/state'.format(self.domuuid), 'stop'.encode('ascii')) + print(bcolours.GREEN + '>>> ' + bcolours.ENDC + '{} - Successfully stopped VM.'.format(self.domuuid)) self.dom = None self.instop = False # Shutdown the VM gracefully def shutdown_vm(self): - print(">>> {} - Gracefully stopping VM.".format(self.domuuid)) + print(bcolours.BLUE + '>>> ' + bcolours.ENDC + '{} - Gracefully stopping VM.'.format(self.domuuid)) self.inshutdown = True self.dom.shutdown() try: @@ -135,7 +149,7 @@ class VMInstance: time.sleep(0.5) if tick >= 60: - print(">>> {} - Shutdown timeout expired.".format(self.domuuid)) + print(bcolours.RED + '>>> ' + bcolours.ENDC + '{} - Shutdown timeout expired.'.format(self.domuuid)) self.stop_vm() self.inshutdown = False return @@ -149,6 +163,7 @@ class VMInstance: pass self.zk.set('/domains/{}/state'.format(self.domuuid), 'stop'.encode('ascii')) + print(bcolours.GREEN + '>>> ' + bcolours.ENDC + '{} - Successfully shutdown VM.'.format(self.domuuid)) self.dom = None self.inshutdown = False @@ -158,14 +173,14 @@ class VMInstance: if dest_conn == None: raise except: - print('>>> {} - Failed to open connection to qemu+tcp://{}/system; aborting migration.'.format(self.dom_uuid, self.hypervisor)) + print(bcolours.RED + '>>> ' + bcolours.ENDC + '{} - Failed to open connection to qemu+tcp://{}/system; aborting migration.'.format(self.dom_uuid, self.hypervisor)) return 1 try: target_dom = self.dom.migrate(dest_conn, libvirt.VIR_MIGRATE_LIVE, None, None, 0) if target_dom == None: raise - print('>>> {} - Migrated successfully.'.format(self.domuuid)) + print(bcolours.GREEN + '>>> ' + bcolours.ENDC + '{} - Migrated successfully.'.format(self.domuuid)) except: dest_conn.close() return 1 @@ -177,10 +192,10 @@ class VMInstance: def migrate_vm(self): self.inmigrate = True - print('>>> {} - Migrating VM to hypervisor "{}".'.format(self.domuuid, self.hypervisor)) + print(bcolours.BLUE + '>>> ' + bcolours.ENDC + '{} - Migrating VM to hypervisor "{}".'.format(self.domuuid, self.hypervisor)) migrate_ret = self.live_migrate_vm(self.hypervisor) if migrate_ret != 0: - print('>>> {} - Could not live migrate VM; shutting down to migrate instead.'.format(self.domuuid)) + print(bcolours.RED + '>>> ' + bcolours.ENDC + '{} - Could not live migrate VM; shutting down to migrate instead.'.format(self.domuuid)) self.shutdown_vm() time.sleep(1) self.zk.set('/domains/{}/state'.format(self.domuuid), 'start'.encode('ascii')) @@ -194,7 +209,7 @@ class VMInstance: # Receive the migration from another host (wait until VM is running) def receive_migrate(self): - print('>>> {} - Receiving migration.'.format(self.domuuid)) + print(bcolours.BLUE + '>>> ' + bcolours.ENDC + '{} - Receiving migration.'.format(self.domuuid)) self.inreceive = True while True: self.dom = lookupByUUID(self.domuuid) @@ -209,7 +224,7 @@ class VMInstance: if not self.domuuid in self.thishypervisor.domain_list: self.thishypervisor.domain_list.append(self.domuuid) - print('>>> {} - Migrated successfully.'.format(self.domuuid)) + print(bcolours.GREEN + '>>> ' + bcolours.ENDC + '{} - Migrated successfully.'.format(self.domuuid)) self.inreceive = False # @@ -274,7 +289,7 @@ def lookupByUUID(tuuid): # Open a libvirt connection conn = libvirt.open(libvirt_name) if conn == None: - print('>>> {} - Failed to open local libvirt connection.'.format(self.domuuid)) + print(bcolours.RED + '>>> ' + bcolours.ENDC + '{} - Failed to open local libvirt connection.'.format(self.domuuid)) return dom # Lookup the UUID diff --git a/fencenode.py b/fencenode.py index 15b1da13..cf910711 100644 --- a/fencenode.py +++ b/fencenode.py @@ -22,16 +22,27 @@ import os, sys, libvirt, uuid, kazoo.client, time +# ANSII colours for output +class bcolours: + PURPLE = '\033[95m' + BLUE = '\033[94m' + GREEN = '\033[92m' + YELLOW = '\033[93m' + RED = '\033[91m' + ENDC = '\033[0m' + BOLD = '\033[1m' + UNDERLINE = '\033[4m' + # # Trigger function # def fence(node_name, zk): time.sleep(3) - print('>>> Fencing node {} via IPMI reboot signal.'.format(node_name)) + print(bcolours.YELLOW + '>>> ' + bcolours.ENDC + 'Fencing node {} via IPMI reboot signal.'.format(node_name)) # DO IPMI FENCE HERE - print('>>> Moving VMs from dead hypervisor {} to new hosts.'.format(node_name)) + print(bcolours.BLUE + '>>> ' + bcolours.ENDC + 'Moving VMs from dead hypervisor {} to new hosts.'.format(node_name)) dead_node_running_domains = zk.get('/nodes/{}/runningdomains'.format(node_name))[0].decode('ascii').split() for dom_uuid in dead_node_running_domains: most_memfree = 0 @@ -47,7 +58,7 @@ def fence(node_name, zk): most_memfree = memfree target_hypervisor = hypervisor - print('>>> Moving VM "{}" to hypervisor "{}".'.format(dom_uuid, target_hypervisor)) + print(bcolours.BLUE + '>>> ' + bcolours.ENDC + 'Moving VM "{}" to hypervisor "{}".'.format(dom_uuid, target_hypervisor)) transaction = zk.transaction() transaction.set_data('/domains/{}/state'.format(dom_uuid), 'start'.encode('ascii')) transaction.set_data('/domains/{}/hypervisor'.format(dom_uuid), target_hypervisor.encode('ascii')) diff --git a/pvcd.py b/pvcd.py index 45e79b4b..f77282d0 100755 --- a/pvcd.py +++ b/pvcd.py @@ -31,8 +31,9 @@ import time import atexit import apscheduler.schedulers.background +# ANSII colours for output class bcolours: - HEADER = '\033[95m' + PURPLE = '\033[95m' BLUE = '\033[94m' GREEN = '\033[92m' YELLOW = '\033[93m' @@ -42,7 +43,7 @@ class bcolours: UNDERLINE = '\033[4m' def help(): - print(bcolours.HEADER + bcolours.BOLD + bcolours.RED + "pvcd - Parallel Virtual Cluster management daemon" + bcolours.ENDC) + print(bcolours.BOLD + "pvcd - Parallel Virtual Cluster management daemon" + bcolours.ENDC) # exit(0) help()