Compare commits
23 Commits
Author | SHA1 | Date | |
---|---|---|---|
afdf254297 | |||
42e776fac1 | |||
dae67a1b7b | |||
b86f8c1e09 | |||
13e309b450 | |||
7ecc6a2635 | |||
73e8149cb0 | |||
4a7246b8c0 | |||
c49351469b | |||
dc03e95bbf | |||
c460aa051a | |||
3ab6365a53 | |||
32613ff119 | |||
2a99a27feb | |||
45f23c12ea | |||
fa1d93e933 | |||
b14bc7e3a3 | |||
4d6842f942 | |||
6ead21a308 | |||
b7c8c2ee3d | |||
d48f58930b | |||
7c36388c8f | |||
e9df043c0a |
35
README.md
35
README.md
@ -40,8 +40,43 @@ The core node and API daemons, as well as the CLI API client, are written in Pyt
|
||||
To get started with PVC, please see the [About](https://parallelvirtualcluster.readthedocs.io/en/latest/about/) page for general information about the project, and the [Getting Started](https://parallelvirtualcluster.readthedocs.io/en/latest/getting-started/) page for details on configuring your first cluster.
|
||||
|
||||
|
||||
## Screenshots
|
||||
|
||||
While PVC's API and internals aren't very screenshot-worthy, here is some example output of the CLI tool.
|
||||
|
||||
<p><img alt="Node listing" src="docs/images/pvc-nodes.png"/><br/><i>Listing the nodes in a cluster</i></p>
|
||||
|
||||
<p><img alt="Network listing" src="docs/images/pvc-networks.png"/><br/><i>Listing the networks in a cluster, showing 3 bridged and 1 IPv4-only managed networks</i></p>
|
||||
|
||||
<p><img alt="VM listing and migration" src="docs/images/pvc-migration.png"/><br/><i>Listing a limited set of VMs and migrating one with status updates</i></p>
|
||||
|
||||
<p><img alt="Node logs" src="docs/images/pvc-nodelog.png"/><br/><i>Viewing the logs of a node (keepalives and VM [un]migration)</i></p>
|
||||
|
||||
|
||||
## Changelog
|
||||
|
||||
#### v0.9.32
|
||||
|
||||
* [CLI Client] Fixes some incorrect colours in network lists
|
||||
* [Documentation] Adds documentation screenshots of CLI client
|
||||
* [Node Daemon] Fixes a bug if VM stats gathering fails
|
||||
|
||||
#### v0.9.31
|
||||
|
||||
* [Packages] Cleans up obsolete Suggests lines
|
||||
* [Node Daemon] Adjusts log text of VM migrations to show the correct source node
|
||||
* [API Daemon] Adjusts the OVA importer to support floppy RASD types for compatability
|
||||
* [API Daemon] Ensures that volume resize commands without a suffix get B appended
|
||||
* [API Daemon] Removes the explicit setting of image-features in PVC; defaulting to the limited set has been moved to the ceph.conf configuration on nodes via PVC Ansible
|
||||
|
||||
#### v0.9.30
|
||||
|
||||
* [Node Daemon] Fixes bug with schema validation
|
||||
|
||||
#### v0.9.29
|
||||
|
||||
* [Node Daemon] Corrects numerous bugs with node logging framework
|
||||
|
||||
#### v0.9.28
|
||||
|
||||
* [CLI Client] Revamp confirmation options for "vm modify" command
|
||||
|
@ -25,7 +25,7 @@ import yaml
|
||||
from distutils.util import strtobool as dustrtobool
|
||||
|
||||
# Daemon version
|
||||
version = '0.9.28'
|
||||
version = '0.9.32'
|
||||
|
||||
# API version
|
||||
API_VERSION = 1.0
|
||||
|
@ -414,6 +414,7 @@ class OVFParser(object):
|
||||
"5": "ide-controller",
|
||||
"6": "scsi-controller",
|
||||
"10": "ethernet-adapter",
|
||||
"14": "floppy",
|
||||
"15": "cdrom",
|
||||
"17": "disk",
|
||||
"20": "other-storage-device",
|
||||
|
@ -491,14 +491,10 @@ def net_sriov_vf_info(config, node, vf):
|
||||
# Output display functions
|
||||
#
|
||||
def getColour(value):
|
||||
if value in ['True', "start"]:
|
||||
return ansiprint.green()
|
||||
elif value in ["restart", "shutdown"]:
|
||||
return ansiprint.yellow()
|
||||
elif value in ["stop", "fail"]:
|
||||
return ansiprint.red()
|
||||
else:
|
||||
if value in ["False", "None"]:
|
||||
return ansiprint.blue()
|
||||
else:
|
||||
return ansiprint.green()
|
||||
|
||||
|
||||
def getOutputColours(network_information):
|
||||
|
@ -2,7 +2,7 @@ from setuptools import setup
|
||||
|
||||
setup(
|
||||
name='pvc',
|
||||
version='0.9.28',
|
||||
version='0.9.32',
|
||||
packages=['pvc', 'pvc.cli_lib'],
|
||||
install_requires=[
|
||||
'Click',
|
||||
|
@ -491,7 +491,7 @@ def add_volume(zkhandler, pool, name, size):
|
||||
size = '{}B'.format(size)
|
||||
|
||||
# 2. Create the volume
|
||||
retcode, stdout, stderr = common.run_os_command('rbd create --size {} --image-feature layering,exclusive-lock {}/{}'.format(size, pool, name))
|
||||
retcode, stdout, stderr = common.run_os_command('rbd create --size {} {}/{}'.format(size, pool, name))
|
||||
if retcode:
|
||||
return False, 'ERROR: Failed to create RBD volume "{}": {}'.format(name, stderr)
|
||||
|
||||
@ -536,6 +536,10 @@ def resize_volume(zkhandler, pool, name, size):
|
||||
if not verifyVolume(zkhandler, pool, name):
|
||||
return False, 'ERROR: No volume with name "{}" is present in pool "{}".'.format(name, pool)
|
||||
|
||||
# Add 'B' if the volume is in bytes
|
||||
if re.match(r'^[0-9]+$', size):
|
||||
size = '{}B'.format(size)
|
||||
|
||||
# 1. Resize the volume
|
||||
retcode, stdout, stderr = common.run_os_command('rbd resize --size {} {}/{}'.format(size, pool, name))
|
||||
if retcode:
|
||||
|
@ -23,6 +23,7 @@ from collections import deque
|
||||
from threading import Thread
|
||||
from queue import Queue
|
||||
from datetime import datetime
|
||||
from time import sleep
|
||||
|
||||
from daemon_lib.zkhandler import ZKHandler
|
||||
|
||||
@ -83,7 +84,8 @@ class Logger(object):
|
||||
self.last_prompt = ''
|
||||
|
||||
if self.config['zookeeper_logging']:
|
||||
self.zookeeper_logger = ZookeeperLogger(config)
|
||||
self.zookeeper_queue = Queue()
|
||||
self.zookeeper_logger = ZookeeperLogger(self.config, self.zookeeper_queue)
|
||||
self.zookeeper_logger.start()
|
||||
|
||||
# Provide a hup function to close and reopen the writer
|
||||
@ -96,9 +98,15 @@ class Logger(object):
|
||||
if self.config['file_logging']:
|
||||
self.writer.close()
|
||||
if self.config['zookeeper_logging']:
|
||||
self.out("Waiting for Zookeeper message queue to drain", state='s')
|
||||
while not self.zookeeper_logger.queue.empty():
|
||||
pass
|
||||
self.out("Waiting 15s for Zookeeper message queue to drain", state='s')
|
||||
|
||||
tick_count = 0
|
||||
while not self.zookeeper_queue.empty():
|
||||
sleep(0.5)
|
||||
tick_count += 1
|
||||
if tick_count > 30:
|
||||
break
|
||||
|
||||
self.zookeeper_logger.stop()
|
||||
self.zookeeper_logger.join()
|
||||
|
||||
@ -145,7 +153,7 @@ class Logger(object):
|
||||
|
||||
# Log to Zookeeper
|
||||
if self.config['zookeeper_logging']:
|
||||
self.zookeeper_logger.queue.put(message)
|
||||
self.zookeeper_queue.put(message)
|
||||
|
||||
# Set last message variables
|
||||
self.last_colour = colour
|
||||
@ -157,19 +165,14 @@ class ZookeeperLogger(Thread):
|
||||
Defines a threaded writer for Zookeeper locks. Threading prevents the blocking of other
|
||||
daemon events while the records are written. They will be eventually-consistent
|
||||
"""
|
||||
def __init__(self, config):
|
||||
def __init__(self, config, zookeeper_queue):
|
||||
self.config = config
|
||||
self.node = self.config['node']
|
||||
self.max_lines = self.config['node_log_lines']
|
||||
self.queue = Queue()
|
||||
self.zkhandler = None
|
||||
self.start_zkhandler()
|
||||
# Ensure the root keys for this are instantiated
|
||||
self.zkhandler.write([
|
||||
('base.logs', ''),
|
||||
(('logs', self.node), '')
|
||||
])
|
||||
self.zookeeper_queue = zookeeper_queue
|
||||
self.connected = False
|
||||
self.running = False
|
||||
self.zkhandler = None
|
||||
Thread.__init__(self, args=(), kwargs=None)
|
||||
|
||||
def start_zkhandler(self):
|
||||
@ -179,10 +182,29 @@ class ZookeeperLogger(Thread):
|
||||
self.zkhandler.disconnect()
|
||||
except Exception:
|
||||
pass
|
||||
self.zkhandler = ZKHandler(self.config, logger=None)
|
||||
self.zkhandler.connect(persistent=True)
|
||||
|
||||
while True:
|
||||
try:
|
||||
self.zkhandler = ZKHandler(self.config, logger=None)
|
||||
self.zkhandler.connect(persistent=True)
|
||||
break
|
||||
except Exception:
|
||||
sleep(0.5)
|
||||
continue
|
||||
|
||||
self.connected = True
|
||||
|
||||
# Ensure the root keys for this are instantiated
|
||||
self.zkhandler.write([
|
||||
('base.logs', ''),
|
||||
(('logs', self.node), '')
|
||||
])
|
||||
|
||||
def run(self):
|
||||
while not self.connected:
|
||||
self.start_zkhandler()
|
||||
sleep(1)
|
||||
|
||||
self.running = True
|
||||
# Get the logs that are currently in Zookeeper and populate our deque
|
||||
raw_logs = self.zkhandler.read(('logs.messages', self.node))
|
||||
@ -192,7 +214,7 @@ class ZookeeperLogger(Thread):
|
||||
while self.running:
|
||||
# Get a new message
|
||||
try:
|
||||
message = self.queue.get(timeout=1)
|
||||
message = self.zookeeper_queue.get(timeout=1)
|
||||
if not message:
|
||||
continue
|
||||
except Exception:
|
||||
@ -205,8 +227,21 @@ class ZookeeperLogger(Thread):
|
||||
date = ''
|
||||
# Add the message to the deque
|
||||
logs.append(f'{date}{message}')
|
||||
# Write the updated messages into Zookeeper
|
||||
self.zkhandler.write([(('logs.messages', self.node), '\n'.join(logs))])
|
||||
|
||||
tick_count = 0
|
||||
while True:
|
||||
try:
|
||||
# Write the updated messages into Zookeeper
|
||||
self.zkhandler.write([(('logs.messages', self.node), '\n'.join(logs))])
|
||||
break
|
||||
except Exception:
|
||||
# The write failed (connection loss, etc.) so retry for 15 seconds
|
||||
sleep(0.5)
|
||||
tick_count += 1
|
||||
if tick_count > 30:
|
||||
break
|
||||
else:
|
||||
continue
|
||||
return
|
||||
|
||||
def stop(self):
|
||||
|
@ -777,7 +777,7 @@ class ZKSchema(object):
|
||||
logger.out(f'Key not found: {self.path(kpath)}', state='w')
|
||||
result = False
|
||||
|
||||
for elem in ['logs', 'node', 'domain', 'network', 'osd', 'pool']:
|
||||
for elem in ['node', 'domain', 'network', 'osd', 'pool']:
|
||||
# First read all the subelements of the key class
|
||||
for child in zkhandler.zk_conn.get_children(self.path(f'base.{elem}')):
|
||||
# For each key in the schema for that particular elem
|
||||
@ -856,7 +856,7 @@ class ZKSchema(object):
|
||||
data = ''
|
||||
zkhandler.zk_conn.create(self.path(kpath), data.encode(zkhandler.encoding))
|
||||
|
||||
for elem in ['logs', 'node', 'domain', 'network', 'osd', 'pool']:
|
||||
for elem in ['node', 'domain', 'network', 'osd', 'pool']:
|
||||
# First read all the subelements of the key class
|
||||
for child in zkhandler.zk_conn.get_children(self.path(f'base.{elem}')):
|
||||
# For each key in the schema for that particular elem
|
||||
|
30
debian/changelog
vendored
30
debian/changelog
vendored
@ -1,3 +1,33 @@
|
||||
pvc (0.9.32-0) unstable; urgency=high
|
||||
|
||||
* [CLI Client] Fixes some incorrect colours in network lists
|
||||
* [Documentation] Adds documentation screenshots of CLI client
|
||||
* [Node Daemon] Fixes a bug if VM stats gathering fails
|
||||
|
||||
-- Joshua M. Boniface <joshua@boniface.me> Thu, 19 Aug 2021 12:37:58 -0400
|
||||
|
||||
pvc (0.9.31-0) unstable; urgency=high
|
||||
|
||||
* [Packages] Cleans up obsolete Suggests lines
|
||||
* [Node Daemon] Adjusts log text of VM migrations to show the correct source node
|
||||
* [API Daemon] Adjusts the OVA importer to support floppy RASD types for compatability
|
||||
* [API Daemon] Ensures that volume resize commands without a suffix get B appended
|
||||
* [API Daemon] Removes the explicit setting of image-features in PVC; defaulting to the limited set has been moved to the ceph.conf configuration on nodes via PVC Ansible
|
||||
|
||||
-- Joshua M. Boniface <joshua@boniface.me> Fri, 30 Jul 2021 12:08:12 -0400
|
||||
|
||||
pvc (0.9.30-0) unstable; urgency=high
|
||||
|
||||
* [Node Daemon] Fixes bug with schema validation
|
||||
|
||||
-- Joshua M. Boniface <joshua@boniface.me> Tue, 20 Jul 2021 00:01:45 -0400
|
||||
|
||||
pvc (0.9.29-0) unstable; urgency=high
|
||||
|
||||
* [Node Daemon] Corrects numerous bugs with node logging framework
|
||||
|
||||
-- Joshua M. Boniface <joshua@boniface.me> Mon, 19 Jul 2021 16:55:41 -0400
|
||||
|
||||
pvc (0.9.28-0) unstable; urgency=high
|
||||
|
||||
* [CLI Client] Revamp confirmation options for "vm modify" command
|
||||
|
1
debian/control
vendored
1
debian/control
vendored
@ -9,7 +9,6 @@ X-Python3-Version: >= 3.2
|
||||
Package: pvc-daemon-node
|
||||
Architecture: all
|
||||
Depends: systemd, pvc-daemon-common, python3-kazoo, python3-psutil, python3-apscheduler, python3-libvirt, python3-psycopg2, python3-dnspython, python3-yaml, python3-distutils, python3-rados, python3-gevent, ipmitool, libvirt-daemon-system, arping, vlan, bridge-utils, dnsmasq, nftables, pdns-server, pdns-backend-pgsql
|
||||
Suggests: pvc-client-api, pvc-client-cli
|
||||
Description: Parallel Virtual Cluster node daemon (Python 3)
|
||||
A KVM/Zookeeper/Ceph-based VM and private cloud manager
|
||||
.
|
||||
|
BIN
docs/images/pvc-migration.png
Normal file
BIN
docs/images/pvc-migration.png
Normal file
Binary file not shown.
After Width: | Height: | Size: 88 KiB |
BIN
docs/images/pvc-networks.png
Normal file
BIN
docs/images/pvc-networks.png
Normal file
Binary file not shown.
After Width: | Height: | Size: 41 KiB |
BIN
docs/images/pvc-nodelog.png
Normal file
BIN
docs/images/pvc-nodelog.png
Normal file
Binary file not shown.
After Width: | Height: | Size: 300 KiB |
BIN
docs/images/pvc-nodes.png
Normal file
BIN
docs/images/pvc-nodes.png
Normal file
Binary file not shown.
After Width: | Height: | Size: 42 KiB |
@ -40,8 +40,43 @@ The core node and API daemons, as well as the CLI API client, are written in Pyt
|
||||
To get started with PVC, please see the [About](https://parallelvirtualcluster.readthedocs.io/en/latest/about/) page for general information about the project, and the [Getting Started](https://parallelvirtualcluster.readthedocs.io/en/latest/getting-started/) page for details on configuring your first cluster.
|
||||
|
||||
|
||||
## Screenshots
|
||||
|
||||
While PVC's API and internals aren't very screenshot-worthy, here is some example output of the CLI tool.
|
||||
|
||||
<p><img alt="Node listing" src="images/pvc-nodes.png"/><br/><i>Listing the nodes in a cluster</i></p>
|
||||
|
||||
<p><img alt="Network listing" src="images/pvc-networks.png"/><br/><i>Listing the networks in a cluster, showing 3 bridged and 1 IPv4-only managed networks</i></p>
|
||||
|
||||
<p><img alt="VM listing and migration" src="images/pvc-migration.png"/><br/><i>Listing a limited set of VMs and migrating one with status updates</i></p>
|
||||
|
||||
<p><img alt="Node logs" src="images/pvc-nodelog.png"/><br/><i>Viewing the logs of a node (keepalives and VM [un]migration)</i></p>
|
||||
|
||||
|
||||
## Changelog
|
||||
|
||||
#### v0.9.32
|
||||
|
||||
* [CLI Client] Fixes some incorrect colours in network lists
|
||||
* [Documentation] Adds documentation screenshots of CLI client
|
||||
* [Node Daemon] Fixes a bug if VM stats gathering fails
|
||||
|
||||
#### v0.9.31
|
||||
|
||||
* [Packages] Cleans up obsolete Suggests lines
|
||||
* [Node Daemon] Adjusts log text of VM migrations to show the correct source node
|
||||
* [API Daemon] Adjusts the OVA importer to support floppy RASD types for compatability
|
||||
* [API Daemon] Ensures that volume resize commands without a suffix get B appended
|
||||
* [API Daemon] Removes the explicit setting of image-features in PVC; defaulting to the limited set has been moved to the ceph.conf configuration on nodes via PVC Ansible
|
||||
|
||||
#### v0.9.30
|
||||
|
||||
* [Node Daemon] Fixes bug with schema validation
|
||||
|
||||
#### v0.9.29
|
||||
|
||||
* [Node Daemon] Corrects numerous bugs with node logging framework
|
||||
|
||||
#### v0.9.28
|
||||
|
||||
* [CLI Client] Revamp confirmation options for "vm modify" command
|
||||
|
@ -56,7 +56,7 @@ import pvcnoded.CephInstance as CephInstance
|
||||
import pvcnoded.MetadataAPIInstance as MetadataAPIInstance
|
||||
|
||||
# Version string for startup output
|
||||
version = '0.9.28'
|
||||
version = '0.9.32'
|
||||
|
||||
###############################################################################
|
||||
# PVCD - node daemon startup program
|
||||
@ -76,8 +76,11 @@ version = '0.9.28'
|
||||
# Daemon functions
|
||||
###############################################################################
|
||||
|
||||
# Ensure the update_timer is None until it's set for real
|
||||
# Ensure update_timer, this_node, and d_domain are None until they're set for real
|
||||
# Ensures cleanup() doesn't fail due to these items not being created yet
|
||||
update_timer = None
|
||||
this_node = None
|
||||
d_domain = None
|
||||
|
||||
|
||||
# Create timer to update this node in Zookeeper
|
||||
@ -110,7 +113,7 @@ try:
|
||||
pvcnoded_config_file = os.environ['PVCD_CONFIG_FILE']
|
||||
except Exception:
|
||||
print('ERROR: The "PVCD_CONFIG_FILE" environment variable must be set before starting pvcnoded.')
|
||||
exit(1)
|
||||
os._exit(1)
|
||||
|
||||
# Set local hostname and domain variables
|
||||
myfqdn = gethostname()
|
||||
@ -142,7 +145,7 @@ def readConfig(pvcnoded_config_file, myhostname):
|
||||
o_config = yaml.load(cfgfile, Loader=yaml.SafeLoader)
|
||||
except Exception as e:
|
||||
print('ERROR: Failed to parse configuration file: {}'.format(e))
|
||||
exit(1)
|
||||
os._exit(1)
|
||||
|
||||
# Handle the basic config (hypervisor-only)
|
||||
try:
|
||||
@ -179,7 +182,7 @@ def readConfig(pvcnoded_config_file, myhostname):
|
||||
}
|
||||
except Exception as e:
|
||||
print('ERROR: Failed to load configuration: {}'.format(e))
|
||||
exit(1)
|
||||
cleanup(failure=True)
|
||||
config = config_general
|
||||
|
||||
# Handle debugging config
|
||||
@ -236,7 +239,7 @@ def readConfig(pvcnoded_config_file, myhostname):
|
||||
|
||||
except Exception as e:
|
||||
print('ERROR: Failed to load configuration: {}'.format(e))
|
||||
exit(1)
|
||||
cleanup(failure=True)
|
||||
config = {**config, **config_networking}
|
||||
|
||||
# Create the by-id address entries
|
||||
@ -250,7 +253,7 @@ def readConfig(pvcnoded_config_file, myhostname):
|
||||
network = ip_network(config[network_key])
|
||||
except Exception:
|
||||
print('ERROR: Network address {} for {} is not valid!'.format(config[network_key], network_key))
|
||||
exit(1)
|
||||
cleanup(failure=True)
|
||||
|
||||
# If we should be autoselected
|
||||
if config[address_key] == 'by-id':
|
||||
@ -270,7 +273,7 @@ def readConfig(pvcnoded_config_file, myhostname):
|
||||
raise
|
||||
except Exception:
|
||||
print('ERROR: Floating address {} for {} is not valid!'.format(config[floating_key], floating_key))
|
||||
exit(1)
|
||||
cleanup(failure=True)
|
||||
|
||||
# Handle the storage config
|
||||
if config['enable_storage']:
|
||||
@ -281,7 +284,7 @@ def readConfig(pvcnoded_config_file, myhostname):
|
||||
}
|
||||
except Exception as e:
|
||||
print('ERROR: Failed to load configuration: {}'.format(e))
|
||||
exit(1)
|
||||
cleanup(failure=True)
|
||||
config = {**config, **config_storage}
|
||||
|
||||
# Handle an empty ipmi_hostname
|
||||
@ -488,6 +491,9 @@ if enable_networking:
|
||||
else:
|
||||
common.run_os_command('ip route add default via {} dev {}'.format(upstream_gateway, 'brupstream'))
|
||||
|
||||
logger.out('Waiting 3s for networking to come up', state='s')
|
||||
time.sleep(3)
|
||||
|
||||
###############################################################################
|
||||
# PHASE 2c - Prepare sysctl for pvcnoded
|
||||
###############################################################################
|
||||
@ -559,8 +565,8 @@ if enable_storage:
|
||||
logger.out('Starting Ceph manager daemon', state='i')
|
||||
common.run_os_command('systemctl start ceph-mgr@{}'.format(myhostname))
|
||||
|
||||
logger.out('Waiting 5s for daemons to start', state='s')
|
||||
time.sleep(5)
|
||||
logger.out('Waiting 3s for daemons to start', state='s')
|
||||
time.sleep(3)
|
||||
|
||||
###############################################################################
|
||||
# PHASE 4 - Attempt to connect to the coordinators and start zookeeper client
|
||||
@ -575,7 +581,7 @@ try:
|
||||
zkhandler.connect(persistent=True)
|
||||
except Exception as e:
|
||||
logger.out('ERROR: Failed to connect to Zookeeper cluster: {}'.format(e), state='e')
|
||||
exit(1)
|
||||
os._exit(1)
|
||||
|
||||
logger.out('Validating Zookeeper schema', state='i')
|
||||
|
||||
@ -696,7 +702,7 @@ else:
|
||||
|
||||
|
||||
# Cleanup function
|
||||
def cleanup():
|
||||
def cleanup(failure=False):
|
||||
global logger, zkhandler, update_timer, d_domain
|
||||
|
||||
logger.out('Terminating pvcnoded and cleaning up', state='s')
|
||||
@ -708,19 +714,19 @@ def cleanup():
|
||||
|
||||
# Waiting for any flushes to complete
|
||||
logger.out('Waiting for any active flushes', state='s')
|
||||
while this_node.flush_thread is not None:
|
||||
time.sleep(0.5)
|
||||
if this_node is not None:
|
||||
while this_node.flush_thread is not None:
|
||||
time.sleep(0.5)
|
||||
|
||||
# Stop console logging on all VMs
|
||||
logger.out('Stopping domain console watchers', state='s')
|
||||
for domain in d_domain:
|
||||
if d_domain[domain].getnode() == myhostname:
|
||||
try:
|
||||
d_domain[domain].console_log_instance.stop()
|
||||
except NameError:
|
||||
pass
|
||||
except AttributeError:
|
||||
pass
|
||||
if d_domain is not None:
|
||||
for domain in d_domain:
|
||||
if d_domain[domain].getnode() == myhostname:
|
||||
try:
|
||||
d_domain[domain].console_log_instance.stop()
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
# Force into secondary coordinator state if needed
|
||||
try:
|
||||
@ -737,13 +743,11 @@ def cleanup():
|
||||
# Stop keepalive thread
|
||||
try:
|
||||
stopKeepaliveTimer()
|
||||
except NameError:
|
||||
pass
|
||||
except AttributeError:
|
||||
pass
|
||||
|
||||
logger.out('Performing final keepalive update', state='s')
|
||||
node_keepalive()
|
||||
logger.out('Performing final keepalive update', state='s')
|
||||
node_keepalive()
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
# Set stop state in Zookeeper
|
||||
zkhandler.write([
|
||||
@ -763,12 +767,17 @@ def cleanup():
|
||||
logger.out('Terminated pvc daemon', state='s')
|
||||
logger.terminate()
|
||||
|
||||
os._exit(0)
|
||||
if failure:
|
||||
retcode = 1
|
||||
else:
|
||||
retcode = 0
|
||||
|
||||
os._exit(retcode)
|
||||
|
||||
|
||||
# Termination function
|
||||
def term(signum='', frame=''):
|
||||
cleanup()
|
||||
cleanup(failure=False)
|
||||
|
||||
|
||||
# Hangup (logrotate) function
|
||||
@ -868,7 +877,7 @@ if enable_hypervisor:
|
||||
lv_conn.close()
|
||||
except Exception as e:
|
||||
logger.out('ERROR: Failed to connect to Libvirt daemon: {}'.format(e), state='e')
|
||||
exit(1)
|
||||
cleanup(failure=True)
|
||||
|
||||
###############################################################################
|
||||
# PHASE 7c - Ensure NFT is running on the local host
|
||||
@ -1666,11 +1675,7 @@ def collect_vm_stats(queue):
|
||||
domain_memory_stats = domain.memoryStats()
|
||||
domain_cpu_stats = domain.getCPUStats(True)[0]
|
||||
except Exception as e:
|
||||
if debug:
|
||||
try:
|
||||
logger.out("Failed getting VM information for {}: {}".format(domain.name(), e), state='d', prefix='vm-thread')
|
||||
except Exception:
|
||||
pass
|
||||
logger.out("Failed getting VM information for {}: {}".format(domain.name(), e), state='w', prefix='vm-thread')
|
||||
continue
|
||||
|
||||
# Ensure VM is present in the domain_list
|
||||
@ -1680,42 +1685,50 @@ def collect_vm_stats(queue):
|
||||
if debug:
|
||||
logger.out("Getting disk statistics for VM {}".format(domain_name), state='d', prefix='vm-thread')
|
||||
domain_disk_stats = []
|
||||
for disk in tree.findall('devices/disk'):
|
||||
disk_name = disk.find('source').get('name')
|
||||
if not disk_name:
|
||||
disk_name = disk.find('source').get('file')
|
||||
disk_stats = domain.blockStats(disk.find('target').get('dev'))
|
||||
domain_disk_stats.append({
|
||||
"name": disk_name,
|
||||
"rd_req": disk_stats[0],
|
||||
"rd_bytes": disk_stats[1],
|
||||
"wr_req": disk_stats[2],
|
||||
"wr_bytes": disk_stats[3],
|
||||
"err": disk_stats[4]
|
||||
})
|
||||
try:
|
||||
for disk in tree.findall('devices/disk'):
|
||||
disk_name = disk.find('source').get('name')
|
||||
if not disk_name:
|
||||
disk_name = disk.find('source').get('file')
|
||||
disk_stats = domain.blockStats(disk.find('target').get('dev'))
|
||||
domain_disk_stats.append({
|
||||
"name": disk_name,
|
||||
"rd_req": disk_stats[0],
|
||||
"rd_bytes": disk_stats[1],
|
||||
"wr_req": disk_stats[2],
|
||||
"wr_bytes": disk_stats[3],
|
||||
"err": disk_stats[4]
|
||||
})
|
||||
except Exception as e:
|
||||
logger.out("Failed to get disk stats for VM {}: {}".format(domain_name, e), state='w', prefix='vm-thread')
|
||||
continue
|
||||
|
||||
if debug:
|
||||
logger.out("Getting network statistics for VM {}".format(domain_name), state='d', prefix='vm-thread')
|
||||
domain_network_stats = []
|
||||
for interface in tree.findall('devices/interface'):
|
||||
interface_type = interface.get('type')
|
||||
if interface_type not in ['bridge']:
|
||||
continue
|
||||
interface_name = interface.find('target').get('dev')
|
||||
interface_bridge = interface.find('source').get('bridge')
|
||||
interface_stats = domain.interfaceStats(interface_name)
|
||||
domain_network_stats.append({
|
||||
"name": interface_name,
|
||||
"bridge": interface_bridge,
|
||||
"rd_bytes": interface_stats[0],
|
||||
"rd_packets": interface_stats[1],
|
||||
"rd_errors": interface_stats[2],
|
||||
"rd_drops": interface_stats[3],
|
||||
"wr_bytes": interface_stats[4],
|
||||
"wr_packets": interface_stats[5],
|
||||
"wr_errors": interface_stats[6],
|
||||
"wr_drops": interface_stats[7]
|
||||
})
|
||||
try:
|
||||
for interface in tree.findall('devices/interface'):
|
||||
interface_type = interface.get('type')
|
||||
if interface_type not in ['bridge']:
|
||||
continue
|
||||
interface_name = interface.find('target').get('dev')
|
||||
interface_bridge = interface.find('source').get('bridge')
|
||||
interface_stats = domain.interfaceStats(interface_name)
|
||||
domain_network_stats.append({
|
||||
"name": interface_name,
|
||||
"bridge": interface_bridge,
|
||||
"rd_bytes": interface_stats[0],
|
||||
"rd_packets": interface_stats[1],
|
||||
"rd_errors": interface_stats[2],
|
||||
"rd_drops": interface_stats[3],
|
||||
"wr_bytes": interface_stats[4],
|
||||
"wr_packets": interface_stats[5],
|
||||
"wr_errors": interface_stats[6],
|
||||
"wr_drops": interface_stats[7]
|
||||
})
|
||||
except Exception as e:
|
||||
logger.out("Failed to get network stats for VM {}: {}".format(domain_name, e), state='w', prefix='vm-thread')
|
||||
continue
|
||||
|
||||
# Create the final dictionary
|
||||
domain_stats = {
|
||||
|
@ -635,7 +635,7 @@ class VMInstance(object):
|
||||
|
||||
self.inreceive = True
|
||||
|
||||
self.logger.out('Receiving VM migration from node "{}"'.format(self.node), state='i', prefix='Domain {}'.format(self.domuuid))
|
||||
self.logger.out('Receiving VM migration from node "{}"'.format(self.last_currentnode), state='i', prefix='Domain {}'.format(self.domuuid))
|
||||
|
||||
# Short delay to ensure sender is in sync
|
||||
time.sleep(0.5)
|
||||
|
Reference in New Issue
Block a user