parallelvirtualcluster/pvc
1
0
Fork 0
Code Issues 11 Pull Requests Projects Releases 114 Wiki Activity

Move new CLI client into place

Browse Source
This commit is contained in:
Joshua Boniface
2023-08-16 23:55:27 -04:00
parent a70273dbae
commit d2bcaec28f
18 changed files with 0 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files

33
client-cli/pvc.py Executable file
View File

@ -0,0 +1,33 @@
#!/usr/bin/env python3
# pvc.py - PVC client command-line interface (stub testing interface)
# Part of the Parallel Virtual Cluster (PVC) system
#
# Copyright (C) 2018-2022 Joshua M. Boniface <joshua@boniface.me>
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, version 3.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program. If not, see <https://www.gnu.org/licenses/>.
#
###############################################################################
from pvc.cli.cli import cli
#
# Main entry point
#
def main():
return cli(obj={})
if __name__ == "__main__":
main()

0
client-cli/pvc/__init__.py Normal file
View File

5842
client-cli/pvc/cli/cli.py Normal file
View File

File diff suppressed because it is too large Load Diff

732
client-cli/pvc/cli/formatters.py Normal file
View File

@ -0,0 +1,732 @@
#!/usr/bin/env python3
# formatters.py - PVC Click CLI output formatters library
# Part of the Parallel Virtual Cluster (PVC) system
#
# Copyright (C) 2018-2023 Joshua M. Boniface <joshua@boniface.me>
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, version 3.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program. If not, see <https://www.gnu.org/licenses/>.
#
###############################################################################
from pvc.lib.node import format_info as node_format_info
from pvc.lib.node import format_list as node_format_list
from pvc.lib.vm import format_vm_tags as vm_format_tags
from pvc.lib.vm import format_vm_vcpus as vm_format_vcpus
from pvc.lib.vm import format_vm_memory as vm_format_memory
from pvc.lib.vm import format_vm_networks as vm_format_networks
from pvc.lib.vm import format_vm_volumes as vm_format_volumes
from pvc.lib.vm import format_info as vm_format_info
from pvc.lib.vm import format_list as vm_format_list
from pvc.lib.network import format_info as network_format_info
from pvc.lib.network import format_list as network_format_list
from pvc.lib.network import format_list_dhcp as network_format_dhcp_list
from pvc.lib.network import format_list_acl as network_format_acl_list
from pvc.lib.network import format_list_sriov_pf as network_format_sriov_pf_list
from pvc.lib.network import format_info_sriov_vf as network_format_sriov_vf_info
from pvc.lib.network import format_list_sriov_vf as network_format_sriov_vf_list
from pvc.lib.storage import format_raw_output as storage_format_raw
from pvc.lib.storage import format_info_benchmark as storage_format_benchmark_info
from pvc.lib.storage import format_list_benchmark as storage_format_benchmark_list
from pvc.lib.storage import format_list_osd as storage_format_osd_list
from pvc.lib.storage import format_list_pool as storage_format_pool_list
from pvc.lib.storage import format_list_volume as storage_format_volume_list
from pvc.lib.storage import format_list_snapshot as storage_format_snapshot_list
from pvc.lib.provisioner import format_list_template as provisioner_format_template_list
from pvc.lib.provisioner import format_list_userdata as provisioner_format_userdata_list
from pvc.lib.provisioner import format_list_script as provisioner_format_script_list
from pvc.lib.provisioner import format_list_ova as provisioner_format_ova_list
from pvc.lib.provisioner import format_list_profile as provisioner_format_profile_list
from pvc.lib.provisioner import format_list_task as provisioner_format_task_status
# Define colour values for use in formatters
ansii = {
"red": "\033[91m",
"blue": "\033[94m",
"cyan": "\033[96m",
"green": "\033[92m",
"yellow": "\033[93m",
"purple": "\033[95m",
"bold": "\033[1m",
"end": "\033[0m",
}
def cli_cluster_status_format_pretty(CLI_CONFIG, data):
"""
Pretty format the full output of cli_cluster_status
"""
# Normalize data to local variables
health = data.get("cluster_health", {}).get("health", -1)
messages = data.get("cluster_health", {}).get("messages", None)
maintenance = data.get("maintenance", "N/A")
primary_node = data.get("primary_node", "N/A")
pvc_version = data.get("pvc_version", "N/A")
upstream_ip = data.get("upstream_ip", "N/A")
total_nodes = data.get("nodes", {}).get("total", 0)
total_vms = data.get("vms", {}).get("total", 0)
total_networks = data.get("networks", 0)
total_osds = data.get("osds", {}).get("total", 0)
total_pools = data.get("pools", 0)
total_volumes = data.get("volumes", 0)
total_snapshots = data.get("snapshots", 0)
if maintenance == "true" or health == -1:
health_colour = ansii["blue"]
elif health > 90:
health_colour = ansii["green"]
elif health > 50:
health_colour = ansii["yellow"]
else:
health_colour = ansii["red"]
output = list()
output.append(f"{ansii['bold']}PVC cluster status:{ansii['end']}")
output.append("")
if health != "-1":
health = f"{health}%"
else:
health = "N/A"
if maintenance == "true":
health = f"{health} (maintenance on)"
output.append(
f"{ansii['purple']}Cluster health:{ansii['end']} {health_colour}{health}{ansii['end']}"
)
if messages is not None and len(messages) > 0:
messages = "\n ".join(sorted(messages))
output.append(f"{ansii['purple']}Health messages:{ansii['end']} {messages}")
output.append("")
output.append(f"{ansii['purple']}Primary node:{ansii['end']} {primary_node}")
output.append(f"{ansii['purple']}PVC version:{ansii['end']} {pvc_version}")
output.append(f"{ansii['purple']}Upstream IP:{ansii['end']} {upstream_ip}")
output.append("")
node_states = ["run,ready"]
node_states.extend(
[
state
for state in data.get("nodes", {}).keys()
if state not in ["total", "run,ready"]
]
)
nodes_strings = list()
for state in node_states:
if state in ["run,ready"]:
state_colour = ansii["green"]
elif state in ["run,flush", "run,unflush", "run,flushed"]:
state_colour = ansii["blue"]
elif "dead" in state or "stop" in state:
state_colour = ansii["red"]
else:
state_colour = ansii["yellow"]
nodes_strings.append(
f"{data.get('nodes', {}).get(state)}/{total_nodes} {state_colour}{state}{ansii['end']}"
)
nodes_string = ", ".join(nodes_strings)
output.append(f"{ansii['purple']}Nodes:{ansii['end']} {nodes_string}")
vm_states = ["start", "disable"]
vm_states.extend(
[
state
for state in data.get("vms", {}).keys()
if state not in ["total", "start", "disable"]
]
)
vms_strings = list()
for state in vm_states:
if state in ["start"]:
state_colour = ansii["green"]
elif state in ["migrate", "disable"]:
state_colour = ansii["blue"]
elif state in ["stop", "fail"]:
state_colour = ansii["red"]
else:
state_colour = ansii["yellow"]
vms_strings.append(
f"{data.get('vms', {}).get(state)}/{total_vms} {state_colour}{state}{ansii['end']}"
)
vms_string = ", ".join(vms_strings)
output.append(f"{ansii['purple']}VMs:{ansii['end']} {vms_string}")
osd_states = ["up,in"]
osd_states.extend(
[
state
for state in data.get("osds", {}).keys()
if state not in ["total", "up,in"]
]
)
osds_strings = list()
for state in osd_states:
if state in ["up,in"]:
state_colour = ansii["green"]
elif state in ["down,out"]:
state_colour = ansii["red"]
else:
state_colour = ansii["yellow"]
osds_strings.append(
f"{data.get('osds', {}).get(state)}/{total_osds} {state_colour}{state}{ansii['end']}"
)
osds_string = " ".join(osds_strings)
output.append(f"{ansii['purple']}OSDs:{ansii['end']} {osds_string}")
output.append(f"{ansii['purple']}Pools:{ansii['end']} {total_pools}")
output.append(f"{ansii['purple']}Volumes:{ansii['end']} {total_volumes}")
output.append(f"{ansii['purple']}Snapshots:{ansii['end']} {total_snapshots}")
output.append(f"{ansii['purple']}Networks:{ansii['end']} {total_networks}")
output.append("")
return "\n".join(output)
def cli_cluster_status_format_short(CLI_CONFIG, data):
"""
Pretty format the health-only output of cli_cluster_status
"""
# Normalize data to local variables
health = data.get("cluster_health", {}).get("health", -1)
messages = data.get("cluster_health", {}).get("messages", None)
maintenance = data.get("maintenance", "N/A")
if maintenance == "true" or health == -1:
health_colour = ansii["blue"]
elif health > 90:
health_colour = ansii["green"]
elif health > 50:
health_colour = ansii["yellow"]
else:
health_colour = ansii["red"]
output = list()
output.append(f"{ansii['bold']}PVC cluster status:{ansii['end']}")
output.append("")
if health != "-1":
health = f"{health}%"
else:
health = "N/A"
if maintenance == "true":
health = f"{health} (maintenance on)"
output.append(
f"{ansii['purple']}Cluster health:{ansii['end']} {health_colour}{health}{ansii['end']}"
)
if messages is not None and len(messages) > 0:
messages = "\n ".join(sorted(messages))
output.append(f"{ansii['purple']}Health messages:{ansii['end']} {messages}")
output.append("")
return "\n".join(output)
def cli_connection_list_format_pretty(CLI_CONFIG, data):
"""
Pretty format the output of cli_connection_list
"""
# Set the fields data
fields = {
"name": {"header": "Name", "length": len("Name") + 1},
"description": {"header": "Description", "length": len("Description") + 1},
"address": {"header": "Address", "length": len("Address") + 1},
"port": {"header": "Port", "length": len("Port") + 1},
"scheme": {"header": "Scheme", "length": len("Scheme") + 1},
"api_key": {"header": "API Key", "length": len("API Key") + 1},
}
# Parse each connection and adjust field lengths
for connection in data:
for field, length in [(f, fields[f]["length"]) for f in fields]:
_length = len(str(connection[field]))
if _length > length:
length = len(str(connection[field])) + 1
fields[field]["length"] = length
# Create the output object and define the line format
output = list()
line = "{bold}{name: <{lname}} {desc: <{ldesc}} {addr: <{laddr}} {port: <{lport}} {schm: <{lschm}} {akey: <{lakey}}{end}"
# Add the header line
output.append(
line.format(
bold=ansii["bold"],
end=ansii["end"],
name=fields["name"]["header"],
lname=fields["name"]["length"],
desc=fields["description"]["header"],
ldesc=fields["description"]["length"],
addr=fields["address"]["header"],
laddr=fields["address"]["length"],
port=fields["port"]["header"],
lport=fields["port"]["length"],
schm=fields["scheme"]["header"],
lschm=fields["scheme"]["length"],
akey=fields["api_key"]["header"],
lakey=fields["api_key"]["length"],
)
)
# Add a line per connection
for connection in data:
output.append(
line.format(
bold="",
end="",
name=connection["name"],
lname=fields["name"]["length"],
desc=connection["description"],
ldesc=fields["description"]["length"],
addr=connection["address"],
laddr=fields["address"]["length"],
port=connection["port"],
lport=fields["port"]["length"],
schm=connection["scheme"],
lschm=fields["scheme"]["length"],
akey=connection["api_key"],
lakey=fields["api_key"]["length"],
)
)
return "\n".join(output)
def cli_connection_detail_format_pretty(CLI_CONFIG, data):
"""
Pretty format the output of cli_connection_detail
"""
# Set the fields data
fields = {
"name": {"header": "Name", "length": len("Name") + 1},
"description": {"header": "Description", "length": len("Description") + 1},
"health": {"header": "Health", "length": len("Health") + 1},
"primary_node": {"header": "Primary", "length": len("Primary") + 1},
"pvc_version": {"header": "Version", "length": len("Version") + 1},
"nodes": {"header": "Nodes", "length": len("Nodes") + 1},
"vms": {"header": "VMs", "length": len("VMs") + 1},
"networks": {"header": "Networks", "length": len("Networks") + 1},
"osds": {"header": "OSDs", "length": len("OSDs") + 1},
"pools": {"header": "Pools", "length": len("Pools") + 1},
"volumes": {"header": "Volumes", "length": len("Volumes") + 1},
"snapshots": {"header": "Snapshots", "length": len("Snapshots") + 1},
}
# Parse each connection and adjust field lengths
for connection in data:
for field, length in [(f, fields[f]["length"]) for f in fields]:
_length = len(str(connection[field]))
if _length > length:
length = len(str(connection[field])) + 1
fields[field]["length"] = length
# Create the output object and define the line format
output = list()
line = "{bold}{name: <{lname}} {desc: <{ldesc}} {chlth}{hlth: <{lhlth}}{endc} {prin: <{lprin}} {vers: <{lvers}} {nods: <{lnods}} {vms: <{lvms}} {nets: <{lnets}} {osds: <{losds}} {pols: <{lpols}} {vols: <{lvols}} {snts: <{lsnts}}{end}"
# Add the header line
output.append(
line.format(
bold=ansii["bold"],
end=ansii["end"],
chlth="",
endc="",
name=fields["name"]["header"],
lname=fields["name"]["length"],
desc=fields["description"]["header"],
ldesc=fields["description"]["length"],
hlth=fields["health"]["header"],
lhlth=fields["health"]["length"],
prin=fields["primary_node"]["header"],
lprin=fields["primary_node"]["length"],
vers=fields["pvc_version"]["header"],
lvers=fields["pvc_version"]["length"],
nods=fields["nodes"]["header"],
lnods=fields["nodes"]["length"],
vms=fields["vms"]["header"],
lvms=fields["vms"]["length"],
nets=fields["networks"]["header"],
lnets=fields["networks"]["length"],
osds=fields["osds"]["header"],
losds=fields["osds"]["length"],
pols=fields["pools"]["header"],
lpols=fields["pools"]["length"],
vols=fields["volumes"]["header"],
lvols=fields["volumes"]["length"],
snts=fields["snapshots"]["header"],
lsnts=fields["snapshots"]["length"],
)
)
# Add a line per connection
for connection in data:
if connection["health"] == "N/A":
health_value = "N/A"
health_colour = ansii["purple"]
else:
health_value = f"{connection['health']}%"
if connection["maintenance"] == "true":
health_colour = ansii["blue"]
elif connection["health"] > 90:
health_colour = ansii["green"]
elif connection["health"] > 50:
health_colour = ansii["yellow"]
else:
health_colour = ansii["red"]
output.append(
line.format(
bold="",
end="",
chlth=health_colour,
endc=ansii["end"],
name=connection["name"],
lname=fields["name"]["length"],
desc=connection["description"],
ldesc=fields["description"]["length"],
hlth=health_value,
lhlth=fields["health"]["length"],
prin=connection["primary_node"],
lprin=fields["primary_node"]["length"],
vers=connection["pvc_version"],
lvers=fields["pvc_version"]["length"],
nods=connection["nodes"],
lnods=fields["nodes"]["length"],
vms=connection["vms"],
lvms=fields["vms"]["length"],
nets=connection["networks"],
lnets=fields["networks"]["length"],
osds=connection["osds"],
losds=fields["osds"]["length"],
pols=connection["pools"],
lpols=fields["pools"]["length"],
vols=connection["volumes"],
lvols=fields["volumes"]["length"],
snts=connection["snapshots"],
lsnts=fields["snapshots"]["length"],
)
)
return "\n".join(output)
def cli_node_info_format_pretty(CLI_CONFIG, data):
"""
Pretty format the basic output of cli_node_info
"""
return node_format_info(CLI_CONFIG, data, long_output=False)
def cli_node_info_format_long(CLI_CONFIG, data):
"""
Pretty format the full output of cli_node_info
"""
return node_format_info(CLI_CONFIG, data, long_output=True)
def cli_node_list_format_pretty(CLI_CONFIG, data):
"""
Pretty format the output of cli_node_list
"""
return node_format_list(CLI_CONFIG, data)
def cli_vm_tag_get_format_pretty(CLI_CONFIG, data):
"""
Pretty format the output of cli_vm_tag_get
"""
return vm_format_tags(CLI_CONFIG, data)
def cli_vm_vcpu_get_format_pretty(CLI_CONFIG, data):
"""
Pretty format the output of cli_vm_vcpu_get
"""
return vm_format_vcpus(CLI_CONFIG, data)
def cli_vm_memory_get_format_pretty(CLI_CONFIG, data):
"""
Pretty format the output of cli_vm_memory_get
"""
return vm_format_memory(CLI_CONFIG, data)
def cli_vm_network_get_format_pretty(CLI_CONFIG, data):
"""
Pretty format the output of cli_vm_network_get
"""
return vm_format_networks(CLI_CONFIG, data)
def cli_vm_volume_get_format_pretty(CLI_CONFIG, data):
"""
Pretty format the output of cli_vm_volume_get
"""
return vm_format_volumes(CLI_CONFIG, data)
def cli_vm_info_format_pretty(CLI_CONFIG, data):
"""
Pretty format the basic output of cli_vm_info
"""
return vm_format_info(CLI_CONFIG, data, long_output=False)
def cli_vm_info_format_long(CLI_CONFIG, data):
"""
Pretty format the full output of cli_vm_info
"""
return vm_format_info(CLI_CONFIG, data, long_output=True)
def cli_vm_list_format_pretty(CLI_CONFIG, data):
"""
Pretty format the output of cli_vm_list
"""
return vm_format_list(CLI_CONFIG, data)
def cli_network_info_format_pretty(CLI_CONFIG, data):
"""
Pretty format the full output of cli_network_info
"""
return network_format_info(CLI_CONFIG, data, long_output=True)
def cli_network_info_format_long(CLI_CONFIG, data):
"""
Pretty format the full output of cli_network_info
"""
return network_format_info(CLI_CONFIG, data, long_output=True)
def cli_network_list_format_pretty(CLI_CONFIG, data):
"""
Pretty format the output of cli_network_list
"""
return network_format_list(CLI_CONFIG, data)
def cli_network_dhcp_list_format_pretty(CLI_CONFIG, data):
"""
Pretty format the output of cli_network_dhcp_list
"""
return network_format_dhcp_list(CLI_CONFIG, data)
def cli_network_acl_list_format_pretty(CLI_CONFIG, data):
"""
Pretty format the output of cli_network_acl_list
"""
return network_format_acl_list(CLI_CONFIG, data)
def cli_network_sriov_pf_list_format_pretty(CLI_CONFIG, data):
"""
Pretty format the output of cli_network_sriov_pf_list
"""
return network_format_sriov_pf_list(CLI_CONFIG, data)
def cli_network_sriov_vf_info_format_pretty(CLI_CONFIG, data):
"""
Pretty format the output of cli_network_sriov_vf_info
"""
return network_format_sriov_vf_info(CLI_CONFIG, data)
def cli_network_sriov_vf_list_format_pretty(CLI_CONFIG, data):
"""
Pretty format the output of cli_network_sriov_vf_list
"""
return network_format_sriov_vf_list(CLI_CONFIG, data)
def cli_storage_status_format_raw(CLI_CONFIG, data):
"""
Direct format the output of cli_storage_status
"""
return storage_format_raw(CLI_CONFIG, data)
def cli_storage_util_format_raw(CLI_CONFIG, data):
"""
Direct format the output of cli_storage_util
"""
return storage_format_raw(CLI_CONFIG, data)
def cli_storage_benchmark_info_format_pretty(CLI_CONFIG, data):
"""
Pretty format the output of cli_storage_benchmark_info
"""
return storage_format_benchmark_info(CLI_CONFIG, data)
def cli_storage_benchmark_list_format_pretty(CLI_CONFIG, data):
"""
Pretty format the output of cli_storage_benchmark_list
"""
return storage_format_benchmark_list(CLI_CONFIG, data)
def cli_storage_osd_list_format_pretty(CLI_CONFIG, data):
"""
Pretty format the output of cli_storage_osd_list
"""
return storage_format_osd_list(CLI_CONFIG, data)
def cli_storage_pool_list_format_pretty(CLI_CONFIG, data):
"""
Pretty format the output of cli_storage_pool_list
"""
return storage_format_pool_list(CLI_CONFIG, data)
def cli_storage_volume_list_format_pretty(CLI_CONFIG, data):
"""
Pretty format the output of cli_storage_volume_list
"""
return storage_format_volume_list(CLI_CONFIG, data)
def cli_storage_snapshot_list_format_pretty(CLI_CONFIG, data):
"""
Pretty format the output of cli_storage_snapshot_list
"""
return storage_format_snapshot_list(CLI_CONFIG, data)
def cli_provisioner_template_system_list_format_pretty(CLI_CONFIG, data):
"""
Pretty format the output of cli_provisioner_template_system_list
"""
return provisioner_format_template_list(CLI_CONFIG, data, template_type="system")
def cli_provisioner_template_network_list_format_pretty(CLI_CONFIG, data):
"""
Pretty format the output of cli_provisioner_template_network_list
"""
return provisioner_format_template_list(CLI_CONFIG, data, template_type="network")
def cli_provisioner_template_storage_list_format_pretty(CLI_CONFIG, data):
"""
Pretty format the output of cli_provisioner_template_storage_list
"""
return provisioner_format_template_list(CLI_CONFIG, data, template_type="storage")
def cli_provisioner_userdata_list_format_pretty(CLI_CONFIG, data):
"""
Pretty format the output of cli_provisioner_userdata_list
"""
return provisioner_format_userdata_list(CLI_CONFIG, data)
def cli_provisioner_script_list_format_pretty(CLI_CONFIG, data):
"""
Pretty format the output of cli_provisioner_script_list
"""
return provisioner_format_script_list(CLI_CONFIG, data)
def cli_provisioner_ova_list_format_pretty(CLI_CONFIG, data):
"""
Pretty format the output of cli_provisioner_ova_list
"""
return provisioner_format_ova_list(CLI_CONFIG, data)
def cli_provisioner_profile_list_format_pretty(CLI_CONFIG, data):
"""
Pretty format the output of cli_provisioner_profile_list
"""
return provisioner_format_profile_list(CLI_CONFIG, data)
def cli_provisioner_status_format_pretty(CLI_CONFIG, data):
"""
Pretty format the output of cli_provisioner_status
"""
return provisioner_format_task_status(CLI_CONFIG, data)

180
client-cli/pvc/cli/helpers.py Normal file
View File

@ -0,0 +1,180 @@
#!/usr/bin/env python3
# helpers.py - PVC Click CLI helper function library
# Part of the Parallel Virtual Cluster (PVC) system
#
# Copyright (C) 2018-2023 Joshua M. Boniface <joshua@boniface.me>
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, version 3.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program. If not, see <https://www.gnu.org/licenses/>.
#
###############################################################################
from click import echo as click_echo
from distutils.util import strtobool
from json import load as jload
from json import dump as jdump
from os import chmod, environ, getpid, path
from socket import gethostname
from sys import argv
from syslog import syslog, openlog, closelog, LOG_AUTH
from yaml import load as yload
from yaml import BaseLoader
DEFAULT_STORE_DATA = {"cfgfile": "/etc/pvc/pvcapid.yaml"}
DEFAULT_STORE_FILENAME = "pvc.json"
DEFAULT_API_PREFIX = "/api/v1"
DEFAULT_NODE_HOSTNAME = gethostname().split(".")[0]
def echo(config, message, newline=True, stderr=False):
"""
Output a message with click.echo respecting our configuration
"""
if config.get("colour", False):
colour = True
else:
colour = None
if config.get("silent", False):
pass
elif config.get("quiet", False) and stderr:
pass
else:
click_echo(message=message, color=colour, nl=newline, err=stderr)
def audit():
"""
Log an audit message to the local syslog AUTH facility
"""
args = argv
args[0] = "pvc"
pid = getpid()
openlog(facility=LOG_AUTH, ident=f"{args[0]}[{pid}]")
syslog(
f"""client audit: command "{' '.join(args)}" by user {environ.get('USER', None)}"""
)
closelog()
def read_config_from_yaml(cfgfile):
"""
Read the PVC API configuration from the local API configuration file
"""
try:
with open(cfgfile) as fh:
api_config = yload(fh, Loader=BaseLoader)["pvc"]["api"]
host = api_config["listen_address"]
port = api_config["listen_port"]
scheme = "https" if strtobool(api_config["ssl"]["enabled"]) else "http"
api_key = (
api_config["authentication"]["tokens"][0]["token"]
if strtobool(api_config["authentication"]["enabled"])
else None
)
except KeyError:
host = None
port = None
scheme = None
api_key = None
return cfgfile, host, port, scheme, api_key
def get_config(store_data, connection=None):
"""
Load CLI configuration from store data
"""
if store_data is None:
return {"badcfg": True}
connection_details = store_data.get(connection, None)
if not connection_details:
connection = "local"
connection_details = DEFAULT_STORE_DATA
if connection_details.get("cfgfile", None) is not None:
if path.isfile(connection_details.get("cfgfile", None)):
description, host, port, scheme, api_key = read_config_from_yaml(
connection_details.get("cfgfile", None)
)
if None in [description, host, port, scheme]:
return {"badcfg": True}
else:
return {"badcfg": True}
# Rewrite a wildcard listener to use localhost instead
if host == "0.0.0.0":
host = "127.0.0.1"
else:
# This is a static configuration, get the details directly
description = connection_details["description"]
host = connection_details["host"]
port = connection_details["port"]
scheme = connection_details["scheme"]
api_key = connection_details["api_key"]
config = dict()
config["debug"] = False
config["connection"] = connection
config["description"] = description
config["api_host"] = f"{host}:{port}"
config["api_scheme"] = scheme
config["api_key"] = api_key
config["api_prefix"] = DEFAULT_API_PREFIX
if connection == "local":
config["verify_ssl"] = False
else:
config["verify_ssl"] = bool(
strtobool(environ.get("PVC_CLIENT_VERIFY_SSL", "True"))
)
return config
def get_store(store_path):
"""
Load store information from the store path
"""
store_file = f"{store_path}/{DEFAULT_STORE_FILENAME}"
with open(store_file) as fh:
try:
store_data = jload(fh)
return store_data
except Exception:
return dict()
def update_store(store_path, store_data):
"""
Update store information to the store path, creating it (with sensible permissions) if needed
"""
store_file = f"{store_path}/{DEFAULT_STORE_FILENAME}"
if not path.exists(store_file):
with open(store_file, "w") as fh:
fh.write("")
chmod(store_file, int(environ.get("PVC_CLIENT_DB_PERMS", "600"), 8))
with open(store_file, "w") as fh:
jdump(store_data, fh, sort_keys=True, indent=4)

124
client-cli/pvc/cli/parsers.py Normal file
View File

@ -0,0 +1,124 @@
#!/usr/bin/env python3
# parsers.py - PVC Click CLI data parser function library
# Part of the Parallel Virtual Cluster (PVC) system
#
# Copyright (C) 2018-2023 Joshua M. Boniface <joshua@boniface.me>
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, version 3.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program. If not, see <https://www.gnu.org/licenses/>.
#
###############################################################################
from os import path
from re import sub
from pvc.cli.helpers import read_config_from_yaml, get_config
import pvc.lib.cluster
def cli_connection_list_parser(connections_config, show_keys_flag):
"""
Parse connections_config into formatable data for cli_connection_list
"""
connections_data = list()
for connection, details in connections_config.items():
if details.get("cfgfile", None) is not None:
if path.isfile(details.get("cfgfile")):
description, address, port, scheme, api_key = read_config_from_yaml(
details.get("cfgfile")
)
else:
continue
if not show_keys_flag and api_key is not None:
api_key = sub(r"[a-z0-9]", "x", api_key)
connections_data.append(
{
"name": connection,
"description": description,
"address": address,
"port": port,
"scheme": scheme,
"api_key": api_key,
}
)
else:
if not show_keys_flag:
details["api_key"] = sub(r"[a-z0-9]", "x", details["api_key"])
connections_data.append(
{
"name": connection,
"description": details["description"],
"address": details["host"],
"port": details["port"],
"scheme": details["scheme"],
"api_key": details["api_key"],
}
)
return connections_data
def cli_connection_detail_parser(connections_config):
"""
Parse connections_config into formatable data for cli_connection_detail
"""
connections_data = list()
for connection, details in connections_config.items():
cluster_config = get_config(connections_config, connection=connection)
if cluster_config.get("badcfg", False):
continue
# Connect to each API and gather cluster status
retcode, retdata = pvc.lib.cluster.get_info(cluster_config)
if retcode == 0:
# Create dummy data of N/A for all fields
connections_data.append(
{
"name": cluster_config["connection"],
"description": cluster_config["description"],
"health": "N/A",
"maintenance": "N/A",
"primary_node": "N/A",
"pvc_version": "N/A",
"nodes": "N/A",
"vms": "N/A",
"networks": "N/A",
"osds": "N/A",
"pools": "N/A",
"volumes": "N/A",
"snapshots": "N/A",
}
)
else:
# Normalize data into nice formattable version
connections_data.append(
{
"name": cluster_config["connection"],
"description": cluster_config["description"],
"health": retdata.get("cluster_health", {}).get("health", "N/A"),
"maintenance": retdata.get("maintenance", "N/A"),
"primary_node": retdata.get("primary_node", "N/A"),
"pvc_version": retdata.get("pvc_version", "N/A"),
"nodes": retdata.get("nodes", {}).get("total", "N/A"),
"vms": retdata.get("vms", {}).get("total", "N/A"),
"networks": retdata.get("networks", "N/A"),
"osds": retdata.get("osds", {}).get("total", "N/A"),
"pools": retdata.get("pools", "N/A"),
"volumes": retdata.get("volumes", "N/A"),
"snapshots": retdata.get("snapshots", "N/A"),
}
)
return connections_data

64
client-cli/pvc/cli/waiters.py Normal file
View File

@ -0,0 +1,64 @@
#!/usr/bin/env python3
# waiters.py - PVC Click CLI output waiters library
# Part of the Parallel Virtual Cluster (PVC) system
#
# Copyright (C) 2018-2023 Joshua M. Boniface <joshua@boniface.me>
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, version 3.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program. If not, see <https://www.gnu.org/licenses/>.
#
###############################################################################
from time import sleep, time
from pvc.cli.helpers import echo
import pvc.lib.node
def cli_node_waiter(config, node, state_field, state_value):
"""
Wait for state transitions for cli_node tasks
{node} is the name of the node
{state_field} is the node_info field to check for {state_value}
{state_value} is the TRANSITIONAL value that, when no longer set, will terminate waiting
"""
# Sleep for this long between API polls
sleep_time = 1
# Print a dot after this many {sleep_time}s
dot_time = 5
t_start = time()
echo(config, "Waiting...", newline=False)
sleep(sleep_time)
count = 0
while True:
count += 1
try:
_retcode, _retdata = pvc.lib.node.node_info(config, node)
if _retdata[state_field] != state_value:
break
else:
raise ValueError
except Exception:
sleep(sleep_time)
if count % dot_time == 0:
echo(config, ".", newline=False)
t_end = time()
echo(config, f" done. [{int(t_end - t_start)}s]")

0
client-cli/pvc/lib/__init__.py Normal file
View File

97
client-cli/pvc/lib/ansiprint.py Normal file
View File

@ -0,0 +1,97 @@
#!/usr/bin/env python3
# ansiprint.py - Printing function for formatted messages
# Part of the Parallel Virtual Cluster (PVC) system
#
# Copyright (C) 2018-2022 Joshua M. Boniface <joshua@boniface.me>
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, version 3.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program. If not, see <https://www.gnu.org/licenses/>.
#
###############################################################################
import datetime
# ANSII colours for output
def red():
return "\033[91m"
def blue():
return "\033[94m"
def cyan():
return "\033[96m"
def green():
return "\033[92m"
def yellow():
return "\033[93m"
def purple():
return "\033[95m"
def bold():
return "\033[1m"
def end():
return "\033[0m"
# Print function
def echo(message, prefix, state):
# Get the date
date = "{} - ".format(datetime.datetime.now().strftime("%Y/%m/%d %H:%M:%S.%f"))
endc = end()
# Continuation
if state == "c":
date = ""
colour = ""
prompt = " "
# OK
elif state == "o":
colour = green()
prompt = ">>> "
# Error
elif state == "e":
colour = red()
prompt = ">>> "
# Warning
elif state == "w":
colour = yellow()
prompt = ">>> "
# Tick
elif state == "t":
colour = purple()
prompt = ">>> "
# Information
elif state == "i":
colour = blue()
prompt = ">>> "
else:
colour = bold()
prompt = ">>> "
# Append space to prefix
if prefix != "":
prefix = prefix + " "
print(colour + prompt + endc + date + prefix + message)

116
client-cli/pvc/lib/cluster.py Normal file
View File

@ -0,0 +1,116 @@
#!/usr/bin/env python3
# cluster.py - PVC CLI client function library, cluster management
# Part of the Parallel Virtual Cluster (PVC) system
#
# Copyright (C) 2018-2022 Joshua M. Boniface <joshua@boniface.me>
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, version 3.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program. If not, see <https://www.gnu.org/licenses/>.
#
###############################################################################
import json
from pvc.lib.common import call_api
def initialize(config, overwrite=False):
"""
Initialize the PVC cluster
API endpoint: GET /api/v1/initialize
API arguments: overwrite, yes-i-really-mean-it
API schema: {json_data_object}
"""
params = {"yes-i-really-mean-it": "yes", "overwrite": overwrite}
response = call_api(config, "post", "/initialize", params=params)
if response.status_code == 200:
retstatus = True
else:
retstatus = False
return retstatus, response.json().get("message", "")
def backup(config):
"""
Get a JSON backup of the cluster
API endpoint: GET /api/v1/backup
API arguments:
API schema: {json_data_object}
"""
response = call_api(config, "get", "/backup")
if response.status_code == 200:
return True, response.json()
else:
return False, response.json().get("message", "")
def restore(config, cluster_data):
"""
Restore a JSON backup to the cluster
API endpoint: POST /api/v1/restore
API arguments: yes-i-really-mean-it
API schema: {json_data_object}
"""
cluster_data_json = json.dumps(cluster_data)
params = {"yes-i-really-mean-it": "yes"}
data = {"cluster_data": cluster_data_json}
response = call_api(config, "post", "/restore", params=params, data=data)
if response.status_code == 200:
retstatus = True
else:
retstatus = False
return retstatus, response.json().get("message", "")
def maintenance_mode(config, state):
"""
Enable or disable PVC cluster maintenance mode
API endpoint: POST /api/v1/status
API arguments: {state}={state}
API schema: {json_data_object}
"""
params = {"state": state}
response = call_api(config, "post", "/status", params=params)
if response.status_code == 200:
retstatus = True
else:
retstatus = False
return retstatus, response.json().get("message", "")
def get_info(config):
"""
Get status of the PVC cluster
API endpoint: GET /api/v1/status
API arguments:
API schema: {json_data_object}
"""
response = call_api(config, "get", "/status")
if response.status_code == 200:
return True, response.json()
else:
return False, response.json().get("message", "")

201
client-cli/pvc/lib/common.py Normal file
View File

@ -0,0 +1,201 @@
#!/usr/bin/env python3
# common.py - PVC CLI client function library, Common functions
# Part of the Parallel Virtual Cluster (PVC) system
#
# Copyright (C) 2018-2022 Joshua M. Boniface <joshua@boniface.me>
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, version 3.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program. If not, see <https://www.gnu.org/licenses/>.
#
###############################################################################
import os
import math
import time
import requests
import click
from urllib3 import disable_warnings
def format_bytes(size_bytes):
byte_unit_matrix = {
"B": 1,
"K": 1024,
"M": 1024 * 1024,
"G": 1024 * 1024 * 1024,
"T": 1024 * 1024 * 1024 * 1024,
"P": 1024 * 1024 * 1024 * 1024 * 1024,
}
human_bytes = "0B"
for unit in sorted(byte_unit_matrix, key=byte_unit_matrix.get):
formatted_bytes = int(math.ceil(size_bytes / byte_unit_matrix[unit]))
if formatted_bytes < 10000:
human_bytes = "{}{}".format(formatted_bytes, unit)
break
return human_bytes
def format_metric(integer):
integer_unit_matrix = {
"": 1,
"K": 1000,
"M": 1000 * 1000,
"B": 1000 * 1000 * 1000,
"T": 1000 * 1000 * 1000 * 1000,
"Q": 1000 * 1000 * 1000 * 1000 * 1000,
}
human_integer = "0"
for unit in sorted(integer_unit_matrix, key=integer_unit_matrix.get):
formatted_integer = int(math.ceil(integer / integer_unit_matrix[unit]))
if formatted_integer < 10000:
human_integer = "{}{}".format(formatted_integer, unit)
break
return human_integer
class UploadProgressBar(object):
def __init__(self, filename, end_message="", end_nl=True):
file_size = os.path.getsize(filename)
file_size_human = format_bytes(file_size)
click.echo("Uploading file (total size {})...".format(file_size_human))
self.length = file_size
self.time_last = int(round(time.time() * 1000)) - 1000
self.bytes_last = 0
self.bytes_diff = 0
self.is_end = False
self.end_message = end_message
self.end_nl = end_nl
if not self.end_nl:
self.end_suffix = " "
else:
self.end_suffix = ""
self.bar = click.progressbar(length=self.length, show_eta=True)
def update(self, monitor):
bytes_cur = monitor.bytes_read
self.bytes_diff += bytes_cur - self.bytes_last
if self.bytes_last == bytes_cur:
self.is_end = True
self.bytes_last = bytes_cur
time_cur = int(round(time.time() * 1000))
if (time_cur - 1000) > self.time_last:
self.time_last = time_cur
self.bar.update(self.bytes_diff)
self.bytes_diff = 0
if self.is_end:
self.bar.update(self.bytes_diff)
self.bytes_diff = 0
click.echo()
click.echo()
if self.end_message:
click.echo(self.end_message + self.end_suffix, nl=self.end_nl)
class ErrorResponse(requests.Response):
def __init__(self, json_data, status_code):
self.json_data = json_data
self.status_code = status_code
def json(self):
return self.json_data
def call_api(
config,
operation,
request_uri,
headers={},
params=None,
data=None,
files=None,
):
# Set the connect timeout to 2 seconds but extremely long (48 hour) data timeout
timeout = (2.05, 172800)
# Craft the URI
uri = "{}://{}{}{}".format(
config["api_scheme"], config["api_host"], config["api_prefix"], request_uri
)
# Craft the authentication header if required
if config["api_key"]:
headers["X-Api-Key"] = config["api_key"]
# Determine the request type and hit the API
disable_warnings()
try:
if operation == "get":
response = requests.get(
uri,
timeout=timeout,
headers=headers,
params=params,
data=data,
verify=config["verify_ssl"],
)
if operation == "post":
response = requests.post(
uri,
timeout=timeout,
headers=headers,
params=params,
data=data,
files=files,
verify=config["verify_ssl"],
)
if operation == "put":
response = requests.put(
uri,
timeout=timeout,
headers=headers,
params=params,
data=data,
files=files,
verify=config["verify_ssl"],
)
if operation == "patch":
response = requests.patch(
uri,
timeout=timeout,
headers=headers,
params=params,
data=data,
verify=config["verify_ssl"],
)
if operation == "delete":
response = requests.delete(
uri,
timeout=timeout,
headers=headers,
params=params,
data=data,
verify=config["verify_ssl"],
)
except Exception as e:
message = "Failed to connect to the API: {}".format(e)
response = ErrorResponse({"message": message}, 500)
# Display debug output
if config["debug"]:
click.echo("API endpoint: {}".format(uri), err=True)
click.echo("Response code: {}".format(response.status_code), err=True)
click.echo("Response headers: {}".format(response.headers), err=True)
click.echo(err=True)
# Return the response object
return response

1495
client-cli/pvc/lib/network.py Normal file
View File

File diff suppressed because it is too large Load Diff

706
client-cli/pvc/lib/node.py Normal file
View File

@ -0,0 +1,706 @@
#!/usr/bin/env python3
# node.py - PVC CLI client function library, node management
# Part of the Parallel Virtual Cluster (PVC) system
#
# Copyright (C) 2018-2022 Joshua M. Boniface <joshua@boniface.me>
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, version 3.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program. If not, see <https://www.gnu.org/licenses/>.
#
###############################################################################
import time
import pvc.lib.ansiprint as ansiprint
from pvc.lib.common import call_api
#
# Primary functions
#
def node_coordinator_state(config, node, action):
"""
Set node coordinator state state (primary/secondary)
API endpoint: POST /api/v1/node/{node}/coordinator-state
API arguments: action={action}
API schema: {"message": "{data}"}
"""
params = {"state": action}
response = call_api(
config,
"post",
"/node/{node}/coordinator-state".format(node=node),
params=params,
)
if response.status_code == 200:
retstatus = True
else:
retstatus = False
return retstatus, response.json().get("message", "")
def node_domain_state(config, node, action):
"""
Set node domain state state (flush/ready)
API endpoint: POST /api/v1/node/{node}/domain-state
API arguments: action={action}, wait={wait}
API schema: {"message": "{data}"}
"""
params = {"state": action}
response = call_api(
config, "post", "/node/{node}/domain-state".format(node=node), params=params
)
if response.status_code == 200:
retstatus = True
else:
retstatus = False
return retstatus, response.json().get("message", "")
def view_node_log(config, node, lines=100):
"""
Return node log lines from the API (and display them in a pager in the main CLI)
API endpoint: GET /node/{node}/log
API arguments: lines={lines}
API schema: {"name":"{node}","data":"{node_log}"}
"""
params = {"lines": lines}
response = call_api(
config, "get", "/node/{node}/log".format(node=node), params=params
)
if response.status_code != 200:
return False, response.json().get("message", "")
node_log = response.json()["data"]
# Shrink the log buffer to length lines
shrunk_log = node_log.split("\n")[-lines:]
loglines = "\n".join(shrunk_log)
return True, loglines
def follow_node_log(config, node, lines=10):
"""
Return and follow node log lines from the API
API endpoint: GET /node/{node}/log
API arguments: lines={lines}
API schema: {"name":"{nodename}","data":"{node_log}"}
"""
# We always grab 200 to match the follow call, but only _show_ `lines` number
params = {"lines": 200}
response = call_api(
config, "get", "/node/{node}/log".format(node=node), params=params
)
if response.status_code != 200:
return False, response.json().get("message", "")
# Shrink the log buffer to length lines
node_log = response.json()["data"]
shrunk_log = node_log.split("\n")[-int(lines) :]
loglines = "\n".join(shrunk_log)
# Print the initial data and begin following
print(loglines, end="")
print("\n", end="")
while True:
# Grab the next line set (200 is a reasonable number of lines per half-second; any more are skipped)
try:
params = {"lines": 200}
response = call_api(
config, "get", "/node/{node}/log".format(node=node), params=params
)
new_node_log = response.json()["data"]
except Exception:
break
# Split the new and old log strings into constitutent lines
old_node_loglines = node_log.split("\n")
new_node_loglines = new_node_log.split("\n")
# Set the node log to the new log value for the next iteration
node_log = new_node_log
# Get the difference between the two sets of lines
old_node_loglines_set = set(old_node_loglines)
diff_node_loglines = [
x for x in new_node_loglines if x not in old_node_loglines_set
]
# If there's a difference, print it out
if len(diff_node_loglines) > 0:
print("\n".join(diff_node_loglines), end="")
print("\n", end="")
# Wait half a second
time.sleep(0.5)
return True, ""
def node_info(config, node):
"""
Get information about node
API endpoint: GET /api/v1/node/{node}
API arguments:
API schema: {json_data_object}
"""
response = call_api(config, "get", "/node/{node}".format(node=node))
if response.status_code == 200:
if isinstance(response.json(), list) and len(response.json()) != 1:
# No exact match, return not found
return False, "Node not found."
else:
# Return a single instance if the response is a list
if isinstance(response.json(), list):
return True, response.json()[0]
# This shouldn't happen, but is here just in case
else:
return True, response.json()
else:
return False, response.json().get("message", "")
def node_list(
config, limit, target_daemon_state, target_coordinator_state, target_domain_state
):
"""
Get list information about nodes (limited by {limit})
API endpoint: GET /api/v1/node
API arguments: limit={limit}
API schema: [{json_data_object},{json_data_object},etc.]
"""
params = dict()
if limit:
params["limit"] = limit
if target_daemon_state:
params["daemon_state"] = target_daemon_state
if target_coordinator_state:
params["coordinator_state"] = target_coordinator_state
if target_domain_state:
params["domain_state"] = target_domain_state
response = call_api(config, "get", "/node", params=params)
if response.status_code == 200:
return True, response.json()
else:
return False, response.json().get("message", "")
#
# Output display functions
#
def getOutputColours(node_information):
node_health = node_information.get("health", "N/A")
if isinstance(node_health, int):
if node_health <= 50:
health_colour = ansiprint.red()
elif node_health <= 90:
health_colour = ansiprint.yellow()
elif node_health <= 100:
health_colour = ansiprint.green()
else:
health_colour = ansiprint.blue()
else:
health_colour = ansiprint.blue()
if node_information["daemon_state"] == "run":
daemon_state_colour = ansiprint.green()
elif node_information["daemon_state"] == "stop":
daemon_state_colour = ansiprint.red()
elif node_information["daemon_state"] == "shutdown":
daemon_state_colour = ansiprint.yellow()
elif node_information["daemon_state"] == "init":
daemon_state_colour = ansiprint.yellow()
elif node_information["daemon_state"] == "dead":
daemon_state_colour = ansiprint.red() + ansiprint.bold()
else:
daemon_state_colour = ansiprint.blue()
if node_information["coordinator_state"] == "primary":
coordinator_state_colour = ansiprint.green()
elif node_information["coordinator_state"] == "secondary":
coordinator_state_colour = ansiprint.blue()
else:
coordinator_state_colour = ansiprint.cyan()
if node_information["domain_state"] == "ready":
domain_state_colour = ansiprint.green()
else:
domain_state_colour = ansiprint.blue()
if node_information["memory"]["allocated"] > node_information["memory"]["total"]:
mem_allocated_colour = ansiprint.yellow()
else:
mem_allocated_colour = ""
if node_information["memory"]["provisioned"] > node_information["memory"]["total"]:
mem_provisioned_colour = ansiprint.yellow()
else:
mem_provisioned_colour = ""
return (
health_colour,
daemon_state_colour,
coordinator_state_colour,
domain_state_colour,
mem_allocated_colour,
mem_provisioned_colour,
)
def format_info(config, node_information, long_output):
(
health_colour,
daemon_state_colour,
coordinator_state_colour,
domain_state_colour,
mem_allocated_colour,
mem_provisioned_colour,
) = getOutputColours(node_information)
# Format a nice output; do this line-by-line then concat the elements at the end
ainformation = []
# Basic information
ainformation.append(
"{}Name:{} {}".format(
ansiprint.purple(),
ansiprint.end(),
node_information["name"],
)
)
ainformation.append(
"{}PVC Version:{} {}".format(
ansiprint.purple(),
ansiprint.end(),
node_information["pvc_version"],
)
)
node_health = node_information.get("health", "N/A")
if isinstance(node_health, int):
node_health_text = f"{node_health}%"
else:
node_health_text = node_health
ainformation.append(
"{}Health:{} {}{}{}".format(
ansiprint.purple(),
ansiprint.end(),
health_colour,
node_health_text,
ansiprint.end(),
)
)
node_health_details = node_information.get("health_details", [])
if long_output:
node_health_messages = "\n ".join(
[f"{plugin['name']}: {plugin['message']}" for plugin in node_health_details]
)
else:
node_health_messages = "\n ".join(
[
f"{plugin['name']}: {plugin['message']}"
for plugin in node_health_details
if int(plugin.get("health_delta", 0)) > 0
]
)
if len(node_health_messages) > 0:
ainformation.append(
"{}Health Plugin Details:{} {}".format(
ansiprint.purple(), ansiprint.end(), node_health_messages
)
)
ainformation.append("")
ainformation.append(
"{}Daemon State:{} {}{}{}".format(
ansiprint.purple(),
ansiprint.end(),
daemon_state_colour,
node_information["daemon_state"],
ansiprint.end(),
)
)
ainformation.append(
"{}Coordinator State:{} {}{}{}".format(
ansiprint.purple(),
ansiprint.end(),
coordinator_state_colour,
node_information["coordinator_state"],
ansiprint.end(),
)
)
ainformation.append(
"{}Domain State:{} {}{}{}".format(
ansiprint.purple(),
ansiprint.end(),
domain_state_colour,
node_information["domain_state"],
ansiprint.end(),
)
)
if long_output:
ainformation.append("")
ainformation.append(
"{}Architecture:{} {}".format(
ansiprint.purple(), ansiprint.end(), node_information["arch"]
)
)
ainformation.append(
"{}Operating System:{} {}".format(
ansiprint.purple(), ansiprint.end(), node_information["os"]
)
)
ainformation.append(
"{}Kernel Version:{} {}".format(
ansiprint.purple(), ansiprint.end(), node_information["kernel"]
)
)
ainformation.append("")
ainformation.append(
"{}Active VM Count:{} {}".format(
ansiprint.purple(), ansiprint.end(), node_information["domains_count"]
)
)
ainformation.append(
"{}Host CPUs:{} {}".format(
ansiprint.purple(), ansiprint.end(), node_information["vcpu"]["total"]
)
)
ainformation.append(
"{}vCPUs:{} {}".format(
ansiprint.purple(), ansiprint.end(), node_information["vcpu"]["allocated"]
)
)
ainformation.append(
"{}Load:{} {}".format(
ansiprint.purple(), ansiprint.end(), node_information["load"]
)
)
ainformation.append(
"{}Total RAM (MiB):{} {}".format(
ansiprint.purple(), ansiprint.end(), node_information["memory"]["total"]
)
)
ainformation.append(
"{}Used RAM (MiB):{} {}".format(
ansiprint.purple(), ansiprint.end(), node_information["memory"]["used"]
)
)
ainformation.append(
"{}Free RAM (MiB):{} {}".format(
ansiprint.purple(), ansiprint.end(), node_information["memory"]["free"]
)
)
ainformation.append(
"{}Allocated RAM (MiB):{} {}{}{}".format(
ansiprint.purple(),
ansiprint.end(),
mem_allocated_colour,
node_information["memory"]["allocated"],
ansiprint.end(),
)
)
ainformation.append(
"{}Provisioned RAM (MiB):{} {}{}{}".format(
ansiprint.purple(),
ansiprint.end(),
mem_provisioned_colour,
node_information["memory"]["provisioned"],
ansiprint.end(),
)
)
# Join it all together
ainformation.append("")
return "\n".join(ainformation)
def format_list(config, node_list):
if node_list == "Node not found.":
return node_list
node_list_output = []
# Determine optimal column widths
node_name_length = 5
pvc_version_length = 8
health_length = 7
daemon_state_length = 7
coordinator_state_length = 12
domain_state_length = 7
domains_count_length = 4
cpu_count_length = 6
load_length = 5
mem_total_length = 6
mem_used_length = 5
mem_free_length = 5
mem_alloc_length = 6
mem_prov_length = 5
for node_information in node_list:
# node_name column
_node_name_length = len(node_information["name"]) + 1
if _node_name_length > node_name_length:
node_name_length = _node_name_length
# node_pvc_version column
_pvc_version_length = len(node_information.get("pvc_version", "N/A")) + 1
if _pvc_version_length > pvc_version_length:
pvc_version_length = _pvc_version_length
# node_health column
node_health = node_information.get("health", "N/A")
if isinstance(node_health, int):
node_health_text = f"{node_health}%"
else:
node_health_text = node_health
_health_length = len(node_health_text) + 1
if _health_length > health_length:
health_length = _health_length
# daemon_state column
_daemon_state_length = len(node_information["daemon_state"]) + 1
if _daemon_state_length > daemon_state_length:
daemon_state_length = _daemon_state_length
# coordinator_state column
_coordinator_state_length = len(node_information["coordinator_state"]) + 1
if _coordinator_state_length > coordinator_state_length:
coordinator_state_length = _coordinator_state_length
# domain_state column
_domain_state_length = len(node_information["domain_state"]) + 1
if _domain_state_length > domain_state_length:
domain_state_length = _domain_state_length
# domains_count column
_domains_count_length = len(str(node_information["domains_count"])) + 1
if _domains_count_length > domains_count_length:
domains_count_length = _domains_count_length
# cpu_count column
_cpu_count_length = len(str(node_information["cpu_count"])) + 1
if _cpu_count_length > cpu_count_length:
cpu_count_length = _cpu_count_length
# load column
_load_length = len(str(node_information["load"])) + 1
if _load_length > load_length:
load_length = _load_length
# mem_total column
_mem_total_length = len(str(node_information["memory"]["total"])) + 1
if _mem_total_length > mem_total_length:
mem_total_length = _mem_total_length
# mem_used column
_mem_used_length = len(str(node_information["memory"]["used"])) + 1
if _mem_used_length > mem_used_length:
mem_used_length = _mem_used_length
# mem_free column
_mem_free_length = len(str(node_information["memory"]["free"])) + 1
if _mem_free_length > mem_free_length:
mem_free_length = _mem_free_length
# mem_alloc column
_mem_alloc_length = len(str(node_information["memory"]["allocated"])) + 1
if _mem_alloc_length > mem_alloc_length:
mem_alloc_length = _mem_alloc_length
# mem_prov column
_mem_prov_length = len(str(node_information["memory"]["provisioned"])) + 1
if _mem_prov_length > mem_prov_length:
mem_prov_length = _mem_prov_length
# Format the string (header)
node_list_output.append(
"{bold}{node_header: <{node_header_length}} {state_header: <{state_header_length}} {resource_header: <{resource_header_length}} {memory_header: <{memory_header_length}}{end_bold}".format(
node_header_length=node_name_length
+ pvc_version_length
+ health_length
+ 2,
state_header_length=daemon_state_length
+ coordinator_state_length
+ domain_state_length
+ 2,
resource_header_length=domains_count_length
+ cpu_count_length
+ load_length
+ 2,
memory_header_length=mem_total_length
+ mem_used_length
+ mem_free_length
+ mem_alloc_length
+ mem_prov_length
+ 4,
bold=ansiprint.bold(),
end_bold=ansiprint.end(),
node_header="Nodes "
+ "".join(
[
"-"
for _ in range(
6, node_name_length + pvc_version_length + health_length + 1
)
]
),
state_header="States "
+ "".join(
[
"-"
for _ in range(
7,
daemon_state_length
+ coordinator_state_length
+ domain_state_length
+ 1,
)
]
),
resource_header="Resources "
+ "".join(
[
"-"
for _ in range(
10, domains_count_length + cpu_count_length + load_length + 1
)
]
),
memory_header="Memory (M) "
+ "".join(
[
"-"
for _ in range(
11,
mem_total_length
+ mem_used_length
+ mem_free_length
+ mem_alloc_length
+ mem_prov_length
+ 3,
)
]
),
)
)
node_list_output.append(
"{bold}{node_name: <{node_name_length}} {node_pvc_version: <{pvc_version_length}} {node_health: <{health_length}} \
{daemon_state_colour}{node_daemon_state: <{daemon_state_length}}{end_colour} {coordinator_state_colour}{node_coordinator_state: <{coordinator_state_length}}{end_colour} {domain_state_colour}{node_domain_state: <{domain_state_length}}{end_colour} \
{node_domains_count: <{domains_count_length}} {node_cpu_count: <{cpu_count_length}} {node_load: <{load_length}} \
{node_mem_total: <{mem_total_length}} {node_mem_used: <{mem_used_length}} {node_mem_free: <{mem_free_length}} {node_mem_allocated: <{mem_alloc_length}} {node_mem_provisioned: <{mem_prov_length}}{end_bold}".format(
node_name_length=node_name_length,
pvc_version_length=pvc_version_length,
health_length=health_length,
daemon_state_length=daemon_state_length,
coordinator_state_length=coordinator_state_length,
domain_state_length=domain_state_length,
domains_count_length=domains_count_length,
cpu_count_length=cpu_count_length,
load_length=load_length,
mem_total_length=mem_total_length,
mem_used_length=mem_used_length,
mem_free_length=mem_free_length,
mem_alloc_length=mem_alloc_length,
mem_prov_length=mem_prov_length,
bold=ansiprint.bold(),
end_bold=ansiprint.end(),
daemon_state_colour="",
coordinator_state_colour="",
domain_state_colour="",
end_colour="",
node_name="Name",
node_pvc_version="Version",
node_health="Health",
node_daemon_state="Daemon",
node_coordinator_state="Coordinator",
node_domain_state="Domain",
node_domains_count="VMs",
node_cpu_count="vCPUs",
node_load="Load",
node_mem_total="Total",
node_mem_used="Used",
node_mem_free="Free",
node_mem_allocated="Alloc",
node_mem_provisioned="Prov",
)
)
# Format the string (elements)
for node_information in sorted(node_list, key=lambda n: n["name"]):
(
health_colour,
daemon_state_colour,
coordinator_state_colour,
domain_state_colour,
mem_allocated_colour,
mem_provisioned_colour,
) = getOutputColours(node_information)
node_health = node_information.get("health", "N/A")
if isinstance(node_health, int):
node_health_text = f"{node_health}%"
else:
node_health_text = node_health
node_list_output.append(
"{bold}{node_name: <{node_name_length}} {node_pvc_version: <{pvc_version_length}} {health_colour}{node_health: <{health_length}}{end_colour} \
{daemon_state_colour}{node_daemon_state: <{daemon_state_length}}{end_colour} {coordinator_state_colour}{node_coordinator_state: <{coordinator_state_length}}{end_colour} {domain_state_colour}{node_domain_state: <{domain_state_length}}{end_colour} \
{node_domains_count: <{domains_count_length}} {node_cpu_count: <{cpu_count_length}} {node_load: <{load_length}} \
{node_mem_total: <{mem_total_length}} {node_mem_used: <{mem_used_length}} {node_mem_free: <{mem_free_length}} {mem_allocated_colour}{node_mem_allocated: <{mem_alloc_length}}{end_colour} {mem_provisioned_colour}{node_mem_provisioned: <{mem_prov_length}}{end_colour}{end_bold}".format(
node_name_length=node_name_length,
pvc_version_length=pvc_version_length,
health_length=health_length,
daemon_state_length=daemon_state_length,
coordinator_state_length=coordinator_state_length,
domain_state_length=domain_state_length,
domains_count_length=domains_count_length,
cpu_count_length=cpu_count_length,
load_length=load_length,
mem_total_length=mem_total_length,
mem_used_length=mem_used_length,
mem_free_length=mem_free_length,
mem_alloc_length=mem_alloc_length,
mem_prov_length=mem_prov_length,
bold="",
end_bold="",
health_colour=health_colour,
daemon_state_colour=daemon_state_colour,
coordinator_state_colour=coordinator_state_colour,
domain_state_colour=domain_state_colour,
mem_allocated_colour=mem_allocated_colour,
mem_provisioned_colour=mem_allocated_colour,
end_colour=ansiprint.end(),
node_name=node_information["name"],
node_pvc_version=node_information.get("pvc_version", "N/A"),
node_health=node_health_text,
node_daemon_state=node_information["daemon_state"],
node_coordinator_state=node_information["coordinator_state"],
node_domain_state=node_information["domain_state"],
node_domains_count=node_information["domains_count"],
node_cpu_count=node_information["vcpu"]["allocated"],
node_load=node_information["load"],
node_mem_total=node_information["memory"]["total"],
node_mem_used=node_information["memory"]["used"],
node_mem_free=node_information["memory"]["free"],
node_mem_allocated=node_information["memory"]["allocated"],
node_mem_provisioned=node_information["memory"]["provisioned"],
)
)
return "\n".join(node_list_output)

2019
client-cli/pvc/lib/provisioner.py Normal file
View File

File diff suppressed because it is too large Load Diff

2601
client-cli/pvc/lib/storage.py Normal file
View File

File diff suppressed because it is too large Load Diff

2040
client-cli/pvc/lib/vm.py Normal file
View File

File diff suppressed because it is too large Load Diff

102
client-cli/pvc/lib/zkhandler.py Normal file
View File

@ -0,0 +1,102 @@
#!/usr/bin/env python3
# zkhandler.py - Secure versioned ZooKeeper updates
# Part of the Parallel Virtual Cluster (PVC) system
#
# Copyright (C) 2018-2022 Joshua M. Boniface <joshua@boniface.me>
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, version 3.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with this program. If not, see <https://www.gnu.org/licenses/>.
#
###############################################################################
import uuid
# Exists function
def exists(zk_conn, key):
stat = zk_conn.exists(key)
if stat:
return True
else:
return False
# Child list function
def listchildren(zk_conn, key):
children = zk_conn.get_children(key)
return children
# Delete key function
def deletekey(zk_conn, key, recursive=True):
zk_conn.delete(key, recursive=recursive)
# Data read function
def readdata(zk_conn, key):
data_raw = zk_conn.get(key)
data = data_raw[0].decode("utf8")
return data
# Data write function
def writedata(zk_conn, kv):
# Start up a transaction
zk_transaction = zk_conn.transaction()
# Proceed one KV pair at a time
for key in sorted(kv):
data = kv[key]
# Check if this key already exists or not
if not zk_conn.exists(key):
# We're creating a new key
zk_transaction.create(key, str(data).encode("utf8"))
else:
# We're updating a key with version validation
orig_data = zk_conn.get(key)
version = orig_data[1].version
# Set what we expect the new version to be
new_version = version + 1
# Update the data
zk_transaction.set_data(key, str(data).encode("utf8"))
# Set up the check
try:
zk_transaction.check(key, new_version)
except TypeError:
print('Zookeeper key "{}" does not match expected version'.format(key))
return False
# Commit the transaction
try:
zk_transaction.commit()
return True
except Exception:
return False
# Write lock function
def writelock(zk_conn, key):
lock_id = str(uuid.uuid1())
lock = zk_conn.WriteLock("{}".format(key), lock_id)
return lock
# Read lock function
def readlock(zk_conn, key):
lock_id = str(uuid.uuid1())
lock = zk_conn.ReadLock("{}".format(key), lock_id)
return lock

20
client-cli/setup.py Normal file
View File

@ -0,0 +1,20 @@
from setuptools import setup
setup(
name="pvc",
version="0.9.63",
packages=["pvc", "pvc.lib"],
install_requires=[
"Click",
"PyYAML",
"lxml",
"colorama",
"requests",
"requests-toolbelt",
],
entry_points={
"console_scripts": [
"pvc = pvc.pvc:cli",
],
},
)