Compare commits

...

15 Commits

Author SHA1 Message Date
7274c1c8e6 Bump version to 1.0 2025-06-05 00:20:59 -04:00
479482ef75 Fix version import location 2025-06-05 00:20:53 -04:00
cfb13fbccf Fix invalid path ref 2025-06-05 00:04:12 -04:00
9ca0217154 Avoid fatal faults if volume stats are invalid
Returns garbage data, but avoids causing the entire API call to fail if
this happens on very old clusters (or after any sort of ZK corruption).
2025-06-04 23:51:12 -04:00
9ec2884e40 Fix bug where RBD list is not updated on rename 2025-03-17 18:05:47 -04:00
e5c8318b8e Integrate version hardcoding into CLI
The get_distribution load was very heavy, so follow the other parts and
just hardcode the version in. Also update bump-version to work with this
and with the new pyproject.toml configuration, and add a custom
user-agent to the API requests leveraging this version.
2025-03-17 18:05:44 -04:00
5831c3a425 Fix build errors 2025-03-17 18:05:41 -04:00
bb77c5f1fc Move lxml imports to runtime
Avoid loading these if unneeded
2025-03-12 23:50:12 -04:00
fc740927cc Switch to modern Python build system
Remove setuptools and use pyproject.toml instead.
2025-03-12 23:46:52 -04:00
34149fe933 Move multipart.encoder import to runtime
Another library with a ridiculously long load time.
2025-03-12 23:11:42 -04:00
a2fed1885c Remove distutils strtobool
This import takes a ridiculously long time just to implement a function
that can be done in one line in O(1) time.
2025-03-12 23:09:44 -04:00
ee055bdb81 Improve loading efficiency of common.py 2025-03-12 22:55:11 -04:00
60967b5606 Fix formatters colour bug for mirror state 2025-03-02 14:58:26 -05:00
89bfbe1fd8 Add translation of domain UUIDs to names
Allows frontends to better handle the domain list gracefully, as humans
don't care about the UUIDs.
2025-02-28 21:52:42 -05:00
be092756a9 Add cluster name to Zookeeper and log+API output 2025-02-27 00:57:07 -05:00
25 changed files with 148 additions and 81 deletions

1
.gitignore vendored
View File

@ -8,3 +8,4 @@ debian/pvc-*/
debian/*.log
debian/*.substvars
debian/files
client-cli/build/

View File

@ -1 +1 @@
0.9.107
1.0

View File

@ -1,5 +1,18 @@
## PVC Changelog
###### [v1.0](https://github.com/parallelvirtualcluster/pvc/releases/tag/v1.0)
**Announcement**: We are pleased to announce PVC 1.0! Functionally speaking, there are only a few minor improvements over the previous 0.9.107, but I believe it's finally time to call this a "1.0" release. Recently I have had much less opportunity to work on PVC as I would like, so some features are still not quite there, but those can arrive in future versions over time.
**Enhancement**: The PVC CLI has been made much more efficient in terms of imports, allowing it to run on much lower spec hardware (in my case, on a small SBC). It's still not perfect, but multi-second import times are no longer an issue. The CLI client has also been moved to a more modern build system in preparation for Debian 13 "Trixie".
* [Daemons] Add cluster name to outputs during startup
* [CLI Client] Translate domain UUIDs to names in full node detail output for better readability
* [CLI Client] Fix colouring bug for mirror state
* [CLI Client] Significantly improve import efficiency throughout the client to avoid long load times on slow hardware
* [CLI Client] Port build to pyproject.toml and increase Debuild compat to 13
* [API Daemon] Fix bug with RBD list update after VM rename
* [API Daemon] Fix bug/crash if Ceph volume stats are invalid/empty
###### [v0.9.107](https://github.com/parallelvirtualcluster/pvc/releases/tag/v0.9.107)
* [Worker Daemon] Fixes a bug where snapshot removal fails during autobackups

View File

@ -81,6 +81,7 @@ def create_app():
print("|--------------------------------------------------------------|")
print("| Parallel Virtual Cluster API daemon v{0: <23} |".format(version))
print("| Debug: {0: <53} |".format(str(config["debug"])))
print("| Cluster: {0: <51} |".format(config["cluster_name"]))
print("| API version: v{0: <46} |".format(API_VERSION))
print(
"| Listen: {0: <52} |".format(

View File

@ -22,7 +22,8 @@ sed -i "s,version = \"${current_version}\",version = \"${new_version}\"," node-d
sed -i "s,version = \"${current_version}\",version = \"${new_version}\"," health-daemon/pvchealthd/Daemon.py
sed -i "s,version = \"${current_version}\",version = \"${new_version}\"," worker-daemon/pvcworkerd/Daemon.py
sed -i "s,version = \"${current_version}\",version = \"${new_version}\"," api-daemon/pvcapid/Daemon.py
sed -i "s,version=\"${current_version}\",version=\"${new_version}\"," client-cli/setup.py
sed -i "s,version = \"${current_version}\",version = \"${new_version}\"," client-cli/pyproject.toml
sed -i "s,VERSION = \"${current_version}\",VERSION = \"${new_version}\"," client-cli/pvc/cli/helpers.py
echo ${new_version} > .version
changelog_tmpdir=$( mktemp -d )
@ -47,7 +48,7 @@ echo -e "${deb_changelog_new}" >> ${deb_changelog_file}
echo -e "${deb_changelog_orig}" >> ${deb_changelog_file}
mv ${deb_changelog_file} debian/changelog
git add node-daemon/pvcnoded/Daemon.py health-daemon/pvchealthd/Daemon.py worker-daemon/pvcworkerd/Daemon.py api-daemon/pvcapid/Daemon.py client-cli/setup.py debian/changelog CHANGELOG.md .version
git add node-daemon/pvcnoded/Daemon.py health-daemon/pvchealthd/Daemon.py worker-daemon/pvcworkerd/Daemon.py api-daemon/pvcapid/Daemon.py client-cli/pvc/cli/helpers.py client-cli/pyproject.toml debian/changelog CHANGELOG.md .version
git commit -v
popd &>/dev/null

View File

@ -25,7 +25,6 @@ from functools import wraps
from json import dump as jdump
from json import dumps as jdumps
from json import loads as jloads
from lxml.etree import fromstring, tostring
from os import environ, makedirs, path
from re import sub, match
from yaml import load as yload
@ -98,10 +97,7 @@ def version(ctx, param, value):
if not value or ctx.resilient_parsing:
return
from pkg_resources import get_distribution
version = get_distribution("pvc").version
echo(CLI_CONFIG, f"Parallel Virtual Cluster CLI client version {version}")
echo(CLI_CONFIG, f"Parallel Virtual Cluster CLI client version {VERSION}")
ctx.exit()
@ -1192,6 +1188,8 @@ def cli_vm_define(
# Verify our XML is sensible
try:
from lxml.etree import fromstring, tostring
xml_data = fromstring(vmconfig_data)
new_cfg = tostring(xml_data, pretty_print=True).decode("utf8")
except Exception:
@ -1377,6 +1375,9 @@ def cli_vm_modify(
# Grab the current config
current_vm_cfg_raw = vm_information.get("xml")
from lxml.etree import fromstring, tostring
xml_data = fromstring(current_vm_cfg_raw)
current_vm_cfgfile = tostring(xml_data, pretty_print=True).decode("utf8").strip()
@ -1435,6 +1436,8 @@ def cli_vm_modify(
# Verify our XML is sensible
try:
from lxml.etree import fromstring, tostring
xml_data = fromstring(new_vm_cfgfile)
new_cfg = tostring(xml_data, pretty_print=True).decode("utf8")
except Exception as e:
@ -3265,6 +3268,9 @@ def cli_vm_dump(filename, domain):
finish(False, 'ERROR: Could not find VM "{}"!'.format(domain))
current_vm_cfg_raw = retdata.get("xml")
from lxml.etree import fromstring, tostring
xml_data = fromstring(current_vm_cfg_raw)
current_vm_cfgfile = tostring(xml_data, pretty_print=True).decode("utf8")
xml = current_vm_cfgfile.strip()

View File

@ -221,7 +221,7 @@ def cli_cluster_status_format_pretty(CLI_CONFIG, data):
continue
if state in ["start"]:
state_colour = ansii["green"]
elif state in ["migrate", "disable", "provision", "mirror"]:
elif state in ["migrate", "disable", "provision"]:
state_colour = ansii["blue"]
elif state in ["mirror"]:
state_colour = ansii["purple"]

View File

@ -20,7 +20,6 @@
###############################################################################
from click import echo as click_echo
from distutils.util import strtobool
from json import load as jload
from json import dump as jdump
from os import chmod, environ, getpid, path, get_terminal_size
@ -31,6 +30,8 @@ from yaml import load as yload
from yaml import SafeLoader
VERSION = "1.0"
DEFAULT_STORE_DATA = {"cfgfile": "/etc/pvc/pvc.conf"}
DEFAULT_STORE_FILENAME = "pvc.json"
DEFAULT_API_PREFIX = "/api/v1"
@ -150,9 +151,7 @@ def get_config(store_data, connection=None):
if connection == "local":
config["verify_ssl"] = False
else:
config["verify_ssl"] = bool(
strtobool(environ.get("PVC_CLIENT_VERIFY_SSL", "True"))
)
config["verify_ssl"] = environ.get("PVC_CLIENT_VERIFY_SSL", "True") == "True"
return config

View File

@ -19,13 +19,15 @@
#
###############################################################################
import os
import math
import time
import requests
import click
from ast import literal_eval
from click import echo, progressbar
from math import ceil
from os.path import getsize
from requests import get, post, put, patch, delete, Response
from requests.exceptions import ConnectionError
from time import time
from urllib3 import disable_warnings
from pvc.cli.helpers import VERSION
def format_bytes(size_bytes):
@ -39,7 +41,7 @@ def format_bytes(size_bytes):
}
human_bytes = "0B"
for unit in sorted(byte_unit_matrix, key=byte_unit_matrix.get):
formatted_bytes = int(math.ceil(size_bytes / byte_unit_matrix[unit]))
formatted_bytes = int(ceil(size_bytes / byte_unit_matrix[unit]))
if formatted_bytes < 10000:
human_bytes = "{}{}".format(formatted_bytes, unit)
break
@ -57,7 +59,7 @@ def format_metric(integer):
}
human_integer = "0"
for unit in sorted(integer_unit_matrix, key=integer_unit_matrix.get):
formatted_integer = int(math.ceil(integer / integer_unit_matrix[unit]))
formatted_integer = int(ceil(integer / integer_unit_matrix[unit]))
if formatted_integer < 10000:
human_integer = "{}{}".format(formatted_integer, unit)
break
@ -97,12 +99,12 @@ def format_age(age_secs):
class UploadProgressBar(object):
def __init__(self, filename, end_message="", end_nl=True):
file_size = os.path.getsize(filename)
file_size = getsize(filename)
file_size_human = format_bytes(file_size)
click.echo("Uploading file (total size {})...".format(file_size_human))
echo("Uploading file (total size {})...".format(file_size_human))
self.length = file_size
self.time_last = int(round(time.time() * 1000)) - 1000
self.time_last = int(round(time() * 1000)) - 1000
self.bytes_last = 0
self.bytes_diff = 0
self.is_end = False
@ -114,7 +116,7 @@ class UploadProgressBar(object):
else:
self.end_suffix = ""
self.bar = click.progressbar(length=self.length, width=20, show_eta=True)
self.bar = progressbar(length=self.length, width=20, show_eta=True)
def update(self, monitor):
bytes_cur = monitor.bytes_read
@ -123,7 +125,7 @@ class UploadProgressBar(object):
self.is_end = True
self.bytes_last = bytes_cur
time_cur = int(round(time.time() * 1000))
time_cur = int(round(time() * 1000))
if (time_cur - 1000) > self.time_last:
self.time_last = time_cur
self.bar.update(self.bytes_diff)
@ -132,13 +134,13 @@ class UploadProgressBar(object):
if self.is_end:
self.bar.update(self.bytes_diff)
self.bytes_diff = 0
click.echo()
click.echo()
echo()
echo()
if self.end_message:
click.echo(self.end_message + self.end_suffix, nl=self.end_nl)
echo(self.end_message + self.end_suffix, nl=self.end_nl)
class ErrorResponse(requests.Response):
class ErrorResponse(Response):
def __init__(self, json_data, status_code, headers):
self.json_data = json_data
self.status_code = status_code
@ -165,6 +167,9 @@ def call_api(
config["api_scheme"], config["api_host"], config["api_prefix"], request_uri
)
# Add custom User-Agent header
headers["User-Agent"] = f"pvc-client-cli/{VERSION}"
# Craft the authentication header if required
if config["api_key"]:
headers["X-Api-Key"] = config["api_key"]
@ -178,7 +183,7 @@ def call_api(
for i in range(3):
failed = False
try:
response = requests.get(
response = get(
uri,
timeout=timeout,
headers=headers,
@ -190,16 +195,14 @@ def call_api(
failed = True
continue
break
except requests.exceptions.ConnectionError:
except ConnectionError:
failed = True
continue
if failed:
error = f"Code {response.status_code}" if response else "Timeout"
raise requests.exceptions.ConnectionError(
f"Failed to connect after 3 tries ({error})"
)
raise ConnectionError(f"Failed to connect after 3 tries ({error})")
if operation == "post":
response = requests.post(
response = post(
uri,
timeout=timeout,
headers=headers,
@ -209,7 +212,7 @@ def call_api(
verify=config["verify_ssl"],
)
if operation == "put":
response = requests.put(
response = put(
uri,
timeout=timeout,
headers=headers,
@ -219,7 +222,7 @@ def call_api(
verify=config["verify_ssl"],
)
if operation == "patch":
response = requests.patch(
response = patch(
uri,
timeout=timeout,
headers=headers,
@ -228,7 +231,7 @@ def call_api(
verify=config["verify_ssl"],
)
if operation == "delete":
response = requests.delete(
response = patch, delete(
uri,
timeout=timeout,
headers=headers,
@ -243,10 +246,10 @@ def call_api(
# Display debug output
if config["debug"]:
click.echo("API endpoint: {}".format(uri), err=True)
click.echo("Response code: {}".format(response.status_code), err=True)
click.echo("Response headers: {}".format(response.headers), err=True)
click.echo(err=True)
echo("API endpoint: {}".format(uri), err=True)
echo("Response code: {}".format(response.status_code), err=True)
echo("Response headers: {}".format(response.headers), err=True)
echo(err=True)
# Return the response object
return response

View File

@ -19,11 +19,6 @@
#
###############################################################################
from requests_toolbelt.multipart.encoder import (
MultipartEncoder,
MultipartEncoderMonitor,
)
import pvc.lib.ansiprint as ansiprint
from pvc.lib.common import UploadProgressBar, call_api, get_wait_retdata
@ -549,6 +544,12 @@ def ova_upload(config, name, ova_file, params):
bar = UploadProgressBar(
ova_file, end_message="Parsing file on remote side...", end_nl=False
)
from requests_toolbelt.multipart.encoder import (
MultipartEncoder,
MultipartEncoderMonitor,
)
upload_data = MultipartEncoder(
fields={"file": ("filename", open(ova_file, "rb"), "application/octet-stream")}
)

View File

@ -23,10 +23,6 @@ import math
from os import path
from json import loads
from requests_toolbelt.multipart.encoder import (
MultipartEncoder,
MultipartEncoderMonitor,
)
import pvc.lib.ansiprint as ansiprint
from pvc.lib.common import UploadProgressBar, call_api, get_wait_retdata
@ -1212,6 +1208,12 @@ def ceph_volume_upload(config, pool, volume, image_format, image_file):
bar = UploadProgressBar(
image_file, end_message="Parsing file on remote side...", end_nl=False
)
from requests_toolbelt.multipart.encoder import (
MultipartEncoder,
MultipartEncoderMonitor,
)
upload_data = MultipartEncoder(
fields={
"file": ("filename", open(image_file, "rb"), "application/octet-stream")

21
client-cli/pyproject.toml Normal file
View File

@ -0,0 +1,21 @@
[build-system]
requires = ["setuptools", "wheel"]
build-backend = "setuptools.build_meta"
[project]
name = "pvc"
version = "1.0"
dependencies = [
"Click",
"PyYAML",
"lxml",
"colorama",
"requests",
"requests-toolbelt",
]
[tool.setuptools]
packages = ["pvc.cli", "pvc.lib"]
[project.scripts]
pvc = "pvc.cli.cli:cli"

View File

@ -1,20 +0,0 @@
from setuptools import setup
setup(
name="pvc",
version="0.9.107",
packages=["pvc.cli", "pvc.lib"],
install_requires=[
"Click",
"PyYAML",
"lxml",
"colorama",
"requests",
"requests-toolbelt",
],
entry_points={
"console_scripts": [
"pvc = pvc.cli.cli:cli",
],
},
)

View File

@ -555,9 +555,16 @@ def getCephVolumes(zkhandler, pool):
def getVolumeInformation(zkhandler, pool, volume):
# Parse the stats data
volume_stats_raw = zkhandler.read(("volume.stats", f"{pool}/{volume}"))
volume_stats = dict(json.loads(volume_stats_raw))
# Format the size to something nicer
volume_stats["size"] = format_bytes_tohuman(volume_stats["size"])
try:
volume_stats = dict(json.loads(volume_stats_raw))
# Format the size to something nicer
volume_stats["size"] = format_bytes_tohuman(volume_stats["size"])
except Exception:
volume_stats = dict(
json.loads(
f'{"name":"{volume}","id":"","size":0,"objects":0,"order":0,"object_size":0,"snapshot_count":0,"block_name_prefix":"","format":0,"features":[],"op_features":[],"flags":[],"create_timestamp":"","access_timestamp":"","modify_timestamp":""}'
)
)
volume_information = {"name": volume, "pool": pool, "stats": volume_stats}
return volume_information

View File

@ -496,6 +496,7 @@ def getClusterInformation(zkhandler):
# Format the status data
cluster_information = {
"cluster_name": zkhandler.read("base.config"),
"cluster_health": getClusterHealthFromFaults(zkhandler, faults_data),
"node_health": getNodeHealth(zkhandler, node_list),
"maintenance": maintenance_state,

View File

@ -1212,3 +1212,7 @@ def get_detect_device(detect_string):
return device
else:
return None
def translate_domains_to_names(zkhandler, domain_list):
return list(zkhandler.read_many([("domain.name", d) for d in domain_list]))

View File

@ -142,7 +142,9 @@ def getNodeInformation(zkhandler, node_name):
node_mem_free = int(_node_mem_free)
node_load = float(_node_load)
node_domains_count = int(_node_domains_count)
node_running_domains = _node_running_domains.split()
node_running_domains = common.translate_domains_to_names(
zkhandler, _node_running_domains.split()
)
try:
node_health = int(_node_health)

View File

@ -605,12 +605,14 @@ def rename_vm(zkhandler, domain, new_domain):
rbd_list.append(disk["name"].split("/")[1])
# Rename each volume in turn
rbd_list_new = []
for idx, rbd in enumerate(rbd_list):
rbd_new = re.sub(r"{}".format(domain), new_domain, rbd)
# Skip renaming if nothing changed
if rbd_new == rbd:
continue
ceph.rename_volume(zkhandler, pool_list[idx], rbd, rbd_new)
rbd_list_new.append(f"{pool_list[idx]}/{rbd_new}")
# Replace the name in the config
vm_config_new = (
@ -627,6 +629,7 @@ def rename_vm(zkhandler, domain, new_domain):
[
(("domain", dom_uuid), new_domain),
(("domain.xml", dom_uuid), vm_config_new),
(("domain.rbdlist", dom_uuid), ",".join(rbd_list_new)),
]
)

15
debian/changelog vendored
View File

@ -1,3 +1,18 @@
pvc (1.0-0) unstable; urgency=high
**Announcement**: We are pleased to announce PVC 1.0! Functionally speaking, there are only a few minor improvements over the previous 0.9.107, but I believe it's finally time to call this a "1.0" release. Recently I have had much less opportunity to work on PVC as I would like, so some features are still not quite there, but those can arrive in future versions over time.
**Enhancement**: The PVC CLI has been made much more efficient in terms of imports, allowing it to run on much lower spec hardware (in my case, on a small SBC). It's still not perfect, but multi-second import times are no longer an issue. The CLI client has also been moved to a more modern build system in preparation for Debian 13 "Trixie".
* [Daemons] Add cluster name to outputs during startup
* [CLI Client] Translate domain UUIDs to names in full node detail output for better readability
* [CLI Client] Fix colouring bug for mirror state
* [CLI Client] Significantly improve import efficiency throughout the client to avoid long load times on slow hardware
* [CLI Client] Port build to pyproject.toml and increase Debuild compat to 13
* [API Daemon] Fix bug with RBD list update after VM rename
* [API Daemon] Fix bug/crash if Ceph volume stats are invalid/empty
-- Joshua M. Boniface <joshua@boniface.me> Thu, 05 Jun 2025 00:04:54 -0400
pvc (0.9.107-0) unstable; urgency=high
* [Worker Daemon] Fixes a bug where snapshot removal fails during autobackups

2
debian/compat vendored
View File

@ -1 +1 @@
9
13

1
debian/control vendored
View File

@ -5,6 +5,7 @@ Maintainer: Joshua Boniface <joshua@boniface.me>
Standards-Version: 3.9.8
Homepage: https://www.boniface.me
X-Python3-Version: >= 3.7
Build-Depends: pybuild-plugin-pyproject, dh-python
Package: pvc-daemon-node
Architecture: all

5
debian/rules vendored
View File

@ -7,13 +7,14 @@ export DH_VERBOSE = 1
dh $@ --with python3
override_dh_python3:
cd $(CURDIR)/client-cli; pybuild --system=distutils --dest-dir=../debian/pvc-client-cli/
cd $(CURDIR)/client-cli; pybuild --system=pyproject --dest-dir=../debian/pvc-client-cli/
mkdir -p debian/pvc-client-cli/usr/lib/python3
mv debian/pvc-client-cli/usr/lib/python3*/* debian/pvc-client-cli/usr/lib/python3/
rm -r $(CURDIR)/client-cli/.pybuild $(CURDIR)/client-cli/pvc.egg-info
override_dh_auto_clean:
find . -name "__pycache__" -o -name ".pybuild" -exec rm -fr {} + || true
find $(CURDIR) -name "__pycache__" -o -name ".pybuild" -exec rm -fr {} + || true
rm -r $(CURDIR)/client-cli/build || true
# If you need to rebuild the Sphinx documentation
# Add spinxdoc to the dh --with line

View File

@ -33,7 +33,7 @@ import os
import signal
# Daemon version
version = "0.9.107"
version = "1.0"
##########################################################
@ -64,6 +64,7 @@ def entrypoint():
logger.out("|--------------------------------------------------------------|")
logger.out("| Parallel Virtual Cluster health daemon v{0: <20} |".format(version))
logger.out("| Debug: {0: <53} |".format(str(config["debug"])))
logger.out("| Cluster: {0: <51} |".format(config["cluster_name"]))
logger.out("| FQDN: {0: <54} |".format(config["node_fqdn"]))
logger.out("| Host: {0: <54} |".format(config["node_hostname"]))
logger.out("| ID: {0: <56} |".format(config["node_id"]))

View File

@ -49,7 +49,7 @@ import re
import json
# Daemon version
version = "0.9.107"
version = "1.0"
##########################################################
@ -83,6 +83,7 @@ def entrypoint():
logger.out("|--------------------------------------------------------------|")
logger.out("| Parallel Virtual Cluster node daemon v{0: <22} |".format(version))
logger.out("| Debug: {0: <53} |".format(str(config["debug"])))
logger.out("| Cluster: {0: <51} |".format(config["cluster_name"]))
logger.out("| FQDN: {0: <54} |".format(config["node_fqdn"]))
logger.out("| Host: {0: <54} |".format(config["node_hostname"]))
logger.out("| ID: {0: <56} |".format(config["node_id"]))
@ -301,6 +302,9 @@ def entrypoint():
# Set up this node in Zookeeper
pvcnoded.util.zookeeper.setup_node(logger, config, zkhandler)
# Set the cluster name in Zookeeper
zkhandler.write([("base.config", config["cluster_name"])])
# Check that the primary node key exists and create it with us as primary if not
try:
current_primary = zkhandler.read("base.config.primary_node")

View File

@ -58,7 +58,7 @@ from daemon_lib.automirror import (
)
# Daemon version
version = "0.9.107"
version = "1.0"
config = cfg.get_configuration()