Bump version to 0.9.73

Move the old manage script to _legacy, and add a new _flask version with
modern Flask tooling. Decide which one to call via pvc-api-db-migrate
using /etc/debian_version call.

.version

@@ -1 +1 @@
-0.9.68
+0.9.73

CHANGELOG.md

@@ -1,5 +1,26 @@
 ## PVC Changelog
 
+###### [v0.9.73](https://github.com/parallelvirtualcluster/pvc/releases/tag/v0.9.73)
+
+  * [Node Daemon] Fixes a bug creating monitoring instance
+
+###### [v0.9.72](https://github.com/parallelvirtualcluster/pvc/releases/tag/v0.9.72)
+
+  * [CLI] Restores old functionality for default node value
+
+###### [v0.9.71](https://github.com/parallelvirtualcluster/pvc/releases/tag/v0.9.71)
+
+  * [API] Adds API support for Debian Bookworm
+
+###### [v0.9.70](https://github.com/parallelvirtualcluster/pvc/releases/tag/v0.9.70)
+
+  * [Node Daemon] Fixes several compatibility issues for Debian 12 "Bookworm"
+
+###### [v0.9.69](https://github.com/parallelvirtualcluster/pvc/releases/tag/v0.9.69)
+
+  * [Node Daemon] Ensures that system load is always 2 decimal places on Bookworm
+  * [Node Daemon] Fixes bug blocking primary takeover at DNS Aggregator start if Patroni is down
+
 ###### [v0.9.68](https://github.com/parallelvirtualcluster/pvc/releases/tag/v0.9.68)
 
   * [CLI] Fixes another bug with network info view

api-daemon/pvc-api-db-upgrade

@@ -11,5 +11,16 @@ if [[ ! -f ${PVC_CONFIG_FILE} ]]; then
 fi
 
 pushd /usr/share/pvc
-./pvcapid-manage.py db upgrade
+
+case "$( cat /etc/debian_version )" in
+    10.*|11.*)
+        # Debian 10 & 11
+        ./pvcapid-manage_legacy.py db upgrade
+    ;;
+    *)
+        # Debian 12+
+        flask --app ./pvcapid-manage_flask.py db upgrade
+    ;;
+esac
+
 popd

api-daemon/pvcapid-manage_flask.py

@@ -0,0 +1,29 @@
+#!/usr/bin/env python3
+
+# pvcapid-manage_flask.py - PVC Database management tasks (via Flask CLI)
+# Part of the Parallel Virtual Cluster (PVC) system
+#
+#    Copyright (C) 2018-2022 Joshua M. Boniface <joshua@boniface.me>
+#
+#    This program is free software: you can redistribute it and/or modify
+#    it under the terms of the GNU General Public License as published by
+#    the Free Software Foundation, version 3.
+#
+#    This program is distributed in the hope that it will be useful,
+#    but WITHOUT ANY WARRANTY; without even the implied warranty of
+#    MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+#    GNU General Public License for more details.
+#
+#    You should have received a copy of the GNU General Public License
+#    along with this program.  If not, see <https://www.gnu.org/licenses/>.
+#
+###############################################################################
+
+from pvcapid.flaskapi import app, db
+from pvcapid.models import *  # noqa F401,F403
+
+from flask_migrate import Migrate
+
+migrate = Migrate(app, db)
+
+# Call flask --app /usr/share/pvc/pvcapid-manage_flask.py db upgrade

api-daemon/pvcapid-manage_legacy.py

@@ -1,6 +1,6 @@
 #!/usr/bin/env python3
 
-# manage.py - PVC Database management tasks
+# pvcapid-manage_legacy.py - PVC Database management tasks (Legacy)
 # Part of the Parallel Virtual Cluster (PVC) system
 #
 #    Copyright (C) 2018-2022 Joshua M. Boniface <joshua@boniface.me>
@@ -19,8 +19,7 @@
 #
 ###############################################################################
 
-from flask_migrate import Migrate, MigrateCommand
-from flask_script import Manager
+from flask_migrate import Migrate, MigrateCommand, Manager
 
 from pvcapid.flaskapi import app, db
 from pvcapid.models import *  # noqa F401,F403

api-daemon/pvcapid-worker.sh

@@ -27,12 +27,8 @@ case "$( cat /etc/debian_version )" in
     10.*)
         CELERY_ARGS="worker --app pvcapid.flaskapi.celery --concurrency 1 --loglevel INFO"
     ;;
-    11.*)
-        CELERY_ARGS="--app pvcapid.flaskapi.celery worker --concurrency 1 --loglevel INFO"
-    ;;
     *)
-        echo "Invalid Debian version found!"
-        exit 1
+        CELERY_ARGS="--app pvcapid.flaskapi.celery worker --concurrency 1 --loglevel INFO"
     ;;
 esac
 

api-daemon/pvcapid/Daemon.py

@@ -27,7 +27,7 @@ from ssl import SSLContext, TLSVersion
 from distutils.util import strtobool as dustrtobool
 
 # Daemon version
-version = "0.9.68"
+version = "0.9.73"
 
 # API version
 API_VERSION = 1.0

client-cli/pvc/cli/cli.py

@@ -569,7 +569,7 @@ def cli_node():
     short_help="Set node as primary coordinator.",
 )
 @connection_req
-@click.argument("node")
+@click.argument("node", default=DEFAULT_NODE_HOSTNAME)
 @click.option(
     "-w",
     "--wait",
@@ -619,7 +619,7 @@ NOTE: There are currently {len(tasks_retdata)} active or queued provisioner task
     short_help="Set node as secondary coordinator.",
 )
 @connection_req
-@click.argument("node")
+@click.argument("node", default=DEFAULT_NODE_HOSTNAME)
 @click.option(
     "-w",
     "--wait",
@@ -671,7 +671,7 @@ NOTE: There are currently {len(tasks_retdata)} active or queued provisioner task
     short_help="Take node out of service.",
 )
 @connection_req
-@click.argument("node")
+@click.argument("node", default=DEFAULT_NODE_HOSTNAME)
 @click.option(
     "-w",
     "--wait",
@@ -709,7 +709,7 @@ def cli_node_flush(
     short_help="Restore node to service.",
 )
 @connection_req
-@click.argument("node")
+@click.argument("node", default=DEFAULT_NODE_HOSTNAME)
 @click.option(
     "-w",
     "--wait",
@@ -747,7 +747,7 @@ def cli_node_ready(
     short_help="View node daemon logs.",
 )
 @connection_req
-@click.argument("node")
+@click.argument("node", default=DEFAULT_NODE_HOSTNAME)
 @click.option(
     "-l",
     "--lines",

client-cli/setup.py

@@ -2,7 +2,7 @@ from setuptools import setup
 
 setup(
     name="pvc",
-    version="0.9.68",
+    version="0.9.73",
     packages=["pvc.cli", "pvc.lib"],
     install_requires=[
         "Click",

debian/changelog

@@ -1,3 +1,34 @@
+pvc (0.9.73-0) unstable; urgency=high
+
+  * [Node Daemon] Fixes a bug creating monitoring instance
+
+ -- Joshua M. Boniface <joshua@boniface.me>  Sat, 02 Sep 2023 02:16:19 -0400
+
+pvc (0.9.72-0) unstable; urgency=high
+
+  * [CLI] Restores old functionality for default node value
+
+ -- Joshua M. Boniface <joshua@boniface.me>  Fri, 01 Sep 2023 16:34:45 -0400
+
+pvc (0.9.71-0) unstable; urgency=high
+
+  * [API] Adds API support for Debian Bookworm
+
+ -- Joshua M. Boniface <joshua@boniface.me>  Fri, 01 Sep 2023 00:30:42 -0400
+
+pvc (0.9.70-0) unstable; urgency=high
+
+  * [Node Daemon] Fixes several compatibility issues for Debian 12 "Bookworm"
+
+ -- Joshua M. Boniface <joshua@boniface.me>  Thu, 31 Aug 2023 14:15:54 -0400
+
+pvc (0.9.69-0) unstable; urgency=high
+
+  * [Node Daemon] Ensures that system load is always 2 decimal places on Bookworm
+  * [Node Daemon] Fixes bug blocking primary takeover at DNS Aggregator start if Patroni is down
+
+ -- Joshua M. Boniface <joshua@boniface.me>  Tue, 29 Aug 2023 22:01:22 -0400
+
 pvc (0.9.68-0) unstable; urgency=high
 
   * [CLI] Fixes another bug with network info view

debian/control

@@ -16,7 +16,7 @@ Description: Parallel Virtual Cluster node daemon (Python 3)
 
 Package: pvc-daemon-api
 Architecture: all
-Depends: systemd, pvc-daemon-common, python3-yaml, python3-flask, python3-flask-restful, python3-celery, python-celery-common, python3-distutils, redis, python3-redis, python3-lxml, python3-flask-migrate, python3-flask-script, fio
+Depends: systemd, pvc-daemon-common, python3-yaml, python3-flask, python3-flask-restful, python3-celery, python-celery-common, python3-distutils, redis, python3-redis, python3-lxml, python3-flask-migrate, fio
 Description: Parallel Virtual Cluster API daemon (Python 3)
  A KVM/Zookeeper/Ceph-based VM and private cloud manager
  .

debian/pvc-daemon-api.install

@@ -1,5 +1,5 @@
 api-daemon/pvcapid.py usr/share/pvc
-api-daemon/pvcapid-manage.py usr/share/pvc
+api-daemon/pvcapid-manage*.py usr/share/pvc
 api-daemon/pvc-api-db-upgrade usr/share/pvc
 api-daemon/pvcapid.sample.yaml etc/pvc
 api-daemon/pvcapid usr/share/pvc

node-daemon/pvcnoded/Daemon.py

@@ -49,7 +49,7 @@ import re
 import json
 
 # Daemon version
-version = "0.9.68"
+version = "0.9.73"
 
 
 ##########################################################

node-daemon/pvcnoded/dnsmasq-zookeeper-leases.py

@@ -77,7 +77,7 @@ def connect_zookeeper():
 
     with open(pvcnoded_config_file, "r") as cfgfile:
         try:
-            o_config = yaml.load(cfgfile)
+            o_config = yaml.load(cfgfile, yaml.SafeLoader)
         except Exception as e:
             print(
                 "ERROR: Failed to parse configuration file: {}".format(e),

node-daemon/pvcnoded/objects/MonitoringInstance.py

@@ -317,18 +317,20 @@ class MonitoringInstance(object):
             return
 
         # Clean up any old plugin data for which a plugin file no longer exists
-        for plugin_key in self.zkhandler.children(
+        plugins_data = self.zkhandler.children(
             ("node.monitoring.data", self.this_node.name)
-        ):
-            if plugin_key not in self.all_plugin_names:
-                self.zkhandler.delete(
-                    (
-                        "node.monitoring.data",
-                        self.this_node.name,
-                        "monitoring_plugin",
-                        plugin_key,
+        )
+        if plugins_data is not None:
+            for plugin_key in plugins_data:
+                if plugin_key not in self.all_plugin_names:
+                    self.zkhandler.delete(
+                        (
+                            "node.monitoring.data",
+                            self.this_node.name,
+                            "monitoring_plugin",
+                            plugin_key,
+                        )
                     )
-                )
 
     def run_plugin(self, plugin):
         time_start = datetime.now()

node-daemon/pvcnoded/objects/NodeInstance.py

@@ -620,9 +620,12 @@ class NodeInstance(object):
         for network in self.d_network:
             self.d_network[network].startDHCPServer()
         # 9. Start DNS aggregator; just continue if we fail
-        if not patroni_failed:
-            self.dns_aggregator.start_aggregator()
-        else:
+        try:
+            if not patroni_failed:
+                self.dns_aggregator.start_aggregator()
+            else:
+                raise
+        except Exception:
             self.logger.out(
                 "Not starting DNS aggregator due to Patroni failures", state="e"
             )

node-daemon/pvcnoded/util/keepalive.py

@@ -338,8 +338,21 @@ def collect_ceph_stats(logger, config, zkhandler, this_node, queue):
             line = re.sub(r"\x1b(\[.*?[@-~]|\].*?(\x07|\x1b\\))", "", line)
             # Split it for parsing
             line = line.split()
-            if len(line) > 1 and line[1].isdigit():
-                # This is an OSD line so parse it
+
+            # Ceph 14 format:
+            #  ['|', '0', '|', 'hv1.p.u.bonilan.net', '|', '318G', '|', '463G', '|', '213', '|', '1430k', '|', '22', '|', '124k', '|', 'exists,up', '|']
+            # Ceph 16 format:
+            #  ['0', 'hv1.t.u.bonilan.net', '2489M', '236G', '0', '0', '0', '0', 'exists,up']
+
+            # Bypass obviously invalid lines
+            if len(line) < 1:
+                continue
+            elif line[0] == "+":
+                continue
+
+            # If line begins with | and second entry is a digit (i.e. OSD ID)
+            if line[0] == "|" and line[1].isdigit():
+                # Parse the line in Ceph 14 format
                 osd_id = line[1]
                 node = line[3].split(".")[0]
                 used = line[5]
@@ -349,20 +362,39 @@ def collect_ceph_stats(logger, config, zkhandler, this_node, queue):
                 rd_ops = line[13]
                 rd_data = line[15]
                 state = line[17]
-                osd_status.update(
-                    {
-                        str(osd_id): {
-                            "node": node,
-                            "used": used,
-                            "avail": avail,
-                            "wr_ops": wr_ops,
-                            "wr_data": wr_data,
-                            "rd_ops": rd_ops,
-                            "rd_data": rd_data,
-                            "state": state,
-                        }
+            # If first entry is a digit (i.e. OSD ID)
+            elif line[0].isdigit():
+                # Parse the line in Ceph 16 format
+                osd_id = line[0]
+                node = line[1].split(".")[0]
+                used = line[2]
+                avail = line[3]
+                wr_ops = line[4]
+                wr_data = line[5]
+                rd_ops = line[6]
+                rd_data = line[7]
+                state = line[8]
+            # Otherwise, it's the header line and is ignored
+            else:
+                continue
+
+            # I don't know why 2018 me used this construct instead of a normal
+            # dictionary update, but it works so not changing it.
+            # ref: bfbe9188ce830381f3f2fa1da11f1973f08eca8c
+            osd_status.update(
+                {
+                    str(osd_id): {
+                        "node": node,
+                        "used": used,
+                        "avail": avail,
+                        "wr_ops": wr_ops,
+                        "wr_data": wr_data,
+                        "rd_ops": rd_ops,
+                        "rd_data": rd_data,
+                        "state": state,
                     }
-                )
+                }
+            )
 
         # Merge them together into a single meaningful dict
         if debug:
@@ -753,7 +785,7 @@ def node_keepalive(logger, config, zkhandler, this_node, monitoring_instance):
     this_node.memtotal = int(psutil.virtual_memory().total / 1024 / 1024)
     this_node.memused = int(psutil.virtual_memory().used / 1024 / 1024)
     this_node.memfree = int(psutil.virtual_memory().free / 1024 / 1024)
-    this_node.cpuload = os.getloadavg()[0]
+    this_node.cpuload = round(os.getloadavg()[0], 2)
 
     # Join against running threads
     if config["enable_hypervisor"]: