Bump version to 0.9.58

1. Move the test_matrix, volume name, and size to module-level variables
so they can be accessed externally if this is imported.
2. Separate the volume creation and volume cleanup into functions.
3. Separate the individual benchmark runs into a function.

This should enable easier calling of the various subcomponents
externally, e.g. for external benchmark scripts.

.version

@@ -1 +1 @@
-0.9.56
+0.9.58

CHANGELOG.md

@@ -1,5 +1,16 @@
 ## PVC Changelog
 
+###### [v0.9.58](https://github.com/parallelvirtualcluster/pvc/releases/tag/v0.9.58)
+
+  * [API] Fixes a bug where migration selector could have case-sensitive operational faults
+
+###### [v0.9.57](https://github.com/parallelvirtualcluster/pvc/releases/tag/v0.9.57)
+
+  * [CLI] Removes an invalid reference to VXLAN
+  * [CLI] Improves the handling of invalid networks in VM lists and on attach
+  * [API] Modularizes the benchmarking library so it can be used externally too
+  * [Daemon Library] Adds a module tag file so it can be used externally too
+
 ###### [v0.9.56](https://github.com/parallelvirtualcluster/pvc/releases/tag/v0.9.56)
 
   **Breaking Change**: Existing provisioner scripts are no longer valid; new example scripts are provided.

api-daemon/pvcapid/Daemon.py

@@ -27,7 +27,7 @@ from ssl import SSLContext, TLSVersion
 from distutils.util import strtobool as dustrtobool
 
 # Daemon version
-version = "0.9.56"
+version = "0.9.58"
 
 # API version
 API_VERSION = 1.0

api-daemon/pvcapid/benchmark.py

@@ -32,6 +32,74 @@ import daemon_lib.common as pvc_common
 import daemon_lib.ceph as pvc_ceph
 
 
+# We run a total of 8 tests, to give a generalized idea of performance on the cluster:
+#   1. A sequential read test of 8GB with a 4M block size
+#   2. A sequential write test of 8GB with a 4M block size
+#   3. A random read test of 8GB with a 4M block size
+#   4. A random write test of 8GB with a 4M block size
+#   5. A random read test of 8GB with a 256k block size
+#   6. A random write test of 8GB with a 256k block size
+#   7. A random read test of 8GB with a 4k block size
+#   8. A random write test of 8GB with a 4k block size
+# Taken together, these 8 results should give a very good indication of the overall storage performance
+# for a variety of workloads.
+test_matrix = {
+    "seq_read": {
+        "direction": "read",
+        "iodepth": "64",
+        "bs": "4M",
+        "rw": "read",
+    },
+    "seq_write": {
+        "direction": "write",
+        "iodepth": "64",
+        "bs": "4M",
+        "rw": "write",
+    },
+    "rand_read_4M": {
+        "direction": "read",
+        "iodepth": "64",
+        "bs": "4M",
+        "rw": "randread",
+    },
+    "rand_write_4M": {
+        "direction": "write",
+        "iodepth": "64",
+        "bs": "4M",
+        "rw": "randwrite",
+    },
+    "rand_read_4K": {
+        "direction": "read",
+        "iodepth": "64",
+        "bs": "4K",
+        "rw": "randread",
+    },
+    "rand_write_4K": {
+        "direction": "write",
+        "iodepth": "64",
+        "bs": "4K",
+        "rw": "randwrite",
+    },
+    "rand_read_4K_lowdepth": {
+        "direction": "read",
+        "iodepth": "1",
+        "bs": "4K",
+        "rw": "randread",
+    },
+    "rand_write_4K_lowdepth": {
+        "direction": "write",
+        "iodepth": "1",
+        "bs": "4K",
+        "rw": "randwrite",
+    },
+}
+
+
+# Specify the benchmark volume name and size
+benchmark_volume_name = "pvcbenchmark"
+benchmark_volume_size = "8G"
+
+
 #
 # Exceptions (used by Celery tasks)
 #
@@ -44,7 +112,7 @@ class BenchmarkError(Exception):
         self, message, job_name=None, db_conn=None, db_cur=None, zkhandler=None
     ):
         self.message = message
-        if job_name is not None:
+        if job_name is not None and db_conn is not None and db_cur is not None:
             # Clean up our dangling result
             query = "DELETE FROM storage_benchmarks WHERE job = %s;"
             args = (job_name,)
@@ -52,6 +120,7 @@ class BenchmarkError(Exception):
             db_conn.commit()
             # Close the database connections cleanly
             close_database(db_conn, db_cur)
+        if job_name is not None and zkhandler is not None:
             zkhandler.disconnect()
 
     def __str__(self):
@@ -116,6 +185,90 @@ def list_benchmarks(job=None):
         return {"message": "No benchmark found."}, 404
 
 
+def prepare_benchmark_volume(
+    pool, job_name=None, db_conn=None, db_cur=None, zkhandler=None
+):
+    # Create the RBD volume
+    retcode, retmsg = pvc_ceph.add_volume(
+        zkhandler, pool, benchmark_volume_name, benchmark_volume_size
+    )
+    if not retcode:
+        raise BenchmarkError(
+            'Failed to create volume "{}" on pool "{}": {}'.format(
+                benchmark_volume_name, pool, retmsg
+            ),
+            job_name=job_name,
+            db_conn=db_conn,
+            db_cur=db_cur,
+            zkhandler=zkhandler,
+        )
+    else:
+        print(retmsg)
+
+
+def cleanup_benchmark_volume(
+    pool, job_name=None, db_conn=None, db_cur=None, zkhandler=None
+):
+    # Remove the RBD volume
+    retcode, retmsg = pvc_ceph.remove_volume(zkhandler, pool, benchmark_volume_name)
+    if not retcode:
+        raise BenchmarkError(
+            'Failed to remove volume "{}" on pool "{}": {}'.format(
+                benchmark_volume_name, pool, retmsg
+            ),
+            job_name=job_name,
+            db_conn=db_conn,
+            db_cur=db_cur,
+            zkhandler=zkhandler,
+        )
+    else:
+        print(retmsg)
+
+
+def run_benchmark_job(
+    test, pool, job_name=None, db_conn=None, db_cur=None, zkhandler=None
+):
+    test_spec = test_matrix[test]
+    print("Running test '{}'".format(test))
+    fio_cmd = """
+            fio \
+                --name={test} \
+                --ioengine=rbd \
+                --pool={pool} \
+                --rbdname={volume} \
+                --output-format=json \
+                --direct=1 \
+                --randrepeat=1 \
+                --numjobs=1 \
+                --time_based \
+                --runtime=75 \
+                --group_reporting \
+                --iodepth={iodepth} \
+                --bs={bs} \
+                --readwrite={rw}
+        """.format(
+        test=test,
+        pool=pool,
+        volume=benchmark_volume_name,
+        iodepth=test_spec["iodepth"],
+        bs=test_spec["bs"],
+        rw=test_spec["rw"],
+    )
+
+    print("Running fio job: {}".format(" ".join(fio_cmd.split())))
+    retcode, stdout, stderr = pvc_common.run_os_command(fio_cmd)
+    if retcode:
+        raise BenchmarkError(
+            "Failed to run fio test: {}".format(stderr),
+            job_name=job_name,
+            db_conn=db_conn,
+            db_cur=db_cur,
+            zkhandler=zkhandler,
+        )
+
+    return loads(stdout)
+
+
 def run_benchmark(self, pool):
     # Runtime imports
     import time
@@ -172,20 +325,13 @@ def run_benchmark(self, pool):
     )
     time.sleep(1)
 
-    volume = "pvcbenchmark"
-
-    # Create the RBD volume
-    retcode, retmsg = pvc_ceph.add_volume(zkhandler, pool, volume, "8G")
-    if not retcode:
-        raise BenchmarkError(
-            'Failed to create volume "{}": {}'.format(volume, retmsg),
-            job_name=job_name,
-            db_conn=db_conn,
-            db_cur=db_cur,
-            zkhandler=zkhandler,
-        )
-    else:
-        print(retmsg)
+    prepare_benchmark_volume(
+        pool,
+        job_name=job_name,
+        db_conn=db_conn,
+        db_cur=db_cur,
+        zkhandler=zkhandler,
+    )
 
     # Phase 2 - benchmark run
     self.update_state(
@@ -194,99 +340,17 @@ def run_benchmark(self, pool):
     )
     time.sleep(1)
 
-    # We run a total of 8 tests, to give a generalized idea of performance on the cluster:
-    #   1. A sequential read test of 8GB with a 4M block size
-    #   2. A sequential write test of 8GB with a 4M block size
-    #   3. A random read test of 8GB with a 4M block size
-    #   4. A random write test of 8GB with a 4M block size
-    #   5. A random read test of 8GB with a 256k block size
-    #   6. A random write test of 8GB with a 256k block size
-    #   7. A random read test of 8GB with a 4k block size
-    #   8. A random write test of 8GB with a 4k block size
-    # Taken together, these 8 results should give a very good indication of the overall storage performance
-    # for a variety of workloads.
-    test_matrix = {
-        "seq_read": {"direction": "read", "iodepth": "64", "bs": "4M", "rw": "read"},
-        "seq_write": {"direction": "write", "iodepth": "64", "bs": "4M", "rw": "write"},
-        "rand_read_4M": {
-            "direction": "read",
-            "iodepth": "64",
-            "bs": "4M",
-            "rw": "randread",
-        },
-        "rand_write_4M": {
-            "direction": "write",
-            "iodepth": "64",
-            "bs": "4M",
-            "rw": "randwrite",
-        },
-        "rand_read_4K": {
-            "direction": "read",
-            "iodepth": "64",
-            "bs": "4K",
-            "rw": "randread",
-        },
-        "rand_write_4K": {
-            "direction": "write",
-            "iodepth": "64",
-            "bs": "4K",
-            "rw": "randwrite",
-        },
-        "rand_read_4K_lowdepth": {
-            "direction": "read",
-            "iodepth": "1",
-            "bs": "4K",
-            "rw": "randread",
-        },
-        "rand_write_4K_lowdepth": {
-            "direction": "write",
-            "iodepth": "1",
-            "bs": "4K",
-            "rw": "randwrite",
-        },
-    }
-
     results = dict()
     for test in test_matrix:
-        print("Running test '{}'".format(test))
-        fio_cmd = """
-            fio \
-                --name={test} \
-                --ioengine=rbd \
-                --pool={pool} \
-                --rbdname={volume} \
-                --output-format=json \
-                --direct=1 \
-                --randrepeat=1 \
-                --numjobs=1 \
-                --time_based \
-                --runtime=75 \
-                --group_reporting \
-                --iodepth={iodepth} \
-                --bs={bs} \
-                --readwrite={rw}
-        """.format(
-            test=test,
-            pool=pool,
-            volume=volume,
-            iodepth=test_matrix[test]["iodepth"],
-            bs=test_matrix[test]["bs"],
-            rw=test_matrix[test]["rw"],
+        results[test] = run_benchmark_job(
+            test,
+            pool,
+            job_name=job_name,
+            db_conn=db_conn,
+            db_cur=db_cur,
+            zkhandler=zkhandler,
         )
 
-        print("Running fio job: {}".format(" ".join(fio_cmd.split())))
-        retcode, stdout, stderr = pvc_common.run_os_command(fio_cmd)
-        if retcode:
-            raise BenchmarkError(
-                "Failed to run fio test: {}".format(stderr),
-                job_name=job_name,
-                db_conn=db_conn,
-                db_cur=db_cur,
-                zkhandler=zkhandler,
-            )
-
-        results[test] = loads(stdout)
-
     # Phase 3 - cleanup
     self.update_state(
         state="RUNNING",
@@ -294,18 +358,13 @@ def run_benchmark(self, pool):
     )
     time.sleep(1)
 
-    # Remove the RBD volume
-    retcode, retmsg = pvc_ceph.remove_volume(zkhandler, pool, volume)
-    if not retcode:
-        raise BenchmarkError(
-            'Failed to remove volume "{}": {}'.format(volume, retmsg),
-            job_name=job_name,
-            db_conn=db_conn,
-            db_cur=db_cur,
-            zkhandler=zkhandler,
-        )
-    else:
-        print(retmsg)
+    cleanup_benchmark_volume(
+        pool,
+        job_name=job_name,
+        db_conn=db_conn,
+        db_cur=db_cur,
+        zkhandler=zkhandler,
+    )
 
     print("Storing result of tests for job '{}' in database".format(job_name))
     try:

api-daemon/pvcapid/helper.py

@@ -539,9 +539,9 @@ def get_vm_meta(zkhandler, vm):
     retdata = {
         "name": vm,
         "node_limit": domain_node_limit,
-        "node_selector": domain_node_selector,
+        "node_selector": domain_node_selector.lower(),
         "node_autostart": domain_node_autostart,
-        "migration_method": domain_migrate_method,
+        "migration_method": domain_migrate_method.lower(),
     }
 
     return retdata, retcode

client-cli/pvc/cli_lib/vm.py

@@ -679,6 +679,10 @@ def vm_networks_add(
     from random import randint
     import pvc.cli_lib.network as pvc_network
 
+    network_exists, _ = pvc_network.net_info(config, network)
+    if not network_exists:
+        return False, "Network {} not found on the cluster.".format(network)
+
     status, domain_information = vm_info(config, vm)
     if not status:
         return status, domain_information
@@ -2016,7 +2020,8 @@ def format_list(config, vm_list, raw):
         tag_list = getNiceTagName(domain_information)
         if len(tag_list) < 1:
             tag_list = ["N/A"]
-        vm_net_colour = ""
+
+        net_invalid_list = []
         for net_vni in net_list:
             if (
                 net_vni not in ["cluster", "storage", "upstream"]
@@ -2024,13 +2029,33 @@ def format_list(config, vm_list, raw):
                 and not re.match(r"^hostdev:.*", net_vni)
             ):
                 if int(net_vni) not in [net["vni"] for net in cluster_net_list]:
-                    vm_net_colour = ansiprint.red()
+                    net_invalid_list.append(True)
+                else:
+                    net_invalid_list.append(False)
+            else:
+                net_invalid_list.append(False)
+
+        net_string_list = []
+        for net_idx, net_vni in enumerate(net_list):
+            if net_invalid_list[net_idx]:
+                net_string_list.append(
+                    "{}{}{}".format(
+                        ansiprint.red(),
+                        net_vni,
+                        ansiprint.end(),
+                    )
+                )
+                # Fix the length due to the extra fake characters
+                vm_nets_length -= len(net_vni)
+                vm_nets_length += len(net_string_list[net_idx])
+            else:
+                net_string_list.append(net_vni)
 
         vm_list_output.append(
             "{bold}{vm_name: <{vm_name_length}} \
 {vm_state_colour}{vm_state: <{vm_state_length}}{end_colour} \
 {vm_tags: <{vm_tags_length}} \
-{vm_net_colour}{vm_networks: <{vm_nets_length}}{end_colour} \
+{vm_networks: <{vm_nets_length}} \
 {vm_memory: <{vm_ram_length}} {vm_vcpu: <{vm_vcpu_length}} \
 {vm_node: <{vm_node_length}} \
 {vm_migrated: <{vm_migrated_length}}{end_bold}".format(
@@ -2049,8 +2074,7 @@ def format_list(config, vm_list, raw):
                 vm_name=domain_information["name"],
                 vm_state=domain_information["state"],
                 vm_tags=",".join(tag_list),
-                vm_net_colour=vm_net_colour,
-                vm_networks=",".join(net_list),
+                vm_networks=",".join(net_string_list),
                 vm_memory=domain_information["memory"],
                 vm_vcpu=domain_information["vcpu"],
                 vm_node=domain_information["node"],

client-cli/pvc/pvc.py

@@ -2404,7 +2404,7 @@ def vm_list(target_node, target_state, target_tag, limit, raw, negate):
 )
 def cli_network():
     """
-    Manage the state of a VXLAN network in the PVC cluster.
+    Manage the state of a network in the PVC cluster.
     """
     pass
 

client-cli/setup.py

@@ -2,7 +2,7 @@ from setuptools import setup
 
 setup(
     name="pvc",
-    version="0.9.56",
+    version="0.9.58",
     packages=["pvc", "pvc.cli_lib"],
     install_requires=[
         "Click",

daemon-common/common.py

@@ -633,7 +633,7 @@ def findTargetNode(zkhandler, dom_uuid):
         search_field = None
 
     # If our search field is invalid, use the default
-    if search_field is None or search_field == "None":
+    if search_field is None or search_field in ["None", "none"]:
         search_field = zkhandler.read("base.config.migration_target_selector")
 
     # Execute the search

daemon-common/vm.py

@@ -308,9 +308,9 @@ def define_vm(
             (("domain.console.log", dom_uuid), ""),
             (("domain.console.vnc", dom_uuid), ""),
             (("domain.meta.autostart", dom_uuid), node_autostart),
-            (("domain.meta.migrate_method", dom_uuid), migration_method),
+            (("domain.meta.migrate_method", dom_uuid), str(migration_method).lower()),
             (("domain.meta.node_limit", dom_uuid), formatted_node_limit),
-            (("domain.meta.node_selector", dom_uuid), node_selector),
+            (("domain.meta.node_selector", dom_uuid), str(node_selector).lower()),
             (("domain.meta.tags", dom_uuid), ""),
             (("domain.migrate.sync_lock", dom_uuid), ""),
         ]
@@ -447,7 +447,9 @@ def modify_vm_metadata(
         update_list.append((("domain.meta.node_limit", dom_uuid), node_limit))
 
     if node_selector is not None:
-        update_list.append((("domain.meta.node_selector", dom_uuid), node_selector))
+        update_list.append(
+            (("domain.meta.node_selector", dom_uuid), str(node_selector).lower())
+        )
 
     if node_autostart is not None:
         update_list.append((("domain.meta.autostart", dom_uuid), node_autostart))
@@ -456,7 +458,9 @@ def modify_vm_metadata(
         update_list.append((("domain.profile", dom_uuid), provisioner_profile))
 
     if migration_method is not None:
-        update_list.append((("domain.meta.migrate_method", dom_uuid), migration_method))
+        update_list.append(
+            (("domain.meta.migrate_method", dom_uuid), str(migration_method).lower())
+        )
 
     if len(update_list) < 1:
         return False, "ERROR: No updates to apply."

debian/changelog

@@ -1,3 +1,18 @@
+pvc (0.9.58-0) unstable; urgency=high
+
+  * [API] Fixes a bug where migration selector could have case-sensitive operational faults
+
+ -- Joshua M. Boniface <joshua@boniface.me>  Mon, 07 Nov 2022 12:27:48 -0500
+
+pvc (0.9.57-0) unstable; urgency=high
+
+  * [CLI] Removes an invalid reference to VXLAN
+  * [CLI] Improves the handling of invalid networks in VM lists and on attach
+  * [API] Modularizes the benchmarking library so it can be used externally too
+  * [Daemon Library] Adds a module tag file so it can be used externally too
+
+ -- Joshua M. Boniface <joshua@boniface.me>  Sun, 06 Nov 2022 01:39:50 -0400
+
 pvc (0.9.56-0) unstable; urgency=high
 
   * [API/Provisioner] Fundamentally revamps the provisioner script framework to provide more extensibility (BREAKING CHANGE)

node-daemon/pvcnoded/Daemon.py

@@ -48,7 +48,7 @@ import re
 import json
 
 # Daemon version
-version = "0.9.56"
+version = "0.9.58"
 
 
 ##########################################################