Bump version to 0.9.78

Converting into human results in imprecise values when specifying bytes
directly, which in turn breaks VMDK image uploads. Instead, just use the
raw bytes value when creating the volume instead of converting it back.

.version

@@ -1 +1 @@
-0.9.77
+0.9.78

CHANGELOG.md

@@ -1,5 +1,10 @@
 ## PVC Changelog
 
+###### [v0.9.78](https://github.com/parallelvirtualcluster/pvc/releases/tag/v0.9.78)
+
+  * [API, Client CLI] Fixes several bugs around image uploads; adds a new query parameter for non-raw images
+  * [API] Ensures RBD images are created with a raw bytes value to avoid rounding errors
+
 ###### [v0.9.77](https://github.com/parallelvirtualcluster/pvc/releases/tag/v0.9.77)
 
   * [Client CLI] Fixes a bug from a bad library import

api-daemon/pvcapid/Daemon.py

@@ -27,7 +27,7 @@ from ssl import SSLContext, TLSVersion
 from distutils.util import strtobool as dustrtobool
 
 # Daemon version
-version = "0.9.77"
+version = "0.9.78"
 
 # API version
 API_VERSION = 1.0

api-daemon/pvcapid/flaskapi.py

@@ -5088,7 +5088,12 @@ class API_Storage_Ceph_Volume_Element_Upload(Resource):
                 "required": True,
                 "location": ["args"],
                 "helptext": "A source image format must be specified.",
-            }
+            },
+            {
+                "name": "file_size",
+                "required": False,
+                "location": ["args"],
+            },
         ]
     )
     @Authenticator
@@ -5113,6 +5118,11 @@ class API_Storage_Ceph_Volume_Element_Upload(Resource):
               - qed
               - vdi
               - vpc
+          - in: query
+            name: file_size
+            type: integer
+            required: false
+            description: The size of the image file, if {image_format} is not "raw"
         responses:
           200:
             description: OK
@@ -5131,7 +5141,10 @@ class API_Storage_Ceph_Volume_Element_Upload(Resource):
               id: Message
         """
         return api_helper.ceph_volume_upload(
-            pool, volume, reqargs.get("image_format", None)
+            pool,
+            volume,
+            reqargs.get("image_format", None),
+            reqargs.get("file_size", None),
         )
 
 

api-daemon/pvcapid/helper.py

@@ -1584,7 +1584,7 @@ def ceph_volume_remove(zkhandler, pool, name):
 
 
 @ZKConnection(config)
-def ceph_volume_upload(zkhandler, pool, volume, img_type):
+def ceph_volume_upload(zkhandler, pool, volume, img_type, file_size=None):
     """
     Upload a raw file via HTTP post to a PVC Ceph volume
     """
@@ -1605,7 +1605,17 @@ def ceph_volume_upload(zkhandler, pool, volume, img_type):
         }
         retcode = 400
         return output, retcode
-    dev_size = retdata[0]["stats"]["size"]
+
+    try:
+        dev_size = retdata[0]["stats"]["size"]
+    except Exception:
+        output = {
+            "message": "Target volume '{}' does not exist in pool '{}'.".format(
+                volume, pool
+            )
+        }
+        retcode = 400
+        return output, retcode
 
     def cleanup_maps_and_volumes():
         # Unmap the target blockdev
@@ -1619,8 +1629,14 @@ def ceph_volume_upload(zkhandler, pool, volume, img_type):
             zkhandler, pool, "{}_tmp".format(volume)
         )
 
-    # Create a temporary block device to store non-raw images
     if img_type == "raw":
+        if file_size != dev_size:
+            output = {
+                "message": f"Image file size {file_size} does not match volume size {dev_size}"
+            }
+            retcode = 400
+            return output, retcode
+
         # Map the target blockdev
         retflag, retdata = pvc_ceph.map_volume(zkhandler, pool, volume)
         if not retflag:
@@ -1659,11 +1675,15 @@ def ceph_volume_upload(zkhandler, pool, volume, img_type):
         cleanup_maps_and_volumes()
         return output, retcode
 
-    # Write the image directly to the blockdev
     else:
+        if file_size is None:
+            output = {"message": "A file size must be specified"}
+            retcode = 400
+            return output, retcode
+
         # Create a temporary blockdev
         retflag, retdata = pvc_ceph.add_volume(
-            zkhandler, pool, "{}_tmp".format(volume), dev_size
+            zkhandler, pool, "{}_tmp".format(volume), file_size
         )
         if not retflag:
             output = {"message": retdata.replace('"', "'")}

client-cli/pvc/cli/cli.py

@@ -3598,7 +3598,7 @@ def cli_storage_volume_upload(pool, name, image_format, image_file):
     If the image format is "raw", the image is uploaded directly to the target volume without modification. Otherwise, it will be converted into raw format by "qemu-img convert" on the remote side before writing using a temporary volume. The image format must be a valid format recognized by "qemu-img", such as "vmdk" or "qcow2".
     """
 
-    if not os.path.exists(image_file):
+    if not path.exists(image_file):
         echo(CLI_CONFIG, "ERROR: File '{}' does not exist!".format(image_file))
         exit(1)
 
@@ -4910,13 +4910,13 @@ def cli_provisioner_ova_upload(name, filename, pool):
     Storage templates, provisioning scripts, and arguments for OVA-type profiles will be ignored and should not be set.
     """
 
-    if not os.path.exists(filename):
+    if not path.exists(filename):
         echo(CLI_CONFIG, "ERROR: File '{}' does not exist!".format(filename))
         exit(1)
 
     params = dict()
     params["pool"] = pool
-    params["ova_size"] = os.path.getsize(filename)
+    params["ova_size"] = path.getsize(filename)
 
     retcode, retdata = pvc.lib.provisioner.ova_upload(
         CLI_CONFIG, name, filename, params

client-cli/pvc/lib/storage.py

@@ -21,6 +21,7 @@
 
 import math
 
+from os import path
 from json import loads
 from requests_toolbelt.multipart.encoder import (
     MultipartEncoder,
@@ -1209,6 +1210,11 @@ def ceph_volume_upload(config, pool, volume, image_format, image_file):
     """
     import click
 
+    if image_format != "raw":
+        file_size = path.getsize(image_file)
+    else:
+        file_size = None
+
     bar = UploadProgressBar(
         image_file, end_message="Parsing file on remote side...", end_nl=False
     )
@@ -1220,7 +1226,7 @@ def ceph_volume_upload(config, pool, volume, image_format, image_file):
     upload_monitor = MultipartEncoderMonitor(upload_data, bar.update)
 
     headers = {"Content-Type": upload_monitor.content_type}
-    params = {"image_format": image_format}
+    params = {"image_format": image_format, "file_size": file_size}
 
     response = call_api(
         config,

client-cli/setup.py

@@ -2,7 +2,7 @@ from setuptools import setup
 
 setup(
     name="pvc",
-    version="0.9.77",
+    version="0.9.78",
     packages=["pvc.cli", "pvc.lib"],
     install_requires=[
         "Click",

daemon-common/ceph.py

@@ -763,9 +763,7 @@ def add_volume(zkhandler, pool, name, size):
 
     # 2. Create the volume
     retcode, stdout, stderr = common.run_os_command(
-        "rbd create --size {} {}/{}".format(
-            format_bytes_tohuman(size_bytes), pool, name
-        )
+        "rbd create --size {}B {}/{}".format(size_bytes, pool, name)
     )
     if retcode:
         return False, 'ERROR: Failed to create RBD volume "{}": {}'.format(name, stderr)

debian/changelog

@@ -1,3 +1,10 @@
+pvc (0.9.78-0) unstable; urgency=high
+
+  * [API, Client CLI] Fixes several bugs around image uploads; adds a new query parameter for non-raw images
+  * [API] Ensures RBD images are created with a raw bytes value to avoid rounding errors
+
+ -- Joshua M. Boniface <joshua@boniface.me>  Sat, 30 Sep 2023 12:57:55 -0400
+
 pvc (0.9.77-0) unstable; urgency=high
 
   * [Client CLI] Fixes a bug from a bad library import

gen-api-doc

@@ -14,7 +14,7 @@ sys.path.append('api-daemon')
 
 import pvcapid.flaskapi as pvc_api
 
-swagger_file = "docs/manuals/swagger.json"
+swagger_file = "swagger.json"
 swagger_data = swagger(pvc_api.app)
 swagger_data['info']['version'] = "1.0"
 swagger_data['info']['title'] = "PVC Client and Provisioner API"
@@ -22,3 +22,5 @@ swagger_data['host'] = "pvc.local:7370"
 
 with open(swagger_file, 'w') as fd:
     fd.write(json.dumps(swagger_data, sort_keys=True, indent=4))
+
+print(f"Swagger file output to {swagger_file}; add it to the PVC documentation repo.")

node-daemon/pvcnoded/Daemon.py

@@ -49,7 +49,7 @@ import re
 import json
 
 # Daemon version
-version = "0.9.77"
+version = "0.9.78"
 
 
 ##########################################################