Add live migrate max downtime selector meta field

Adds a new flag to VM metadata to allow setting the VM live migration max downtime. This will enable very busy VMs that hang live migration to have this value changed.
2024-01-10 16:13:31 -05:00
parent 38eeb78423
commit 09269f182c
17 changed files with 283 additions and 30 deletions
--- a/client-cli/pvc/cli/cli.py
+++ b/client-cli/pvc/cli/cli.py
@ -1098,6 +1098,14 @@ def cli_vm():
    type=click.Choice(["none", "live", "shutdown"]),
    help="The preferred migration method of the VM between nodes; saved with VM.",
 )
+@click.option(
+    "-d",
+    "--max-downtime",
+    "migration_max_downtime",
+    default=300,
+    show_default=True,
+    help="The maximum time in milliseconds that a VM can be down for during a live migration; busy VMs may require a larger downtime.",
+)
@click.option(
    "-g",
    "--tag",
@ -1122,6 +1130,7 @@ def cli_vm_define(
    node_selector,
    node_autostart,
    migration_method,
+    migration_max_downtime,
    user_tags,
    protected_tags,
 ):
@ -1135,10 +1144,12 @@ def cli_vm_define(
      * "load": choose the node with the lowest current load average
      * "vms": choose the node with the least number of provisioned VMs

-    For most clusters, "mem" should be sufficient, but others may be used based on the cluster workload and available resources. The following caveats should be considered:
+    For most clusters, the migration method selector ("--method"/"-m") "mem" should be sufficient, but others may be used based on the cluster workload and available resources. The following caveats should be considered:
      * "mem" looks at the free memory of the node in general, ignoring the amount provisioned to VMs; if any VM's internal memory usage changes, this value would be affected.
      * "memprov" looks at the provisioned memory, not the allocated memory; thus, stopped or disabled VMs are counted towards a node's memory for this selector, even though their memory is not actively in use.
      * "load" looks at the system load of the node in general, ignoring load in any particular VMs; if any VM's CPU usage changes, this value would be affected. This might be preferable on clusters with some very CPU intensive VMs.
+
+    For most VMs, the 300ms default maximum downtime ("--max-downtime"/"-d") should be sufficient. However very busy VMs with a lot of memory pressure or CPU load may require a larger downtime to properly migrate. Generally, keep this at the default unless you know the VM will be extremely busy, or you find you have problems migrating it later. Reasonable values range from 100ms to 2000ms (2 seconds).
    """

    # Open the XML file
@ -1160,6 +1171,7 @@ def cli_vm_define(
        node_selector,
        node_autostart,
        migration_method,
+        migration_max_downtime,
        user_tags,
        protected_tags,
    )
@ -1205,6 +1217,13 @@ def cli_vm_define(
    type=click.Choice(["none", "live", "shutdown"]),
    help="The preferred migration method of the VM between nodes.",
 )
+@click.option(
+    "-d",
+    "--max-downtime",
+    "migration_max_downtime",
+    default=None,
+    help="The maximum time in milliseconds that a VM can be down for during a live migration; busy VMs may require a larger downtime.",
+)
@click.option(
    "-p",
    "--profile",
@ -1220,12 +1239,13 @@ def cli_vm_meta(
    node_selector,
    node_autostart,
    migration_method,
+    migration_max_downtime,
    provisioner_profile,
 ):
    """
    Modify the PVC metadata of existing virtual machine DOMAIN. At least one option to update must be specified. DOMAIN may be a UUID or name.

-    For details on the "--node-selector"/"-s" values, please see help for the command "pvc vm define".
+    For details on the available option values, please see help for the command "pvc vm define".
    """

    if (
@ -1233,6 +1253,7 @@ def cli_vm_meta(
        and node_selector is None
        and node_autostart is None
        and migration_method is None
+        and migration_max_downtime is None
        and provisioner_profile is None
    ):
        finish(False, "At least one metadata option must be specified to update.")
@ -1244,6 +1265,7 @@ def cli_vm_meta(
        node_selector,
        node_autostart,
        migration_method,
+        migration_max_downtime,
        provisioner_profile,
    )
    finish(retcode, retmsg)
@ -4456,6 +4478,13 @@ def cli_provisioner_template_system():
    default=None,  # Use cluster default
    help="The preferred migration method of the VM between nodes",
 )
+@click.option(
+    "--max-downtime",
+    "migration_max_downtime",
+    default=300,
+    show_default=True,
+    help="The maximum time in milliseconds that a VM can be down for during a live migration; busy VMs may require a larger downtime.",
+)
 def cli_provisioner_template_system_add(
    name,
    vcpus,
@ -4467,11 +4496,12 @@ def cli_provisioner_template_system_add(
    node_selector,
    node_autostart,
    migration_method,
+    migration_max_downtime,
 ):
    """
    Add a new system template NAME to the PVC cluster provisioner.

-    For details on the possible "--node-selector" values, please see help for the command "pvc vm define".
+    For details on the possible option values, please see help for the command "pvc vm define".
    """
    params = dict()
    params["name"] = name
@ -4489,6 +4519,8 @@ def cli_provisioner_template_system_add(
        params["node_autostart"] = node_autostart
    if migration_method:
        params["migration_method"] = migration_method
+    if migration_max_downtime:
+        params["migration_max_downtime"] = migration_max_downtime

    retcode, retdata = pvc.lib.provisioner.template_add(
        CLI_CONFIG, params, template_type="system"
@ -4551,6 +4583,12 @@ def cli_provisioner_template_system_add(
    default=None,  # Use cluster default
    help="The preferred migration method of the VM between nodes",
 )
+@click.option(
+    "--max-downtime",
+    "migration_max_downtime",
+    default=None,
+    help="The maximum time in milliseconds that a VM can be down for during a live migration; busy VMs may require a larger downtime.",
+)
 def cli_provisioner_template_system_modify(
    name,
    vcpus,
@ -4562,11 +4600,12 @@ def cli_provisioner_template_system_modify(
    node_selector,
    node_autostart,
    migration_method,
+    migration_max_downtime,
 ):
    """
    Add a new system template NAME to the PVC cluster provisioner.

-    For details on the possible "--node-selector" values, please see help for the command "pvc vm define".
+    For details on the possible option values, please see help for the command "pvc vm define".
    """
    params = dict()
    params["vcpus"] = vcpus
@ -4578,6 +4617,7 @@ def cli_provisioner_template_system_modify(
    params["node_selector"] = node_selector
    params["node_autostart"] = node_autostart
    params["migration_method"] = migration_method
+    params["migration_max_downtime"] = migration_max_downtime

    retcode, retdata = pvc.lib.provisioner.template_modify(
        CLI_CONFIG, params, name, template_type="system"
--- a/client-cli/pvc/lib/provisioner.py
+++ b/client-cli/pvc/lib/provisioner.py
@ -779,7 +779,8 @@ def format_list_template_system(template_data):
    template_node_limit_length = 6
    template_node_selector_length = 9
    template_node_autostart_length = 10
-    template_migration_method_length = 10
+    template_migration_method_length = 12
+    template_migration_max_downtime_length = 13

    for template in template_data:
        # template_name column
@ -826,6 +827,17 @@ def format_list_template_system(template_data):
        _template_migration_method_length = len(str(template["migration_method"])) + 1
        if _template_migration_method_length > template_migration_method_length:
            template_migration_method_length = _template_migration_method_length
+        # template_migration_max_downtime column
+        _template_migration_max_downtime_length = (
+            len(str(template["migration_max_downtime"])) + 1
+        )
+        if (
+            _template_migration_max_downtime_length
+            > template_migration_max_downtime_length
+        ):
+            template_migration_max_downtime_length = (
+                _template_migration_max_downtime_length
+            )

    # Format the string (header)
    template_list_output.append(
@ -842,7 +854,8 @@ def format_list_template_system(template_data):
            + template_node_selector_length
            + template_node_autostart_length
            + template_migration_method_length
-            + 3,
+            + template_migration_max_downtime_length
+            + 4,
            template_header="System Templates "
            + "".join(
                ["-" for _ in range(17, template_name_length + template_id_length)]
@ -874,7 +887,8 @@ def format_list_template_system(template_data):
                        + template_node_selector_length
                        + template_node_autostart_length
                        + template_migration_method_length
-                        + 2,
+                        + template_migration_max_downtime_length
+                        + 3,
                    )
                ]
            ),
@ -891,7 +905,8 @@ def format_list_template_system(template_data):
 {template_node_limit: <{template_node_limit_length}} \
 {template_node_selector: <{template_node_selector_length}} \
 {template_node_autostart: <{template_node_autostart_length}} \
-{template_migration_method: <{template_migration_method_length}}{end_bold}".format(
+{template_migration_method: <{template_migration_method_length}} \
+{template_migration_max_downtime: <{template_migration_max_downtime_length}}{end_bold}".format(
            bold=ansiprint.bold(),
            end_bold=ansiprint.end(),
            template_name_length=template_name_length,
@ -905,6 +920,7 @@ def format_list_template_system(template_data):
            template_node_selector_length=template_node_selector_length,
            template_node_autostart_length=template_node_autostart_length,
            template_migration_method_length=template_migration_method_length,
+            template_migration_max_downtime_length=template_migration_max_downtime_length,
            template_name="Name",
            template_id="ID",
            template_vcpu="vCPUs",
@ -915,7 +931,8 @@ def format_list_template_system(template_data):
            template_node_limit="Limit",
            template_node_selector="Selector",
            template_node_autostart="Autostart",
-            template_migration_method="Migration",
+            template_migration_method="Mig. Method",
+            template_migration_max_downtime="Max Downtime",
        )
    )

@ -931,7 +948,8 @@ def format_list_template_system(template_data):
 {template_node_limit: <{template_node_limit_length}} \
 {template_node_selector: <{template_node_selector_length}} \
 {template_node_autostart: <{template_node_autostart_length}} \
-{template_migration_method: <{template_migration_method_length}}{end_bold}".format(
+{template_migration_method: <{template_migration_method_length}} \
+{template_migration_max_downtime: <{template_migration_max_downtime_length}}{end_bold}".format(
                template_name_length=template_name_length,
                template_id_length=template_id_length,
                template_vcpu_length=template_vcpu_length,
@ -943,6 +961,7 @@ def format_list_template_system(template_data):
                template_node_selector_length=template_node_selector_length,
                template_node_autostart_length=template_node_autostart_length,
                template_migration_method_length=template_migration_method_length,
+                template_migration_max_downtime_length=template_migration_max_downtime_length,
                bold="",
                end_bold="",
                template_name=str(template["name"]),
@ -956,6 +975,7 @@ def format_list_template_system(template_data):
                template_node_selector=str(template["node_selector"]),
                template_node_autostart=str(template["node_autostart"]),
                template_migration_method=str(template["migration_method"]),
+                template_migration_max_downtime=f"{str(template['migration_max_downtime'])} ms",
            )
        )

--- a/client-cli/pvc/lib/vm.py
+++ b/client-cli/pvc/lib/vm.py
@ -205,6 +205,7 @@ def vm_metadata(
    node_selector,
    node_autostart,
    migration_method,
+    migration_max_downtime,
    provisioner_profile,
 ):
    """
@ -229,6 +230,9 @@ def vm_metadata(
    if migration_method is not None:
        params["migration_method"] = migration_method

+    if migration_max_downtime is not None:
+        params["migration_max_downtime"] = migration_max_downtime
+
    if provisioner_profile is not None:
        params["profile"] = provisioner_profile

@ -1637,14 +1641,14 @@ def format_info(config, domain_information, long_output):
        )
    )
    ainformation.append(
-        "{}Current Node:{}       {}".format(
+        "{}Current node:{}       {}".format(
            ansiprint.purple(), ansiprint.end(), domain_information["node"]
        )
    )
    if not domain_information["last_node"]:
        domain_information["last_node"] = "N/A"
    ainformation.append(
-        "{}Previous Node:{}      {}".format(
+        "{}Previous node:{}      {}".format(
            ansiprint.purple(), ansiprint.end(), domain_information["last_node"]
        )
    )
@ -1676,15 +1680,12 @@ def format_info(config, domain_information, long_output):
        formatted_node_autostart = "True"

    if not domain_information.get("migration_method"):
-        formatted_migration_method = "Any"
+        formatted_migration_method = "Live, Shutdown"
    else:
-        formatted_migration_method = str(domain_information["migration_method"]).title()
-
-    ainformation.append(
-        "{}Migration selector:{} {}".format(
-            ansiprint.purple(), ansiprint.end(), formatted_node_selector
+        formatted_migration_method = (
+            f"{str(domain_information['migration_method']).title()} only"
        )
-    )
+
    ainformation.append(
        "{}Node limit:{}         {}".format(
            ansiprint.purple(), ansiprint.end(), formatted_node_limit
@ -1700,10 +1701,22 @@ def format_info(config, domain_information, long_output):
        )
    )
    ainformation.append(
-        "{}Migration Method:{}   {}".format(
+        "{}Migration method:{}   {}".format(
            ansiprint.purple(), ansiprint.end(), formatted_migration_method
        )
    )
+    ainformation.append(
+        "{}Migration selector:{} {}".format(
+            ansiprint.purple(), ansiprint.end(), formatted_node_selector
+        )
+    )
+    ainformation.append(
+        "{}Max live downtime:{}  {}".format(
+            ansiprint.purple(),
+            ansiprint.end(),
+            f"{domain_information['migration_max_downtime']} ms",
+        )
+    )

    # Tag list
    tags_name_length = 5