Add cluster backup + restore functionality

Adds cluster backup (JSON dump) and restore functions for use in
disaster recovery.

Further, adds additional confirmation to the initialization (as well as
restore) endpoints to avoid accidental triggering, and also groups the
init, backup, and restore commands in the CLI into a new "task"
subsection.

client-cli/cli_lib/cluster.py

@@ -31,10 +31,55 @@ def initialize(config):
     Initialize the PVC cluster
 
     API endpoint: GET /api/v1/initialize
+    API arguments: yes-i-really-mean-it
+    API schema: {json_data_object}
+    """
+    params = {
+        'yes-i-really-mean-it': 'yes'
+    }
+    response = call_api(config, 'post', '/initialize', params=params)
+
+    if response.status_code == 200:
+        retstatus = True
+    else:
+        retstatus = False
+
+    return retstatus, response.json().get('message', '')
+
+
+def backup(config):
+    """
+    Get a JSON backup of the cluster
+
+    API endpoint: GET /api/v1/backup
     API arguments:
     API schema: {json_data_object}
     """
-    response = call_api(config, 'post', '/initialize')
+    response = call_api(config, 'get', '/backup')
+
+    if response.status_code == 200:
+        return True, response.json()
+    else:
+        return False, response.json().get('message', '')
+
+
+def restore(config, cluster_data):
+    """
+    Restore a JSON backup to the cluster
+
+    API endpoint: POST /api/v1/restore
+    API arguments: yes-i-really-mean-it
+    API schema: {json_data_object}
+    """
+    cluster_data_json = json.dumps(cluster_data)
+
+    params = {
+        'yes-i-really-mean-it': 'yes'
+    }
+    data = {
+        'cluster_data': cluster_data_json
+    }
+    response = call_api(config, 'post', '/restore', params=params, data=data)
 
     if response.status_code == 200:
         retstatus = True

client-cli/pvc.py

@@ -4078,6 +4078,71 @@ def status_cluster(oformat):
     cleanup(retcode, retdata)
 
 
+###############################################################################
+# pvc task
+###############################################################################
+@click.group(name='task', short_help='Perform PVC cluster tasks.', context_settings=CONTEXT_SETTINGS)
+def cli_task():
+    """
+    Perform administrative tasks against the PVC cluster.
+    """
+    pass
+
+
+###############################################################################
+# pvc task backup
+###############################################################################
+@click.command(name='backup', short_help='Create JSON backup of cluster.')
+@click.option(
+    '-f', '--file', 'filename',
+    default=None, type=click.File(),
+    help='Write backup data to this file.'
+)
+@cluster_req
+def task_backup(filename):
+    """
+    Create a JSON-format backup of the cluster Zookeeper database.
+    """
+
+    retcode, retdata = pvc_cluster.backup(config)
+    if filename:
+        with open(filename, 'wb') as fh:
+            fh.write(retdata)
+            retdata = 'Data written to {}'.format(filename)
+    cleanup(retcode, retdata)
+
+
+###############################################################################
+# pvc task restore
+###############################################################################
+@click.command(name='restore', short_help='Restore JSON backup to cluster.')
+@click.option(
+    '-f', '--file', 'filename',
+    required=True, default=None, type=click.File(),
+    help='Read backup data from this file.'
+)
+@click.option(
+    '-y', '--yes', 'confirm_flag',
+    is_flag=True, default=False,
+    help='Confirm the restore'
+)
+@cluster_req
+def task_restore(filename, confirm_flag):
+    """
+    Restore the JSON backup data from a file to the cluster.
+    """
+
+    if not confirm_flag:
+        try:
+            click.confirm('Replace all existing cluster data from coordinators with backup file "{}"'.format(filename.name), prompt_suffix='? ', abort=True)
+        except Exception:
+            exit(0)
+
+    cluster_data = json.loads(filename.read())
+    retcode, retmsg = pvc_cluster.restore(config, cluster_data)
+    cleanup(retcode, retmsg)
+
+
 ###############################################################################
 # pvc init
 ###############################################################################
@@ -4085,10 +4150,10 @@ def status_cluster(oformat):
 @click.option(
     '-y', '--yes', 'confirm_flag',
     is_flag=True, default=False,
-    help='Confirm the removal'
+    help='Confirm the initialization'
 )
 @cluster_req
-def init_cluster(confirm_flag):
+def task_init(confirm_flag):
     """
     Perform initialization of a new PVC cluster.
     """
@@ -4323,6 +4388,10 @@ cli_provisioner.add_command(provisioner_status)
 cli_maintenance.add_command(maintenance_on)
 cli_maintenance.add_command(maintenance_off)
 
+cli_task.add_command(task_backup)
+cli_task.add_command(task_restore)
+cli_task.add_command(task_init)
+
 cli.add_command(cli_cluster)
 cli.add_command(cli_node)
 cli.add_command(cli_vm)
@@ -4330,8 +4399,8 @@ cli.add_command(cli_network)
 cli.add_command(cli_storage)
 cli.add_command(cli_provisioner)
 cli.add_command(cli_maintenance)
+cli.add_command(cli_task)
 cli.add_command(status_cluster)
-cli.add_command(init_cluster)
 
 
 #