From 4a7c6db9b20215af2d101eaa3434528fdf2a419d Mon Sep 17 00:00:00 2001 From: Joshua Boniface Date: Mon, 2 Dec 2019 20:24:38 -0500 Subject: [PATCH] Initial provisioner configuration Features all the components of creating various templates and scripts for the provisioner, as well as VM profiles combining these. --- client-provisioner/client_lib | 1 + .../examples/provisioning_script.py | 168 ++++ .../provisioner_lib/provisioner.py | 606 ++++++++++++ client-provisioner/pvc-provisioner.py | 932 ++++++++++++++++++ .../pvc-provisioner.sample.yaml | 55 ++ client-provisioner/pvc-provisioner.service | 16 + client-provisioner/schema.sql | 12 + docs/architecture/provisioner.md | 305 ++++++ 8 files changed, 2095 insertions(+) create mode 120000 client-provisioner/client_lib create mode 100644 client-provisioner/examples/provisioning_script.py create mode 100755 client-provisioner/provisioner_lib/provisioner.py create mode 100755 client-provisioner/pvc-provisioner.py create mode 100644 client-provisioner/pvc-provisioner.sample.yaml create mode 100644 client-provisioner/pvc-provisioner.service create mode 100644 client-provisioner/schema.sql create mode 100644 docs/architecture/provisioner.md diff --git a/client-provisioner/client_lib b/client-provisioner/client_lib new file mode 120000 index 00000000..37daac79 --- /dev/null +++ b/client-provisioner/client_lib @@ -0,0 +1 @@ +../client-common \ No newline at end of file diff --git a/client-provisioner/examples/provisioning_script.py b/client-provisioner/examples/provisioning_script.py new file mode 100644 index 00000000..1cd85ef2 --- /dev/null +++ b/client-provisioner/examples/provisioning_script.py @@ -0,0 +1,168 @@ +#!/usr/bin/env python3 + +# provisioing_script.py - PVC Provisioner example script +# Part of the Parallel Virtual Cluster (PVC) system +# +# Copyright (C) 2018-2019 Joshua M. Boniface +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +# +############################################################################### + +# This script provides an example of a PVC provisioner script. It will install +# a Debian system, of the release specified in the keyword argument `deb_release` +# and from the mirror specified in the keyword argument `deb_mirror`, and +# including the packages specified in the keyword argument `deb_packages` (a list +# of strings, which is then joined together as a CSV and passed to debootstrap), +# to the configured disks, configure fstab, and install GRUB. Any later config +# should be done within the VM, for instance via cloud-init. + +# This script can thus be used as an example or reference implementation of a +# PVC provisioner script and expanded upon as required. + +# This script will run under root privileges as the provisioner does. Be careful +# with that. + +import os + +# Installation function - performs a debootstrap install of a Debian system +# Note that the only arguments are keyword arguments. +def install(**kwargs): + # The provisioner has already mounted the disks on kwargs['temporary_directory']. + # by this point, so we can get right to running the debootstrap after setting + # some nicer variable names; you don't necessarily have to do this. + vm_name = kwargs['vm_name'] + vm_id = kwargs['vm_id'] + temporary_directory = kwargs['temporary_directory'] + disks = kwargs['disks'] + networks = kwargs['networks'] + # Our own required arguments. We should, though are not required to, handle + # failures of these gracefully, should administrators forget to specify them. + try: + deb_release = kwargs['deb_release'] + except: + deb_release = "stable" + try: + deb_mirror = kwargs['deb_mirror'] + except: + deb_mirror = "http://ftp.debian.org/debian" + try: + deb_packages = kwargs['deb_packages'] + except: + deb_packages = ["linux-image-amd64", "grub-pc", "cloud-init", "python3-cffi-backend"] + + # We need to know our root disk + root_disk = None + for disk in disks: + if disk['mountpoint'] == '/': + root_disk = disk + if not root_disk: + return + print(root_disk) + + # Ensure we have debootstrap intalled on the provisioner system; this is a + # good idea to include if you plan to use anything that is not part of the + # base Debian host system, just in case the provisioner host is not properly + # configured already. + os.system( + "apt-get install -y debootstrap" + ) + + # Perform a deboostrap installation + os.system( + "debootstrap --include={pkgs} {suite} {target} {mirror}".format( + suite=deb_release, + target=temporary_directory, + mirror=deb_mirror, + pkgs=','.join(deb_packages) + ) + ) + + # Bind mount the devfs + os.system( + "mount --bind /dev {}/dev".format( + temporary_directory + ) + ) + + # Create an fstab entry for each disk + fstab_file = "{}/etc/fstab".format(temporary_directory) + for disk in disks: + # We assume SSD-based/-like storage, and dislike atimes + options = "defaults,discard,noatime,nodiratime" + + # The root and var volumes have specific values + if disk['mountpoint'] == "/": + dump = 0 + cpass = 1 + elif disk['mountpoint'] == '/var': + dump = 0 + cpass = 2 + else: + dump = 0 + cpass = 0 + + # Append the fstab line + with open(fstab_file, 'a') as fh: + fh.write("/dev/{disk} {mountpoint} {filesystem} {options} {dump} {cpass}\n".format( + disk=disk['name'], + mountpoint=disk['mountpoint'], + filesystem=disk['filesystem'], + options=options, + dump=dump, + cpass=cpass + )) + + # Write the GRUB configuration + grubcfg_file = "{}/etc/default/grub".format(temporary_directory) + with open(grubcfg_file, 'w') as fh: + fh.write("""# Written by the PVC provisioner +GRUB_DEFAULT=0 +GRUB_TIMEOUT=1 +GRUB_DISTRIBUTOR="PVC Virtual Machine" +GRUB_CMDLINE_LINUX_DEFAULT="root=/dev/{root_disk} console=tty0 console=ttyS0,115200n8" +GRUB_CMDLINE_LINUX="" +GRUB_TERMINAL=console +GRUB_SERIAL_COMMAND="serial --speed=115200 --unit=0 --word=8 --parity=no --stop=1" +GRUB_DISABLE_LINUX_UUID=false +""".format(root_disk=root_disk['name'])) + + # Chroot and install GRUB so we can boot, then exit the chroot + # EXITING THE CHROOT IS VERY IMPORTANT OR THE FOLLOWING STAGES OF THE PROVISIONER + # WILL FAIL IN UNEXPECTED WAYS! Keep this in mind when using chroot in your scripts. + real_root = os.open("/", os.O_RDONLY) + os.chroot(temporary_directory) + fake_root = os.open("/", os.O_RDONLY) + os.fchdir(fake_root) + os.system( + "grub-install /dev/rbd/{}".format(root_disk['volume']) + ) + os.system( + "update-grub" + ) + # Restore our original root + os.fchdir(real_root) + os.chroot(".") + os.fchdir(real_root) + os.close(fake_root) + os.close(real_root) + + # Unmount the bound devfs + os.system( + "umount {}/dev".format( + temporary_directory + ) + ) + + # Everything else is done via cloud-init diff --git a/client-provisioner/provisioner_lib/provisioner.py b/client-provisioner/provisioner_lib/provisioner.py new file mode 100755 index 00000000..5fc90961 --- /dev/null +++ b/client-provisioner/provisioner_lib/provisioner.py @@ -0,0 +1,606 @@ +#!/usr/bin/env python3 + +# pvcapi.py - PVC HTTP API functions +# Part of the Parallel Virtual Cluster (PVC) system +# +# Copyright (C) 2018-2019 Joshua M. Boniface +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +# +############################################################################### + +import flask +import json +import psycopg2 +import psycopg2.extras +import os +import re + +import client_lib.common as pvc_common +import client_lib.vm as pvc_vm +import client_lib.network as pvc_network +import client_lib.ceph as pvc_ceph + +# +# Common functions +# + +# Database connections +def open_database(config): + conn = psycopg2.connect( + host=config['database_host'], + port=config['database_port'], + dbname=config['database_name'], + user=config['database_user'], + password=config['database_password'] + ) + cur = conn.cursor(cursor_factory=psycopg2.extras.RealDictCursor) + return conn, cur + +def close_database(conn, cur, failed=False): + if not failed: + conn.commit() + cur.close() + conn.close() + +# +# Template List functions +# +def list_template(limit, table, is_fuzzy=True): + if limit: + if is_fuzzy: + # Handle fuzzy vs. non-fuzzy limits + if not re.match('\^.*', limit): + limit = '%' + limit + else: + limit = limit[1:] + if not re.match('.*\$', limit): + limit = limit + '%' + else: + limit = limit[:-1] + + args = (limit, ) + query = "SELECT * FROM {} WHERE name LIKE %s;".format(table) + else: + args = () + query = "SELECT * FROM {};".format(table) + + conn, cur = open_database(config) + cur.execute(query, args) + data = cur.fetchall() + + if table == 'network_template': + for template_id, template_data in enumerate(data): + # Fetch list of VNIs from network table + query = "SELECT vni FROM network WHERE network_template = %s;" + args = (template_data['id'],) + cur.execute(query, args) + vnis = cur.fetchall() + data[template_id]['networks'] = vnis + + if table == 'storage_template': + for template_id, template_data in enumerate(data): + # Fetch list of VNIs from network table + query = "SELECT * FROM storage WHERE storage_template = %s;" + args = (template_data['id'],) + cur.execute(query, args) + disks = cur.fetchall() + data[template_id]['disks'] = disks + + close_database(conn, cur) + return data + +def list_template_system(limit, is_fuzzy=True): + """ + Obtain a list of system templates. + """ + data = list_template(limit, 'system_template', is_fuzzy) + return data + +def list_template_network(limit, is_fuzzy=True): + """ + Obtain a list of network templates. + """ + data = list_template(limit, 'network_template', is_fuzzy) + return data + +def list_template_network_vnis(name): + """ + Obtain a list of network template VNIs. + """ + data = list_template(name, 'network_template', is_fuzzy=False)[0] + networks = data['networks'] + return networks + +def list_template_storage(limit, is_fuzzy=True): + """ + Obtain a list of storage templates. + """ + data = list_template(limit, 'storage_template', is_fuzzy) + return data + +def list_template_storage_disks(name): + """ + Obtain a list of storage template disks. + """ + data = list_template(name, 'storage_template', is_fuzzy=False)[0] + disks = data['disks'] + return disks + +def template_list(limit): + system_templates = list_template_system(limit) + network_templates = list_template_network(limit) + storage_templates = list_template_storage(limit) + + return { "system_templates": system_templates, "network_templates": network_templates, "storage_templates": storage_templates } + +# +# Template Create functions +# +def create_template_system(name, vcpu_count, vram_mb, serial=False, vnc=False, vnc_bind=None): + if list_template_system(name, is_fuzzy=False): + retmsg = { "message": "The system template {} already exists".format(name) } + retcode = 400 + return flask.jsonify(retmsg), retcode + + query = "INSERT INTO system_template (name, vcpu_count, vram_mb, serial, vnc, vnc_bind) VALUES (%s, %s, %s, %s, %s, %s);" + args = (name, vcpu_count, vram_mb, serial, vnc, vnc_bind) + + conn, cur = open_database(config) + try: + cur.execute(query, args) + retmsg = { "name": name } + retcode = 200 + except psycopg2.IntegrityError as e: + retmsg = { "message": "Failed to create entry {}".format(name), "error": e } + retcode = 400 + close_database(conn, cur) + return flask.jsonify(retmsg), retcode + +def create_template_network(name, mac_template=None): + if list_template_network(name, is_fuzzy=False): + retmsg = { "message": "The network template {} already exists".format(name) } + retcode = 400 + return flask.jsonify(retmsg), retcode + + conn, cur = open_database(config) + try: + query = "INSERT INTO network_template (name, mac_template) VALUES (%s, %s);" + args = (name, mac_template) + cur.execute(query, args) + retmsg = { "name": name } + retcode = 200 + except psycopg2.IntegrityError as e: + retmsg = { "message": "Failed to create entry {}".format(name), "error": e } + retcode = 400 + close_database(conn, cur) + return flask.jsonify(retmsg), retcode + +def create_template_network_element(name, network): + if not list_template_network(name, is_fuzzy=False): + retmsg = { "message": "The network template {} does not exist".format(name) } + retcode = 400 + return flask.jsonify(retmsg), retcode + + networks = list_template_network_vnis(name) + found_vni = False + for network in networks: + if network['vni'] == vni: + found_vni = True + if found_vni: + retmsg = { "message": "The VNI {} in network template {} already exists".format(vni, name) } + retcode = 400 + return flask.jsonify(retmsg), retcode + + conn, cur = open_database(config) + try: + query = "SELECT id FROM network_template WHERE name = %s;" + args = (name,) + cur.execute(query, args) + template_id = cur.fetchone()['id'] + query = "INSERT INTO network (network_template, vni) VALUES (%s, %s);" + args = (template_id, network) + cur.execute(query, args) + retmsg = { "name": name, "vni": network } + retcode = 200 + except psycopg2.IntegrityError as e: + retmsg = { "message": "Failed to create entry {}".format(network), "error": e } + retcode = 400 + close_database(conn, cur) + return flask.jsonify(retmsg), retcode + +def create_template_storage(name): + if list_template_storage(name, is_fuzzy=False): + retmsg = { "message": "The storage template {} already exists".format(name) } + retcode = 400 + return flask.jsonify(retmsg), retcode + + conn, cur = open_database(config) + try: + query = "INSERT INTO storage_template (name) VALUES (%s);" + args = (name,) + cur.execute(query, args) + retmsg = { "name": name } + retcode = 200 + except psycopg2.IntegrityError as e: + retmsg = { "message": "Failed to create entry {}".format(name), "error": e } + retcode = 400 + close_database(conn, cur) + return flask.jsonify(retmsg), retcode + +def create_template_storage_element(name, disk_id, disk_size_gb, mountpoint=None, filesystem=None): + if not list_template_storage(name, is_fuzzy=False): + retmsg = { "message": "The storage template {} does not exist".format(name) } + retcode = 400 + return flask.jsonify(retmsg), retcode + + disks = list_template_storage_disks(name) + found_disk = False + for disk in disks: + if disk['disk_id'] == disk_id: + found_disk = True + if found_disk: + retmsg = { "message": "The disk {} in storage template {} already exists".format(disk_id, name) } + retcode = 400 + return flask.jsonify(retmsg), retcode + + if mountpoint and not filesystem: + retmsg = { "message": "A filesystem must be specified along with a mountpoint." } + retcode = 400 + return flask.jsonify(retmsg), retcode + + conn, cur = open_database(config) + try: + query = "SELECT id FROM storage_template WHERE name = %s;" + args = (name,) + cur.execute(query, args) + template_id = cur.fetchone()['id'] + query = "INSERT INTO storage (storage_template, disk_id, disk_size_gb, mountpoint, filesystem) VALUES (%s, %s, %s, %s, %s);" + args = (template_id, disk_id, disk_size_gb, mountpoint, filesystem) + cur.execute(query, args) + retmsg = { "name": name, "disk_id": disk_id } + retcode = 200 + except psycopg2.IntegrityError as e: + retmsg = { "message": "Failed to create entry {}".format(disk_id), "error": e } + retcode = 400 + close_database(conn, cur) + return flask.jsonify(retmsg), retcode + +def delete_template_system(name): + if not list_template_system(name, is_fuzzy=False): + retmsg = { "message": "The system template {} does not exist".format(name) } + retcode = 400 + return flask.jsonify(retmsg), retcode + + conn, cur = open_database(config) + try: + query = "DELETE FROM system_template WHERE name = %s;" + args = (name,) + cur.execute(query, args) + retmsg = { "name": name } + retcode = 200 + except psycopg2.IntegrityError as e: + retmsg = { "message": "Failed to delete entry {}".format(name), "error": e } + retcode = 400 + close_database(conn, cur) + return flask.jsonify(retmsg), retcode + +def delete_template_network(name): + if not list_template_network(name, is_fuzzy=False): + retmsg = { "message": "The network template {} does not exist".format(name) } + retcode = 400 + return flask.jsonify(retmsg), retcode + + conn, cur = open_database(config) + try: + query = "SELECT id FROM network_template WHERE name = %s;" + args = (name,) + cur.execute(query, args) + template_id = cur.fetchone()['id'] + query = "DELETE FROM network WHERE network_template = %s;" + args = (template_id,) + cur.execute(query, args) + query = "DELETE FROM network_template WHERE name = %s;" + args = (name,) + cur.execute(query, args) + retmsg = { "name": name } + retcode = 200 + except psycopg2.IntegrityError as e: + retmsg = { "message": "Failed to delete entry {}".format(name), "error": e } + retcode = 400 + close_database(conn, cur) + return flask.jsonify(retmsg), retcode + +def delete_template_network_element(name, vni): + if not list_template_network(name, is_fuzzy=False): + retmsg = { "message": "The network template {} does not exist".format(name) } + retcode = 400 + return flask.jsonify(retmsg), retcode + + networks = list_template_network_vnis(name) + found_vni = False + for network in networks: + if network['vni'] == vni: + found_vni = True + if not found_vni: + retmsg = { "message": "The VNI {} in network template {} does not exist".format(vni, name) } + retcode = 400 + return flask.jsonify(retmsg), retcode + + conn, cur = open_database(config) + try: + query = "SELECT id FROM network_template WHERE name = %s;" + args = (name,) + cur.execute(query, args) + template_id = cur.fetchone()['id'] + query = "DELETE FROM network WHERE network_template = %s and vni = %s;" + args = (template_id, vni) + cur.execute(query, args) + retmsg = { "name": name, "vni": vni } + retcode = 200 + except psycopg2.IntegrityError as e: + retmsg = { "message": "Failed to delete entry {}".format(name), "error": e } + retcode = 400 + close_database(conn, cur) + return flask.jsonify(retmsg), retcode + +def delete_template_storage(name): + if not list_template_storage(name, is_fuzzy=False): + retmsg = { "message": "The storage template {} does not exist".format(name) } + retcode = 400 + return flask.jsonify(retmsg), retcode + + conn, cur = open_database(config) + try: + query = "SELECT id FROM storage_template WHERE name = %s;" + args = (name,) + cur.execute(query, args) + template_id = cur.fetchone()['id'] + query = "DELETE FROM storage WHERE storage_template = %s;" + args = (template_id,) + cur.execute(query, args) + query = "DELETE FROM storage_template WHERE name = %s;" + args = (name,) + cur.execute(query, args) + retmsg = { "name": name } + retcode = 200 + except psycopg2.IntegrityError as e: + retmsg = { "message": "Failed to delete entry {}".format(name), "error": e } + retcode = 400 + close_database(conn, cur) + return flask.jsonify(retmsg), retcode + +def delete_template_storage_element(name, disk_id): + if not list_template_storage(name, is_fuzzy=False): + retmsg = { "message": "The storage template {} does not exist".format(name) } + retcode = 400 + return flask.jsonify(retmsg), retcode + + disks = list_template_storage_disks(name) + found_disk = False + for disk in disks: + if disk['disk_id'] == disk_id: + found_disk = True + if not found_disk: + retmsg = { "message": "The disk {} in storage template {} does not exist".format(disk_id, name) } + retcode = 400 + return flask.jsonify(retmsg), retcode + + conn, cur = open_database(config) + try: + query = "SELECT id FROM storage_template WHERE name = %s;" + args = (name,) + cur.execute(query, args) + template_id = cur.fetchone()['id'] + query = "DELETE FROM storage WHERE storage_template = %s and disk_id = %s;" + args = (template_id, disk_id) + cur.execute(query, args) + retmsg = { "name": name, "disk_id": disk_id } + retcode = 200 + except psycopg2.IntegrityError as e: + retmsg = { "message": "Failed to delete entry {}".format(name), "error": e } + retcode = 400 + close_database(conn, cur) + return flask.jsonify(retmsg), retcode + +# +# Script functions +# +def list_script(limit, is_fuzzy=True): + if limit: + if is_fuzzy: + # Handle fuzzy vs. non-fuzzy limits + if not re.match('\^.*', limit): + limit = '%' + limit + else: + limit = limit[1:] + if not re.match('.*\$', limit): + limit = limit + '%' + else: + limit = limit[:-1] + + query = "SELECT * FROM {} WHERE name LIKE %s;".format('script') + args = (limit, ) + else: + query = "SELECT * FROM {};".format('script') + args = () + + conn, cur = open_database(config) + cur.execute(query, args) + data = cur.fetchall() + close_database(conn, cur) + return data + +def create_script(name, script): + if list_script(name, is_fuzzy=False): + retmsg = { "message": "The script {} already exists".format(name) } + retcode = 400 + return flask.jsonify(retmsg), retcode + + conn, cur = open_database(config) + try: + query = "INSERT INTO script (name, script) VALUES (%s, %s);" + args = (name, script) + cur.execute(query, args) + retmsg = { "name": name } + retcode = 200 + except psycopg2.IntegrityError as e: + retmsg = { "message": "Failed to create entry {}".format(name), "error": e } + retcode = 400 + close_database(conn, cur) + return flask.jsonify(retmsg), retcode + +def delete_script(name): + if not list_script(name, is_fuzzy=False): + retmsg = { "message": "The script {} does not exist".format(name) } + retcode = 400 + return flask.jsonify(retmsg), retcode + + conn, cur = open_database(config) + try: + query = "DELETE FROM script WHERE name = %s;" + args = (name,) + cur.execute(query, args) + retmsg = { "name": name } + retcode = 200 + except psycopg2.IntegrityError as e: + retmsg = { "message": "Failed to delete entry {}".format(name), "error": e } + retcode = 400 + close_database(conn, cur) + return flask.jsonify(retmsg), retcode + +# +# Profile functions +# +def list_profile(limit, is_fuzzy=True): + if limit: + if is_fuzzy: + # Handle fuzzy vs. non-fuzzy limits + if not re.match('\^.*', limit): + limit = '%' + limit + else: + limit = limit[1:] + if not re.match('.*\$', limit): + limit = limit + '%' + else: + limit = limit[:-1] + + query = "SELECT * FROM {} WHERE name LIKE %s;".format('profile') + args = (limit, ) + else: + query = "SELECT * FROM {};".format('profile') + args = () + + conn, cur = open_database(config) + cur.execute(query, args) + orig_data = cur.fetchall() + data = list() + for profile in orig_data: + profile_data = dict() + profile_data['name'] = profile['name'] + for etype in 'system_template', 'network_template', 'storage_template', 'script': + query = 'SELECT name from {} WHERE id = %s'.format(etype) + args = (profile[etype],) + cur.execute(query, args) + name = cur.fetchone()['name'] + profile_data[etype] = name + data.append(profile_data) + close_database(conn, cur) + return data + +def create_profile(name, system_template, network_template, storage_template, script): + if list_profile(name, is_fuzzy=False): + retmsg = { "message": "The profile {} already exists".format(name) } + retcode = 400 + return flask.jsonify(retmsg), retcode + + system_templates = list_template_system(None) + system_template_id = None + for template in system_templates: + if template['name'] == system_template: + system_template_id = template['id'] + if not system_template_id: + retmsg = { "message": "The system template {} for profile {} does not exist".format(system_template, name) } + retcode = 400 + return flask.jsonify(retmsg), retcode + + network_templates = list_template_network(None) + network_template_id = None + for template in network_templates: + if template['name'] == network_template: + network_template_id = template['id'] + if not network_template_id: + retmsg = { "message": "The network template {} for profile {} does not exist".format(network_template, name) } + retcode = 400 + return flask.jsonify(retmsg), retcode + + storage_templates = list_template_storage(None) + storage_template_id = None + for template in storage_templates: + if template['name'] == storage_template: + storage_template_id = template['id'] + if not storage_template_id: + retmsg = { "message": "The storage template {} for profile {} does not exist".format(storage_template, name) } + retcode = 400 + return flask.jsonify(retmsg), retcode + + scripts = list_script(None) + script_id = None + for scr in scripts: + if scr['name'] == script: + script_id = scr['id'] + if not script_id: + retmsg = { "message": "The script {} for profile {} does not exist".format(script, name) } + retcode = 400 + return flask.jsonify(retmsg), retcode + + conn, cur = open_database(config) + try: + query = "INSERT INTO profile (name, system_template, network_template, storage_template, script) VALUES (%s, %s, %s, %s, %s);" + args = (name, system_template_id, network_template_id, storage_template_id, script_id) + cur.execute(query, args) + retmsg = { "name": name } + retcode = 200 + except psycopg2.IntegrityError as e: + retmsg = { "message": "Failed to create entry {}".format(name), "error": e } + retcode = 400 + close_database(conn, cur) + return flask.jsonify(retmsg), retcode + +def delete_profile(name): + if not list_profile(name, is_fuzzy=False): + retmsg = { "message": "The profile {} does not exist".format(name) } + retcode = 400 + return flask.jsonify(retmsg), retcode + + conn, cur = open_database(config) + try: + query = "DELETE FROM profile WHERE name = %s;" + args = (name,) + cur.execute(query, args) + retmsg = { "name": name } + retcode = 200 + except psycopg2.IntegrityError as e: + retmsg = { "message": "Failed to delete entry {}".format(name), "error": e } + retcode = 400 + close_database(conn, cur) + return flask.jsonify(retmsg), retcode + +# +# Job functions +# +def create_vm(vm_name, profile_name): + pass + + diff --git a/client-provisioner/pvc-provisioner.py b/client-provisioner/pvc-provisioner.py new file mode 100755 index 00000000..086e6543 --- /dev/null +++ b/client-provisioner/pvc-provisioner.py @@ -0,0 +1,932 @@ +#!/usr/bin/env python3 + +# pvc-provisioner.py - PVC Provisioner API interface +# Part of the Parallel Virtual Cluster (PVC) system +# +# Copyright (C) 2018-2019 Joshua M. Boniface +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +# +############################################################################### + +import flask +import json +import yaml +import os +import uu + +import gevent.pywsgi + +import provisioner_lib.provisioner as pvcprovisioner + +# Parse the configuration file +try: + pvc_config_file = os.environ['PVC_CONFIG_FILE'] +except: + print('Error: The "PVC_CONFIG_FILE" environment variable must be set before starting pvc-provisioner.') + exit(1) + +print('Starting PVC Provisioner daemon') + +# Read in the config +try: + with open(pvc_config_file, 'r') as cfgfile: + o_config = yaml.load(cfgfile) +except Exception as e: + print('Failed to parse configuration file: {}'.format(e)) + exit(1) + +try: + # Create the config object + config = { + 'debug': o_config['pvc']['debug'], + 'listen_address': o_config['pvc']['provisioner']['listen_address'], + 'listen_port': int(o_config['pvc']['provisioner']['listen_port']), + 'auth_enabled': o_config['pvc']['provisioner']['authentication']['enabled'], + 'auth_secret_key': o_config['pvc']['provisioner']['authentication']['secret_key'], + 'auth_tokens': o_config['pvc']['provisioner']['authentication']['tokens'], + 'ssl_enabled': o_config['pvc']['provisioner']['ssl']['enabled'], + 'ssl_key_file': o_config['pvc']['provisioner']['ssl']['key_file'], + 'ssl_cert_file': o_config['pvc']['provisioner']['ssl']['cert_file'], + 'database_host': o_config['pvc']['provisioner']['database']['host'], + 'database_port': int(o_config['pvc']['provisioner']['database']['port']), + 'database_name': o_config['pvc']['provisioner']['database']['name'], + 'database_user': o_config['pvc']['provisioner']['database']['user'], + 'database_password': o_config['pvc']['provisioner']['database']['pass'] + } + + # Set the config object in the pvcapi namespace + pvcprovisioner.config = config +except Exception as e: + print('{}'.format(e)) + exit(1) + +# Try to connect to the database or fail +try: + print('Verifying connectivity to database') + conn, cur = pvcprovisioner.open_database(config) + pvcprovisioner.close_database(conn, cur) +except Exception as e: + print('{}'.format(e)) + exit(1) + +api = flask.Flask(__name__) + +if config['debug']: + api.config['DEBUG'] = True + +if config['auth_enabled']: + api.config["SECRET_KEY"] = config['auth_secret_key'] + +# Authentication decorator function +def authenticator(function): + def authenticate(*args, **kwargs): + # No authentication required + if not config['auth_enabled']: + return function(*args, **kwargs) + + # Session-based authentication + if 'token' in flask.session: + return function(*args, **kwargs) + + # Key header-based authentication + if 'X-Api-Key' in flask.request.headers: + if any(token for token in secret_tokens if flask.request.headers.get('X-Api-Key') == token): + return function(*args, **kwargs) + else: + return "X-Api-Key Authentication failed\n", 401 + + # All authentications failed + return "X-Api-Key Authentication required\n", 401 + + authenticate.__name__ = function.__name__ + return authenticate + +@api.route('/api/v1', methods=['GET']) +def api_root(): + return flask.jsonify({"message": "PVC Provisioner API version 1"}), 209 + +@api.route('/api/v1/auth/login', methods=['GET', 'POST']) +def api_auth_login(): + # Just return a 200 if auth is disabled + if not config['auth_enabled']: + return flask.jsonify({"message": "Authentication is disabled."}), 200 + + if flask.request.method == 'GET': + return ''' +
+

+ Enter your authentication token: + + +

+
+ ''' + + if flask.request.method == 'POST': + if any(token for token in config['auth_tokens'] if flask.request.values['token'] in token['token']): + flask.session['token'] = flask.request.form['token'] + return flask.redirect(flask.url_for('api_root')) + else: + return flask.jsonify({"message": "Authentication failed"}), 401 + +@api.route('/api/v1/auth/logout', methods=['GET', 'POST']) +def api_auth_logout(): + # Just return a 200 if auth is disabled + if not config['auth_enabled']: + return flask.jsonify({"message": "Authentication is disabled."}), 200 + + # remove the username from the session if it's there + flask.session.pop('token', None) + return flask.redirect(flask.url_for('api_root')) + +# +# Template endpoints +# +@api.route('/api/v1/template', methods=['GET']) +@authenticator +def api_template_root(): + """ + /template - Manage provisioning templates for VM creation. + + GET: List all templates in the provisioning system. + ?limit: Specify a limit to queries. Fuzzy by default; use ^ and $ to force exact matches. + """ + # Get name limit + if 'limit' in flask.request.values: + limit = flask.request.values['limit'] + else: + limit = None + + return flask.jsonify(pvcprovisioner.template_list(limit)), 200 + +@api.route('/api/v1/template/system', methods=['GET', 'POST']) +@authenticator +def api_template_system_root(): + """ + /template/system - Manage system provisioning templates for VM creation. + + GET: List all system templates in the provisioning system. + ?limit: Specify a limit to queries. Fuzzy by default; use ^ and $ to force exact matches. + * type: text + * optional: true + * requires: N/A + + POST: Add new system template. + ?name: The name of the template. + * type: text + * optional: false + * requires: N/A + ?vcpus: The number of VCPUs. + * type: integer + * optional: false + * requires: N/A + ?vram: The amount of RAM in MB. + * type: integer, Megabytes (MB) + * optional: false + * requires: N/A + ?serial: Enable serial console. + * type: boolean + * optional: false + * requires: N/A + ?vnc: True/False, enable VNC console. + * type: boolean + * optional: false + * requires: N/A + ?vnc_bind: Address to bind VNC to. + * default: '127.0.0.1' + * type: IP Address (or '0.0.0.0' wildcard) + * optional: true + * requires: vnc=True + """ + if flask.request.method == 'GET': + # Get name limit + if 'limit' in flask.request.values: + limit = flask.request.values['limit'] + else: + limit = None + + return flask.jsonify(pvcprovisioner.list_template_system(limit)), 200 + + if flask.request.method == 'POST': + # Get name data + if 'name' in flask.request.values: + name = flask.request.values['name'] + else: + return flask.jsonify({"message": "A name must be specified."}), 400 + + # Get vcpus data + if 'vcpus' in flask.request.values: + try: + vcpu_count = int(flask.request.values['vcpus']) + except: + return flask.jsonify({"message": "A vcpus value must be an integer."}), 400 + else: + return flask.jsonify({"message": "A vcpus value must be specified."}), 400 + + # Get vram data + if 'vram' in flask.request.values: + try: + vram_mb = int(flask.request.values['vram']) + except: + return flask.jsonify({"message": "A vram integer value in Megabytes must be specified."}), 400 + else: + return flask.jsonify({"message": "A vram integer value in Megabytes must be specified."}), 400 + + # Get serial configuration + if 'serial' in flask.request.values and flask.request.values['serial']: + serial = True + else: + serial = False + + # Get VNC configuration + if 'vnc' in flask.request.values and flask.request.values['vnc']: + vnc = True + + if 'vnc_bind' in flask.request.values: + vnc_bind = flask.request.values['vnc_bind_address'] + else: + vnc_bind = None + else: + vnc = False + vnc_bind = None + + return pvcprovisioner.create_template_system(name, vcpu_count, vram_mb, serial, vnc, vnc_bind) + +@api.route('/api/v1/template/system/