diff --git a/debian/control b/debian/control index cc0106f4..1f96e9f4 100644 --- a/debian/control +++ b/debian/control @@ -8,8 +8,8 @@ X-Python3-Version: >= 3.2 Package: pvc-daemon Architecture: all -Depends: python3-kazoo, python3-psutil, python3-apscheduler, python3-libvirt, python3-psycopg2, python3-dnspython, python3-yaml, ipmitool, libvirt-daemon-system, arping, vlan, bridge-utils, dnsmasq, nftables, pdns-server, pdns-backend-pgsql -Suggests: pvc-client-cli +Depends: pvc-client-common, python3-kazoo, python3-psutil, python3-apscheduler, python3-libvirt, python3-psycopg2, python3-dnspython, python3-yaml, ipmitool, libvirt-daemon-system, arping, vlan, bridge-utils, dnsmasq, nftables, pdns-server, pdns-backend-pgsql +Suggests: pvc-client-api, pvc-client-cli Description: Parallel Virtual Cluster virtualization daemon (Python 3) A KVM/Zookeeper/Ceph-based VM and private cloud manager . diff --git a/node-daemon/pvcd.sample.yaml b/node-daemon/pvcd.sample.yaml index 157cea8f..ed6dc887 100644 --- a/node-daemon/pvcd.sample.yaml +++ b/node-daemon/pvcd.sample.yaml @@ -25,8 +25,6 @@ pvc: enable_storage: True # enable_api: Enable or disable the API client, if installed, when node is Primary enable_api: True - # enable_provisioner: Enable or disable the Provisioner client, if installed, when node is Primary - enable_provisioner: True # cluster: Cluster-level configuration cluster: # coordinators: The list of cluster coordinator hostnames @@ -80,6 +78,20 @@ pvc: user: pvcdns # pass: PostgreSQL user password, randomly generated pass: pvcdns + # metadata: Metadata API subsystem + metadata: + # database: Patroni PostgreSQL database configuration + database: + # host: PostgreSQL hostname, invariably 'localhost' + host: localhost + # port: PostgreSQL port, invariably 'localhost' + port: 5432 + # name: PostgreSQL database name, invariably 'pvcprov' + name: pvcprov + # user: PostgreSQL username, invariable 'pvcprov' + user: pvcprov + # pass: PostgreSQL user password, randomly generated + pass: pvcprov # system: Local PVC instance configuration system: # intervals: Intervals for keepalives and fencing diff --git a/node-daemon/pvcd/Daemon.py b/node-daemon/pvcd/Daemon.py index f4e6397d..ec213c36 100644 --- a/node-daemon/pvcd/Daemon.py +++ b/node-daemon/pvcd/Daemon.py @@ -52,6 +52,7 @@ import pvcd.NodeInstance as NodeInstance import pvcd.VXNetworkInstance as VXNetworkInstance import pvcd.DNSAggregatorInstance as DNSAggregatorInstance import pvcd.CephInstance as CephInstance +import pvcd.MetadataAPIInstance as MetadataAPIInstance ############################################################################### # PVCD - node daemon startup program @@ -194,6 +195,11 @@ def readConfig(pvcd_config_file, myhostname): 'pdns_postgresql_dbname': o_config['pvc']['coordinator']['dns']['database']['name'], 'pdns_postgresql_user': o_config['pvc']['coordinator']['dns']['database']['user'], 'pdns_postgresql_password': o_config['pvc']['coordinator']['dns']['database']['pass'], + 'metadata_postgresql_host': o_config['pvc']['coordinator']['metadata']['database']['host'], + 'metadata_postgresql_port': o_config['pvc']['coordinator']['metadata']['database']['port'], + 'metadata_postgresql_dbname': o_config['pvc']['coordinator']['metadata']['database']['name'], + 'metadata_postgresql_user': o_config['pvc']['coordinator']['metadata']['database']['user'], + 'metadata_postgresql_password': o_config['pvc']['coordinator']['metadata']['database']['pass'], 'vni_dev': o_config['pvc']['system']['configuration']['networking']['cluster']['device'], 'vni_mtu': o_config['pvc']['system']['configuration']['networking']['cluster']['mtu'], 'vni_dev_ip': o_config['pvc']['system']['configuration']['networking']['cluster']['address'], @@ -726,13 +732,16 @@ pool_list = [] volume_list = dict() # Dict of Lists if enable_networking: - # Create an instance of the DNS Aggregator if we're a coordinator + # Create an instance of the DNS Aggregator and Metadata API if we're a coordinator if config['daemon_mode'] == 'coordinator': dns_aggregator = DNSAggregatorInstance.DNSAggregatorInstance(zk_conn, config, logger) + metadata_api = MetadataAPIInstance.MetadataAPIInstance(zk_conn, config, logger) else: dns_aggregator = None + metadata_api = None else: dns_aggregator = None + metadata_api = None # Node objects @zk_conn.ChildrenWatch('/nodes') @@ -742,7 +751,7 @@ def update_nodes(new_node_list): # Add any missing nodes to the list for node in new_node_list: if not node in node_list: - d_node[node] = NodeInstance.NodeInstance(node, myhostname, zk_conn, config, logger, d_node, d_network, d_domain, dns_aggregator) + d_node[node] = NodeInstance.NodeInstance(node, myhostname, zk_conn, config, logger, d_node, d_network, d_domain, dns_aggregator, metadata_api) # Remove any deleted nodes from the list for node in node_list: diff --git a/node-daemon/pvcd/MetadataAPIInstance.py b/node-daemon/pvcd/MetadataAPIInstance.py new file mode 100644 index 00000000..15e02c4e --- /dev/null +++ b/node-daemon/pvcd/MetadataAPIInstance.py @@ -0,0 +1,186 @@ +#!/usr/bin/env python3 + +# MetadataAPIInstance.py - Class implementing an EC2-compatible cloud-init Metadata server +# Part of the Parallel Virtual Cluster (PVC) system +# +# Copyright (C) 2018-2019 Joshua M. Boniface +# +# This program is free software: you can redistribute it and/or modify +# it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# This program is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with this program. If not, see . +# +############################################################################### + +import gevent.pywsgi +import flask +import threading +import sys +import psycopg2 +from psycopg2.extras import RealDictCursor + +# The metadata server requires client libraries +import client_lib.vm as pvc_vm +import client_lib.network as pvc_network + +class MetadataAPIInstance(object): + mdapi = flask.Flask(__name__) + + # Initialization function + def __init__(self, zk_conn, config, logger): + self.zk_conn = zk_conn + self.config = config + self.logger = logger + self.thread = None + self.md_http_server = None + + # Add flask routes inside our instance + def add_routes(self): + @self.mdapi.route('/', methods=['GET']) + def api_root(): + return flask.jsonify({"message": "PVC Provisioner Metadata API version 1"}), 209 + + @self.mdapi.route('//meta-data/', methods=['GET']) + def api_metadata_root(version): + metadata = """instance-id\nname\nprofile""" + return metadata, 200 + + @self.mdapi.route('//meta-data/instance-id', methods=['GET']) + def api_metadata_instanceid(version): + source_address = flask.request.__dict__['environ']['REMOTE_ADDR'] + vm_details = self.get_vm_details(source_address) + instance_id = vm_details['uuid'] + return instance_id, 200 + + @self.mdapi.route('//meta-data/name', methods=['GET']) + def api_metadata_hostname(version): + source_address = flask.request.__dict__['environ']['REMOTE_ADDR'] + vm_details = self.get_vm_details(source_address) + vm_name = vm_details['name'] + return vm_name, 200 + + @self.mdapi.route('//meta-data/profile', methods=['GET']) + def api_metadata_profile(version): + source_address = flask.request.__dict__['environ']['REMOTE_ADDR'] + vm_details = self.get_vm_details(source_address) + vm_profile = vm_details['profile'] + return vm_profile, 200 + + @self.mdapi.route('//user-data', methods=['GET']) + def api_userdata(version): + source_address = flask.request.__dict__['environ']['REMOTE_ADDR'] + vm_details = self.get_vm_details(source_address) + vm_profile = vm_details['profile'] + # Get the userdata + userdata = self.get_profile_userdata(vm_profile) + self.logger.out("Returning userdata for profile {}".format(vm_profile), state='i', prefix='Metadata API') + return flask.Response(userdata) + + def launch_wsgi(self): + try: + self.add_routes() + self.md_http_server = gevent.pywsgi.WSGIServer( + ('169.254.169.254', 80), + self.mdapi, + log=sys.stdout, + error_log=sys.stdout + ) + self.md_http_server.serve_forever() + except Exception as e: + self.logger.out('Error starting Metadata API: {}'.format(e), state='e') + + # WSGI start/stop + def start(self): + # Launch Metadata API + self.logger.out('Starting Metadata API at 169.254.169.254:80', state='i') + self.thread = threading.Thread(target=self.launch_wsgi) + self.thread.start() + self.logger.out('Successfully started Metadata API thread', state='o') + + def stop(self): + self.logger.out('Stopping Metadata API at 169.254.169.254:80', state='i') + if self.thread and self.md_http_server: + try: + self.md_http_server.stop() + self.md_http_server.close() + self.logger.out('Successfully stopped Metadata API', state='o') + except Exception as e: + self.logger.out('Error stopping Metadata API: {}'.format(e), state='e') + + # Helper functions + def open_database(self): + conn = psycopg2.connect( + host=self.config['metadata_postgresql_host'], + port=self.config['metadata_postgresql_port'], + dbname=self.config['metadata_postgresql_dbname'], + user=self.config['metadata_postgresql_user'], + password=self.config['metadata_postgresql_password'] + ) + cur = conn.cursor(cursor_factory=RealDictCursor) + return conn, cur + + def close_database(self, conn, cur): + cur.close() + conn.close() + + # Obtain a list of templates + def get_profile_userdata(self, vm_profile): + query = """SELECT userdata FROM profile + JOIN userdata_template ON profile.userdata_template = userdata_template.id + WHERE profile.name = %s; + """ + args = (vm_profile,) + + conn, cur = self.open_database() + cur.execute(query, args) + data_raw = cur.fetchone() + self.close_database(conn, cur) + data = data_raw['userdata'] + return data + + # VM details function + def get_vm_details(self, source_address): + # Start connection to Zookeeper + _discard, networks = pvc_network.get_list(self.zk_conn, None) + + # Figure out which server this is via the DHCP address + host_information = dict() + networks_managed = (x for x in networks if x['type'] == 'managed') + for network in networks_managed: + network_leases = pvc_network.getNetworkDHCPLeases(self.zk_conn, network['vni']) + for network_lease in network_leases: + information = pvc_network.getDHCPLeaseInformation(self.zk_conn, network['vni'], network_lease) + try: + if information['ip4_address'] == source_address: + host_information = information + except: + pass + + # Get our real information on the host; now we can start querying about it + client_hostname = host_information['hostname'] + client_macaddr = host_information['mac_address'] + client_ipaddr = host_information['ip4_address'] + + # Find the VM with that MAC address - we can't assume that the hostname is actually right + _discard, vm_list = pvc_vm.get_list(self.zk_conn, None, None, None) + vm_name = None + vm_details = dict() + for vm in vm_list: + try: + for network in vm['networks']: + if network['mac'] == client_macaddr: + vm_name = vm['name'] + vm_details = vm + except: + pass + + return vm_details + diff --git a/node-daemon/pvcd/NodeInstance.py b/node-daemon/pvcd/NodeInstance.py index 95eebee6..732f7080 100644 --- a/node-daemon/pvcd/NodeInstance.py +++ b/node-daemon/pvcd/NodeInstance.py @@ -34,7 +34,7 @@ import pvcd.common as common class NodeInstance(object): # Initialization function - def __init__(self, name, this_node, zk_conn, config, logger, d_node, d_network, d_domain, dns_aggregator): + def __init__(self, name, this_node, zk_conn, config, logger, d_node, d_network, d_domain, dns_aggregator, metadata_api): # Passed-in variables on creation self.name = name self.this_node = this_node @@ -53,6 +53,7 @@ class NodeInstance(object): self.d_network = d_network self.d_domain = d_domain self.dns_aggregator = dns_aggregator + self.metadata_api = metadata_api # Printable lists self.active_node_list = [] self.flushed_node_list = [] @@ -269,8 +270,9 @@ class NodeInstance(object): for network in self.d_network: self.d_network[network].stopDHCPServer() self.d_network[network].removeGateways() - self.removeFloatingAddresses() self.dns_aggregator.stop_aggregator() + self.metadata_api.stop() + self.removeFloatingAddresses() def become_primary(self): # Establish a lock @@ -318,6 +320,7 @@ class NodeInstance(object): # Start the DNS aggregator instance time.sleep(1) self.dns_aggregator.start_aggregator() + self.metadata_api.start() # Start the clients if self.config['enable_api']: @@ -327,6 +330,17 @@ class NodeInstance(object): common.run_os_command("systemctl start pvc-provisioner-worker.service") def createFloatingAddresses(self): + # Metadata link-local IP + self.logger.out( + 'Creating Metadata link-local IP {}/{} on interface {}'.format( + '169.254.169.254', + '32', + 'lo' + ), + state='o' + ) + common.createIPAddress('169.254.169.254', '32', 'lo') + # VNI floating IP self.logger.out( 'Creating floating management IP {}/{} on interface {}'.format( @@ -337,6 +351,7 @@ class NodeInstance(object): state='o' ) common.createIPAddress(self.vni_ipaddr, self.vni_cidrnetmask, 'brcluster') + # Upstream floating IP self.logger.out( 'Creating floating upstream IP {}/{} on interface {}'.format( @@ -349,6 +364,17 @@ class NodeInstance(object): common.createIPAddress(self.upstream_ipaddr, self.upstream_cidrnetmask, self.upstream_dev) def removeFloatingAddresses(self): + # Metadata link-local IP + self.logger.out( + 'Removing Metadata link-local IP {}/{} from interface {}'.format( + '169.254.169.254', + '32', + 'lo' + ), + state='o' + ) + common.removeIPAddress('169.254.169.254', '32', 'lo') + # VNI floating IP self.logger.out( 'Removing floating management IP {}/{} from interface {}'.format( @@ -359,6 +385,7 @@ class NodeInstance(object): state='o' ) common.removeIPAddress(self.vni_ipaddr, self.vni_cidrnetmask, 'brcluster') + # Upstream floating IP self.logger.out( 'Removing floating upstream IP {}/{} from interface {}'.format(