diff --git a/daemon-common/daemon_lib/common.py b/daemon-common/daemon_lib/common.py index 255e5174..aa2ef492 100644 --- a/daemon-common/daemon_lib/common.py +++ b/daemon-common/daemon_lib/common.py @@ -23,6 +23,8 @@ import subprocess import threading import signal +import os +import time import daemon_lib.ansiiprint as ansiiprint @@ -38,7 +40,8 @@ class OSDaemon(object): def signal(self, sent_signal): signal_map = { 'hup': signal.SIGHUP, - 'int': signal.SIGINT + 'int': signal.SIGINT, + 'term': signal.SIGTERM } self.proc.send_signal(signal_map[sent_signal]) @@ -52,7 +55,7 @@ def run_os_command(command_string, background=False, environment=None): command = command_string.split() if background: def runcmd(): - subprocess.Popen( + subprocess.run( command, env=environment, stdout=subprocess.PIPE, @@ -60,12 +63,20 @@ def run_os_command(command_string, background=False, environment=None): ) thread = threading.Thread(target=runcmd, args=()) thread.start() - return 0 + return 0, None, None else: - command_output = subprocess.Popen( + command_output = subprocess.run( command, env=environment, stdout=subprocess.PIPE, stderr=subprocess.PIPE, ) - return command_output.returncode + return command_output.returncode, command_output.stdout.decode('ascii'), command_output.stderr.decode('ascii') + +# Reload the firewall rules of the system +def reload_firewall_rules(rules_dir): + ansiiprint.echo('Updating firewall rules', '', 'o') + rules_file = '{}/base.nft'.format(rules_dir) + retcode, stdout, stderr = run_os_command('/usr/sbin/nft -f {}'.format(rules_file)) + if retcode != 0: + ansiiprint.echo('Failed to reload rules: {}'.format(stderr), '', 'e') diff --git a/router-daemon/pvcrd/Daemon.py b/router-daemon/pvcrd/Daemon.py index 7be946d5..b8bcf18d 100644 --- a/router-daemon/pvcrd/Daemon.py +++ b/router-daemon/pvcrd/Daemon.py @@ -44,8 +44,13 @@ print(ansiiprint.bold() + "pvcrd - Parallel Virtual Cluster router daemon" + ans # Set sysctl to enable routing before we do anything else common.run_os_command('sysctl net.ipv4.ip_forward=1') common.run_os_command('sysctl net.ipv4.conf.all.send_redirects=1') +common.run_os_command('sysctl net.ipv4.conf.all.rp_filter=0') +common.run_os_command('sysctl net.ipv4.conf.default.rp_filter=0') +common.run_os_command('sysctl net.ipv4.conf.all.accept_source_route=1') common.run_os_command('sysctl net.ipv4.conf.all.accept_source_route=1') common.run_os_command('sysctl net.ipv6.ip_forward=1') +common.run_os_command('sysctl net.ipv6.conf.all.rp_filter=0') +common.run_os_command('sysctl net.ipv6.conf.default.rp_filter=0') common.run_os_command('sysctl net.ipv6.conf.all.send_redirects=1') common.run_os_command('sysctl net.ipv6.conf.all.accept_source_route=1') @@ -108,6 +113,10 @@ def readConfig(pvcrd_config_file, myhostname): # Get config config = readConfig(pvcrd_config_file, myhostname) +# Add some static config elements +config['nftables_rules_dir'] = '/var/lib/pvc/nftables' +config['dnsmasq_hosts_dir'] = '/var/lib/pvc/dnsmasq' + # Set up our VNI interface vni_dev = config['vni_dev'] vni_dev_ip = config['vni_dev_ip'] @@ -231,6 +240,45 @@ s_network = dict() router_list = [] network_list = [] +# Create our config dirs +common.run_os_command( + '/bin/mkdir --parents {}/networks'.format( + config['nftables_rules_dir'] + ) +) +common.run_os_command( + '/bin/mkdir --parents {}/static'.format( + config['nftables_rules_dir'] + ) +) +common.run_os_command( + '/bin/mkdir --parents {}'.format( + config['dnsmasq_hosts_dir'] + ) +) + +# Set up the basic features of the nftables firewall +nftables_base_rules = """# Base rules +flush ruleset +add table inet filter +add chain inet filter forward {{ type filter hook forward priority 0; }} +include "{rulesdir}/static/*" +include "{rulesdir}/networks/*" +""".format( + rulesdir=config['nftables_rules_dir'] +) + +# Write the basic firewall config +print(nftables_base_rules) +nftables_base_filename = '{}/base.nft'.format(config['nftables_rules_dir']) +nftables_update_filename = '{}/update'.format(config['nftables_rules_dir']) +with open(nftables_base_filename, 'w') as nfbasefile: + nfbasefile.write(nftables_base_rules) + open(nftables_update_filename, 'a').close() + +# +# Router instances +# @zk_conn.ChildrenWatch('/routers') def updaterouters(new_router_list): global router_list @@ -246,6 +294,9 @@ def updaterouters(new_router_list): this_router = t_router[myhostname] update_zookeeper = this_router.update_zookeeper +# +# Network instances +# @zk_conn.ChildrenWatch('/networks') def updatenetworks(new_network_list): global network_list @@ -260,6 +311,7 @@ def updatenetworks(new_network_list): if this_router.network_state == 'primary': s_network[network].stopDHCPServer() s_network[network].removeGatewayAddress() + s_network[network].removeFirewall() s_network[network].removeNetwork() del(s_network[network]) network_list = new_network_list diff --git a/router-daemon/pvcrd/RouterInstance.py b/router-daemon/pvcrd/RouterInstance.py index 2c010a53..7426eb19 100644 --- a/router-daemon/pvcrd/RouterInstance.py +++ b/router-daemon/pvcrd/RouterInstance.py @@ -236,6 +236,11 @@ class RouterInstance(): ansiiprint.echo('{}Secondary router:{} {}'.format(ansiiprint.bold(), ansiiprint.end(), ' '.join(self.secondary_router_list)), '', 'c') ansiiprint.echo('{}Inactive routers:{} {}'.format(ansiiprint.bold(), ansiiprint.end(), ' '.join(self.inactive_router_list)), '', 'c') + # Reload firewall rules if needed + if os.path.isfile('{}/update'.format(self.config['nftables_rules_dir'])): + common.reload_firewall_rules(self.config['nftables_rules_dir']) + os.remove('{}/update'.format(self.config['nftables_rules_dir'])) + # # Fence thread entry function # diff --git a/router-daemon/pvcrd/VXNetworkInstance.py b/router-daemon/pvcrd/VXNetworkInstance.py index 52cdf89f..d29a8b85 100644 --- a/router-daemon/pvcrd/VXNetworkInstance.py +++ b/router-daemon/pvcrd/VXNetworkInstance.py @@ -22,6 +22,7 @@ import os import sys +from textwrap import dedent import daemon_lib.ansiiprint as ansiiprint import daemon_lib.zkhandler as zkhandler @@ -48,14 +49,13 @@ class VXNetworkInstance(): self.vxlan_nic = 'vxlan{}'.format(self.vni) self.bridge_nic = 'br{}'.format(self.vni) - self.firewall_rules = {} + self.nftables_update_filename = '{}/update'.format(config['nftables_rules_dir']) + self.nftables_netconf_filename = '{}/networks/{}.nft'.format(config['nftables_rules_dir'], self.vni) + self.firewall_rules = [] self.dhcp_server_daemon = None - - self.dnsmasq_hostsdir = '/var/lib/dnsmasq/{}'.format(self.vni) - self.dhcp_reservations = zkhandler.listchildren(self.zk_conn, '/networks/{}/dhcp_reservations'.format(self.vni)) - - self.createNetwork() + self.dnsmasq_hostsdir = '{}/{}'.format(config['dnsmasq_hosts_dir'], self.vni) + self.dhcp_reservations = None # Zookeper handlers for changed states @self.zk_conn.DataWatch('/networks/{}'.format(self.vni)) @@ -142,44 +142,72 @@ class VXNetworkInstance(): self.dhcp_end = data.decode('ascii') @self.zk_conn.ChildrenWatch('/networks/{}/dhcp_reservations'.format(self.vni)) - def watch_network_dhcp_reservations(reservations, event=''): + def watch_network_dhcp_reservations(new_reservations, event=''): if event and event.type == 'DELETED': # The key has been deleted after existing before; terminate this watcher # because this class instance is about to be reaped in Daemon.py return False - if self.dhcp_reservations != reservations: - for reservation in reservations: - if reservation not in self.dhcp_reservations: - # Add new reservation file - filename = '{}/{}'.format(self.dnsmasq_hostsdir, reservation) - ipaddr = zkhandler.readdata( - self.zk_conn, - '/networks/{}/dhcp_reservations/{}/ipaddr'.format( - self.vni, - reservation - ) - ) - entry = '{},{}'.format(reservation, ipaddr) - outfile = open(filename, 'w') - outfile.write(entry) - outfile.close() + if self.dhcp_reservations != new_reservations: + old_reservations = self.dhcp_reservations + self.dhcp_reservations = new_reservations + self.updateDHCPReservations(old_reservations, new_reservations) - for reservation in self.dhcp_reservations: - if reservation not in reservations: - filename = '{}/{}'.format(self.dnsmasq_hostsdir, reservation) - # Remove old reservation file - try: - os.remove(filename) - self.dhcp_server_daemon.signal('hup') - except: - pass + @self.zk_conn.ChildrenWatch('/networks/{}/firewall_rules'.format(self.vni)) + def watch_network_firewall_rules(new_rules, event=''): + if event and event.type == 'DELETED': + # The key has been deleted after existing before; terminate this watcher + # because this class instance is about to be reaped in Daemon.py + return False - self.dhcp_reservations = reservations + if self.firewall_rules != new_rules: + old_rules = self.firewall_rules + self.firewall_rules = new_rules + self.updateFirewallRules(old_rules, new_rules) + + self.createNetwork() + self.createFirewall() def getvni(self): return self.vni + def updateDHCPReservations(self, old_reservations_list, new_reservations_list): + for reservation in new_reservations_list: + if reservation not in old_reservations_list: + # Add new reservation file + filename = '{}/{}'.format(self.dnsmasq_hostsdir, reservation) + ipaddr = zkhandler.readdata( + self.zk_conn, + '/networks/{}/dhcp_reservations/{}/ipaddr'.format( + self.vni, + reservation + ) + ) + entry = '{},{}'.format(reservation, ipaddr) + outfile = open(filename, 'w') + outfile.write(entry) + outfile.close() + + for reservation in old_reservations_list: + if reservation not in new_reservations_list: + # Remove old reservation file + filename = '{}/{}'.format(self.dnsmasq_hostsdir, reservation) + try: + os.remove(filename) + self.dhcp_server_daemon.signal('hup') + except: + pass + + def updateFirewallRules(self, old_rules_list, new_rules_list): + for rule in new_rules_list: + if rule not in old_rules_list: + # Add new rule entry + pass + + for rule in old_rules_list: + if rule not in new_rules_list: + pass + def createNetwork(self): ansiiprint.echo( 'Creating VNI {} device on interface {}'.format( @@ -218,6 +246,23 @@ class VXNetworkInstance(): ) ) + def createFirewall(self): + nftables_network_rules = """# Rules for network {chainname} +add chain inet filter {chainname} +add rule inet filter {chainname} counter +# Jump from forward chain to this chain when matching netaddr +add rule inet filter forward ip saddr {netaddr} counter jump {chainname} +add rule inet filter forward ip daddr {netaddr} counter jump {chainname} +""".format( + netaddr=self.ip_network, + chainname=self.vxlan_nic + ) + print(nftables_network_rules) + with open(self.nftables_netconf_filename, 'w') as nfbasefile: + nfbasefile.write(dedent(nftables_network_rules)) + open(self.nftables_update_filename, 'a').close() + pass + def createGatewayAddress(self): if self.this_router.getnetworkstate() == 'primary': ansiiprint.echo( @@ -229,6 +274,12 @@ class VXNetworkInstance(): '', 'o' ) + print('ip address add {}/{} dev {}'.format( + self.ip_gateway, + self.ip_cidrnetmask, + self.bridge_nic + )) + common.run_os_command( 'ip address add {}/{} dev {}'.format( self.ip_gateway, @@ -330,6 +381,11 @@ class VXNetworkInstance(): ) ) + def removeFirewall(self): + os.remove(self.nftables_netconf_filename) + open(self.nftables_update_filename, 'a').close() + pass + def removeGatewayAddress(self): ansiiprint.echo( 'Removing gateway {} from interface {} (VNI {})'.format( @@ -358,4 +414,4 @@ class VXNetworkInstance(): '', 'o' ) - self.dhcp_server_daemon.signal('int') + self.dhcp_server_daemon.signal('term')