Print fenced state in red

If the health daemon starts too soon during a node bootup, it will
generate generate tons of erroneous faults while the node starts up.
Adds a conditional wait for the current node daemon to be in "run"
state before the health daemon really starts up.

client-cli/pvc/lib/node.py

@@ -249,6 +249,8 @@ def getOutputColours(node_information):
         daemon_state_colour = ansiprint.yellow()
     elif node_information["daemon_state"] == "dead":
         daemon_state_colour = ansiprint.red() + ansiprint.bold()
+    elif node_information["daemon_state"] == "fenced":
+        daemon_state_colour = ansiprint.red()
     else:
         daemon_state_colour = ansiprint.blue()
 

daemon-common/cluster.py

@@ -241,7 +241,9 @@ def getNodeHealth(zkhandler, node_list):
                 node_health_messages.append(f"'{entry['name']}': {entry['message']}")
 
         node_health_entry = {
-            "health": int(node_health_value),
+            "health": int(node_health_value)
+            if isinstance(node_health_value, int)
+            else node_health_value,
             "messages": node_health_messages,
         }
         node_health[node] = node_health_entry

health-daemon/pvchealthd/Daemon.py

@@ -80,6 +80,11 @@ def entrypoint():
     # Connect to Zookeeper and return our handler and current schema version
     zkhandler, _ = pvchealthd.util.zookeeper.connect(logger, config)
 
+    logger.out("Waiting for node daemon to be operating", state="s")
+    while zkhandler.read(("node.state.daemon", config["node_hostname"])) != "run":
+        sleep(5)
+    logger.out("Node daemon in run state, continuing health daemon startup", state="s")
+
     # Define a cleanup function
     def cleanup(failure=False):
         nonlocal logger, zkhandler, monitoring_instance