Fix private production pilot recovery, screen transfer and MCP dispatch

source/tests/test_node_recovery.py

@@ -0,0 +1,341 @@
+"""Socket fault injection against the real FBPAIR broker and enrolled identities."""
+import errno
+import hashlib
+import selectors
+import shlex
+import signal
+import socket
+import ssl
+import subprocess
+import sys
+import threading
+import time
+import unittest
+import urllib.error
+from unittest import mock
+
+from framebridge_expanded import broker, node
+import test_fleet_broker_node as fixtures
+
+
+class StopLoop(BaseException):
+    pass
+
+
+class NodeRecoveryTests(unittest.TestCase):
+    setUp = fixtures.BrokerNodeTests.setUp
+    tearDown = fixtures.BrokerNodeTests.tearDown
+    make_node = fixtures.BrokerNodeTests.make_node
+    pair = fixtures.BrokerNodeTests.pair
+
+    @staticmethod
+    def disconnect(handler):
+        handler.close_connection = True
+        handler.connection.shutdown(socket.SHUT_RDWR)
+        handler.connection.close()
+
+    def test_transient_poll_disconnects_backoff_cap_and_inventory_recovery(self):
+        state, ident = self.pair("recover-node")
+        node.run(state, once=True)
+        store = self.server.expanded_store
+        store.node_last_seen[ident] = time.time() - 10
+        self.assertFalse(broker.list_confirmed_nodes()[0]["ready"])
+        identity_before = (state / "node/identity.json").read_bytes()
+        original = self.server.RequestHandlerClass.do_POST
+        real_post = broker.post_json
+        attempts, waits, envelopes = [], [], []
+        recovered = []
+
+        def fault(handler):
+            if handler.path.endswith("/node/poll"):
+                attempts.append(1)
+                if len(attempts) <= 8:
+                    self.disconnect(handler)
+                    return
+            return original(handler)
+
+        def post(url, envelope, **kwargs):
+            if url.endswith("/node/poll"):
+                envelopes.append(envelope)
+            result = real_post(url, envelope, **kwargs)
+            if url.endswith("/node/poll"):
+                recovered.append(True)
+            return result
+
+        def pause(seconds):
+            if recovered:
+                raise StopLoop
+            self.assertFalse(store.node_runtime_status(ident)["ready"])
+            waits.append(seconds)
+
+        with mock.patch.object(self.server.RequestHandlerClass, "do_POST", fault), \
+                mock.patch.object(broker, "post_json", side_effect=post), \
+                mock.patch.object(node, "time", wraps=time) as clock, \
+                self.assertLogs(node._LOG, level="WARNING") as logs:
+            clock.sleep.side_effect = pause
+            with self.assertRaises(StopLoop):
+                node.run(state)
+        self.assertEqual(waits, [1, 2, 4, 8, 16, 30, 30, 30])
+        self.assertEqual(len(attempts), 9)
+        self.assertEqual(len({e["nonce"] for e in envelopes}), 9)
+        self.assertTrue(store.node_runtime_status(ident)["ready"])
+        self.assertEqual((state / "node/identity.json").read_bytes(), identity_before)
+        self.assertTrue(any("NODE_BROKER_RECOVERED" in line for line in logs.output))
+        self.assertTrue(all("signature" not in line and "payload" not in line for line in logs.output))
+
+    def test_result_disconnect_before_and_after_acceptance_never_reexecutes(self):
+        state, ident = self.pair("ack-node")
+        marker = self.root / "executed.txt"
+        real_post = broker.post_json
+        original_post = self.server.RequestHandlerClass.do_POST
+        original_send = self.server.RequestHandlerClass.send_json
+        for after_acceptance in (False, True):
+            with self.subTest(after_acceptance=after_acceptance):
+                marker.write_text("")
+                script = "printf x >> " + shlex.quote(str(marker))
+                payload = {"authorized": True, "job_id": "recovery-command", "timeout_s": 10,
+                           "script": script, "sha256": hashlib.sha256(script.encode()).hexdigest()}
+                queued, error = self.server.expanded_store.queue_admin(ident, "admin.exec", payload)
+                self.assertIsNone(error)
+                posted, dispatched, waits, dropped, delivered = [], [], [], [], []
+
+                def execute(value):
+                    dispatched.append(value["job_id"])
+                    subprocess.run(["/bin/sh", "-c", value["script"]], check=True)
+                    return {"ok": True, "exit_code": "0"}
+
+                def fault_post(handler):
+                    if not after_acceptance and handler.path.endswith("/node/result") and not dropped:
+                        dropped.append(True)
+                        self.disconnect(handler)
+                        return
+                    return original_post(handler)
+
+                def fault_send(handler, code, value):
+                    if after_acceptance and handler.path.endswith("/node/result") and not dropped:
+                        dropped.append(True)
+                        self.disconnect(handler)
+                        return
+                    return original_send(handler, code, value)
+
+                def post(url, envelope, **kwargs):
+                    if url.endswith("/node/result"):
+                        posted.append(envelope)
+                    result = real_post(url, envelope, **kwargs)
+                    if url.endswith("/node/result"):
+                        delivered.append(True)
+                    return result
+
+                def pause(seconds):
+                    if delivered:
+                        raise StopLoop
+                    waits.append(seconds)
+
+                with mock.patch.object(self.server.RequestHandlerClass, "do_POST", fault_post), \
+                        mock.patch.object(self.server.RequestHandlerClass, "send_json", fault_send), \
+                        mock.patch.object(broker, "post_json", side_effect=post), \
+                        mock.patch("framebridge_expanded.runtime.adapter_request", side_effect=execute), \
+                        mock.patch.object(node, "time", wraps=time) as clock, \
+                        self.assertLogs(node._LOG, level="WARNING"):
+                    clock.sleep.side_effect = pause
+                    with self.assertRaises(StopLoop):
+                        node.run(state)
+                self.assertEqual(marker.read_text(), "x")
+                self.assertEqual(dispatched, ["recovery-command"])
+                self.assertEqual(waits, [1])
+                self.assertEqual(len(posted), 2)
+                self.assertNotEqual(posted[0]["nonce"], posted[1]["nonce"])
+                self.assertEqual(posted[0]["payload"], posted[1]["payload"])
+                saved = self.server.expanded_store.get_result(queued["request_id"])
+                self.assertEqual(saved["state"], "complete")
+                self.assertTrue(saved["result"]["ok"])
+
+    def test_broker_restart_after_execution_is_terminal_without_replay(self):
+        state, ident = self.pair("orphan-node")
+        marker = self.root / "orphan-effect.txt"
+        script = "printf x >> " + shlex.quote(str(marker))
+        payload = {"authorized": True, "job_id": "orphan-command", "timeout_s": 10,
+                   "script": script, "sha256": hashlib.sha256(script.encode()).hexdigest()}
+        queued, error = self.server.expanded_store.queue_admin(ident, "admin.exec", payload)
+        self.assertIsNone(error)
+        real_post = broker.post_json
+        result_attempts = []
+
+        def execute(value):
+            subprocess.run(["/bin/sh", "-c", value["script"]], check=True)
+            return {"ok": True, "exit_code": "0"}
+
+        def post(url, envelope, **kwargs):
+            if url.endswith("/node/result"):
+                result_attempts.append(1)
+                self.assertEqual(marker.read_text(), "x")
+                self.server.shutdown()
+                self.server.server_close()
+                self.thread.join(2)
+                self.server = broker.make_server(port=self.port, state_dir=self.central)
+                self.thread = threading.Thread(target=self.server.serve_forever, daemon=True)
+                self.thread.start()
+                self.assertIsNone(self.server.expanded_store.get_result(queued["request_id"]))
+            return real_post(url, envelope, **kwargs)
+
+        with mock.patch.object(broker, "post_json", side_effect=post), \
+                mock.patch("framebridge_expanded.runtime.adapter_request", side_effect=execute) as adapter, \
+                mock.patch.object(node, "time", wraps=time) as clock:
+            with self.assertRaisesRegex(broker.BrokerError, "^BROKER_HTTP_REJECTED_400$"):
+                node.run(state)
+            adapter.assert_called_once()
+            clock.sleep.assert_not_called()
+        self.assertEqual(result_attempts, [1])
+        self.assertEqual(marker.read_text(), "x")
+        # A later operator restart may poll, but must never reconstruct/replay
+        # the orphaned command from its lost acknowledgement.
+        self.assertTrue(node.run(state, once=True)["idle"])
+        self.assertEqual(marker.read_text(), "x")
+
+    def test_gateway_failures_recover_without_replaying_signed_envelope(self):
+        state, _ = self.pair("gateway-node")
+        original = self.server.RequestHandlerClass.do_POST
+        codes, waits, successes = [502, 503, 504], [], []
+
+        def gateway(handler):
+            if codes:
+                handler.send_json(codes.pop(0), {"ok": False})
+                return
+            successes.append(True)
+            return original(handler)
+
+        def pause(seconds):
+            if successes:
+                raise StopLoop
+            waits.append(seconds)
+
+        with mock.patch.object(self.server.RequestHandlerClass, "do_POST", gateway), \
+                mock.patch.object(node, "time", wraps=time) as clock, \
+                self.assertLogs(node._LOG, level="WARNING"):
+            clock.sleep.side_effect = pause
+            with self.assertRaises(StopLoop):
+                node.run(state)
+        self.assertEqual(waits, [1, 2, 4])
+
+    def test_auth_and_invalid_json_fail_without_retry(self):
+        state, _ = self.pair("reject-node")
+        for code, body in [(403, b'{"ok":false}'), (500, b'{"ok":false}'),
+                           (200, b'not-json'), (200, b'{"ok":false}')]:
+            with self.subTest(code=code, body=body):
+                attempts = []
+
+                def reject(handler):
+                    attempts.append(1)
+                    handler.send_response(code)
+                    handler.send_header("Content-Length", str(len(body)))
+                    handler.end_headers()
+                    handler.wfile.write(body)
+
+                with mock.patch.object(self.server.RequestHandlerClass, "do_POST", reject), \
+                        mock.patch.object(node, "time", wraps=time) as clock:
+                    with self.assertRaises((broker.BrokerError, node.NodeError)):
+                        node.run(state)
+                    clock.sleep.assert_not_called()
+                self.assertEqual(len(attempts), 1)
+
+    def test_tls_and_unknown_transport_causes_are_terminal(self):
+        state, _ = self.pair("tls-node")
+        for cause in [urllib.error.URLError(ssl.SSLCertVerificationError("untrusted test CA")),
+                      urllib.error.URLError(socket.gaierror(socket.EAI_NONAME, "invalid host")),
+                      OSError(errno.EACCES, "denied"), ValueError("invalid response")]:
+            with self.subTest(cause=type(cause).__name__), \
+                    mock.patch("urllib.request.OpenerDirector.open", side_effect=cause) as opened, \
+                    mock.patch.object(node, "time", wraps=time) as clock:
+                with self.assertRaises(broker.BrokerError):
+                    node.run(state)
+                self.assertEqual(opened.call_count, 1)
+                clock.sleep.assert_not_called()
+
+    def test_once_is_fail_fast_and_retry_wait_propagates_cancellation(self):
+        state, _ = self.pair("cancel-node")
+        attempts = []
+
+        def fault(handler):
+            attempts.append(1)
+            self.disconnect(handler)
+
+        with mock.patch.object(self.server.RequestHandlerClass, "do_POST", fault), \
+                mock.patch.object(node, "time", wraps=time) as clock:
+            with self.assertRaises(broker.BrokerError):
+                node.run(state, once=True)
+            clock.sleep.assert_not_called()
+            clock.sleep.side_effect = KeyboardInterrupt
+            with self.assertLogs(node._LOG, level="WARNING"), self.assertRaises(KeyboardInterrupt):
+                node.run(state)
+            self.assertEqual(len(attempts), 2)
+            clock.sleep.assert_called_once_with(1)
+
+    def test_real_broker_restart_recovers_same_node_process_and_identity(self):
+        state, ident = self.pair("restart-node")
+        identity_before = (state / "node/identity.json").read_bytes()
+        code = "from framebridge_expanded.node import run; run(state_dir=" + repr(str(state)) + ")"
+        child = subprocess.Popen([sys.executable, "-u", "-c", code],
+                                 stdout=subprocess.DEVNULL, stderr=subprocess.PIPE, text=True)
+
+        def wait_ready():
+            deadline = time.monotonic() + 8
+            while time.monotonic() < deadline:
+                self.assertIsNone(child.poll(), "node exited during recovery")
+                if self.server.expanded_store.node_runtime_status(ident)["ready"]:
+                    return
+                threading.Event().wait(0.02)
+            self.fail("node heartbeat did not recover")
+
+        try:
+            wait_ready()
+            self.server.shutdown()
+            self.server.server_close()
+            self.thread.join(2)
+            with selectors.DefaultSelector() as selector:
+                selector.register(child.stderr, selectors.EVENT_READ)
+                self.assertTrue(selector.select(10), "child did not report broker interruption")
+                self.assertIn("NODE_BROKER_RETRY", child.stderr.readline())
+            self.assertIsNone(child.poll())
+            self.server = broker.make_server(port=self.port, state_dir=self.central)
+            self.assertFalse(self.server.expanded_store.node_runtime_status(ident)["ready"])
+            self.thread = threading.Thread(target=self.server.serve_forever, daemon=True)
+            self.thread.start()
+            wait_ready()
+            self.assertEqual((state / "node/identity.json").read_bytes(), identity_before)
+            self.assertEqual(broker.list_confirmed_nodes()[0]["node_id"], ident)
+        finally:
+            child.terminate()
+            try:
+                child.wait(timeout=3)
+            finally:
+                if child.poll() is None:
+                    child.kill()
+                    child.wait(timeout=3)
+                child.stderr.close()
+
+    def test_sigterm_stops_real_process_during_retry_wait(self):
+        state, _ = self.pair("stop-node")
+        self.server.shutdown()
+        self.server.server_close()
+        self.thread.join(2)
+        code = "from framebridge_expanded.node import run; run(state_dir=" + repr(str(state)) + ")"
+        child = subprocess.Popen([sys.executable, "-u", "-c", code],
+                                 stdout=subprocess.DEVNULL, stderr=subprocess.PIPE, text=True)
+        try:
+            with selectors.DefaultSelector() as selector:
+                selector.register(child.stderr, selectors.EVENT_READ)
+                self.assertTrue(selector.select(10), "child did not report retry")
+                self.assertIn("NODE_BROKER_RETRY", child.stderr.readline())
+            started = time.monotonic()
+            child.terminate()
+            self.assertEqual(child.wait(timeout=3), -signal.SIGTERM)
+            self.assertLess(time.monotonic() - started, 3)
+        finally:
+            if child.poll() is None:
+                child.kill()
+                child.wait(timeout=3)
+            child.stderr.close()
+
+
+if __name__ == "__main__":
+    unittest.main()