342 lines
16 KiB
Python
342 lines
16 KiB
Python
"""Socket fault injection against the real FBPAIR broker and enrolled identities."""
|
|
import errno
|
|
import hashlib
|
|
import selectors
|
|
import shlex
|
|
import signal
|
|
import socket
|
|
import ssl
|
|
import subprocess
|
|
import sys
|
|
import threading
|
|
import time
|
|
import unittest
|
|
import urllib.error
|
|
from unittest import mock
|
|
|
|
from framebridge_expanded import broker, node
|
|
import test_fleet_broker_node as fixtures
|
|
|
|
|
|
class StopLoop(BaseException):
|
|
pass
|
|
|
|
|
|
class NodeRecoveryTests(unittest.TestCase):
|
|
setUp = fixtures.BrokerNodeTests.setUp
|
|
tearDown = fixtures.BrokerNodeTests.tearDown
|
|
make_node = fixtures.BrokerNodeTests.make_node
|
|
pair = fixtures.BrokerNodeTests.pair
|
|
|
|
@staticmethod
|
|
def disconnect(handler):
|
|
handler.close_connection = True
|
|
handler.connection.shutdown(socket.SHUT_RDWR)
|
|
handler.connection.close()
|
|
|
|
def test_transient_poll_disconnects_backoff_cap_and_inventory_recovery(self):
|
|
state, ident = self.pair("recover-node")
|
|
node.run(state, once=True)
|
|
store = self.server.expanded_store
|
|
store.node_last_seen[ident] = time.time() - 10
|
|
self.assertFalse(broker.list_confirmed_nodes()[0]["ready"])
|
|
identity_before = (state / "node/identity.json").read_bytes()
|
|
original = self.server.RequestHandlerClass.do_POST
|
|
real_post = broker.post_json
|
|
attempts, waits, envelopes = [], [], []
|
|
recovered = []
|
|
|
|
def fault(handler):
|
|
if handler.path.endswith("/node/poll"):
|
|
attempts.append(1)
|
|
if len(attempts) <= 8:
|
|
self.disconnect(handler)
|
|
return
|
|
return original(handler)
|
|
|
|
def post(url, envelope, **kwargs):
|
|
if url.endswith("/node/poll"):
|
|
envelopes.append(envelope)
|
|
result = real_post(url, envelope, **kwargs)
|
|
if url.endswith("/node/poll"):
|
|
recovered.append(True)
|
|
return result
|
|
|
|
def pause(seconds):
|
|
if recovered:
|
|
raise StopLoop
|
|
self.assertFalse(store.node_runtime_status(ident)["ready"])
|
|
waits.append(seconds)
|
|
|
|
with mock.patch.object(self.server.RequestHandlerClass, "do_POST", fault), \
|
|
mock.patch.object(broker, "post_json", side_effect=post), \
|
|
mock.patch.object(node, "time", wraps=time) as clock, \
|
|
self.assertLogs(node._LOG, level="WARNING") as logs:
|
|
clock.sleep.side_effect = pause
|
|
with self.assertRaises(StopLoop):
|
|
node.run(state)
|
|
self.assertEqual(waits, [1, 2, 4, 8, 16, 30, 30, 30])
|
|
self.assertEqual(len(attempts), 9)
|
|
self.assertEqual(len({e["nonce"] for e in envelopes}), 9)
|
|
self.assertTrue(store.node_runtime_status(ident)["ready"])
|
|
self.assertEqual((state / "node/identity.json").read_bytes(), identity_before)
|
|
self.assertTrue(any("NODE_BROKER_RECOVERED" in line for line in logs.output))
|
|
self.assertTrue(all("signature" not in line and "payload" not in line for line in logs.output))
|
|
|
|
def test_result_disconnect_before_and_after_acceptance_never_reexecutes(self):
|
|
state, ident = self.pair("ack-node")
|
|
marker = self.root / "executed.txt"
|
|
real_post = broker.post_json
|
|
original_post = self.server.RequestHandlerClass.do_POST
|
|
original_send = self.server.RequestHandlerClass.send_json
|
|
for after_acceptance in (False, True):
|
|
with self.subTest(after_acceptance=after_acceptance):
|
|
marker.write_text("")
|
|
script = "printf x >> " + shlex.quote(str(marker))
|
|
payload = {"authorized": True, "job_id": "recovery-command", "timeout_s": 10,
|
|
"script": script, "sha256": hashlib.sha256(script.encode()).hexdigest()}
|
|
queued, error = self.server.expanded_store.queue_admin(ident, "admin.exec", payload)
|
|
self.assertIsNone(error)
|
|
posted, dispatched, waits, dropped, delivered = [], [], [], [], []
|
|
|
|
def execute(value):
|
|
dispatched.append(value["job_id"])
|
|
subprocess.run(["/bin/sh", "-c", value["script"]], check=True)
|
|
return {"ok": True, "exit_code": "0"}
|
|
|
|
def fault_post(handler):
|
|
if not after_acceptance and handler.path.endswith("/node/result") and not dropped:
|
|
dropped.append(True)
|
|
self.disconnect(handler)
|
|
return
|
|
return original_post(handler)
|
|
|
|
def fault_send(handler, code, value):
|
|
if after_acceptance and handler.path.endswith("/node/result") and not dropped:
|
|
dropped.append(True)
|
|
self.disconnect(handler)
|
|
return
|
|
return original_send(handler, code, value)
|
|
|
|
def post(url, envelope, **kwargs):
|
|
if url.endswith("/node/result"):
|
|
posted.append(envelope)
|
|
result = real_post(url, envelope, **kwargs)
|
|
if url.endswith("/node/result"):
|
|
delivered.append(True)
|
|
return result
|
|
|
|
def pause(seconds):
|
|
if delivered:
|
|
raise StopLoop
|
|
waits.append(seconds)
|
|
|
|
with mock.patch.object(self.server.RequestHandlerClass, "do_POST", fault_post), \
|
|
mock.patch.object(self.server.RequestHandlerClass, "send_json", fault_send), \
|
|
mock.patch.object(broker, "post_json", side_effect=post), \
|
|
mock.patch("framebridge_expanded.runtime.adapter_request", side_effect=execute), \
|
|
mock.patch.object(node, "time", wraps=time) as clock, \
|
|
self.assertLogs(node._LOG, level="WARNING"):
|
|
clock.sleep.side_effect = pause
|
|
with self.assertRaises(StopLoop):
|
|
node.run(state)
|
|
self.assertEqual(marker.read_text(), "x")
|
|
self.assertEqual(dispatched, ["recovery-command"])
|
|
self.assertEqual(waits, [1])
|
|
self.assertEqual(len(posted), 2)
|
|
self.assertNotEqual(posted[0]["nonce"], posted[1]["nonce"])
|
|
self.assertEqual(posted[0]["payload"], posted[1]["payload"])
|
|
saved = self.server.expanded_store.get_result(queued["request_id"])
|
|
self.assertEqual(saved["state"], "complete")
|
|
self.assertTrue(saved["result"]["ok"])
|
|
|
|
def test_broker_restart_after_execution_is_terminal_without_replay(self):
|
|
state, ident = self.pair("orphan-node")
|
|
marker = self.root / "orphan-effect.txt"
|
|
script = "printf x >> " + shlex.quote(str(marker))
|
|
payload = {"authorized": True, "job_id": "orphan-command", "timeout_s": 10,
|
|
"script": script, "sha256": hashlib.sha256(script.encode()).hexdigest()}
|
|
queued, error = self.server.expanded_store.queue_admin(ident, "admin.exec", payload)
|
|
self.assertIsNone(error)
|
|
real_post = broker.post_json
|
|
result_attempts = []
|
|
|
|
def execute(value):
|
|
subprocess.run(["/bin/sh", "-c", value["script"]], check=True)
|
|
return {"ok": True, "exit_code": "0"}
|
|
|
|
def post(url, envelope, **kwargs):
|
|
if url.endswith("/node/result"):
|
|
result_attempts.append(1)
|
|
self.assertEqual(marker.read_text(), "x")
|
|
self.server.shutdown()
|
|
self.server.server_close()
|
|
self.thread.join(2)
|
|
self.server = broker.make_server(port=self.port, state_dir=self.central)
|
|
self.thread = threading.Thread(target=self.server.serve_forever, daemon=True)
|
|
self.thread.start()
|
|
self.assertIsNone(self.server.expanded_store.get_result(queued["request_id"]))
|
|
return real_post(url, envelope, **kwargs)
|
|
|
|
with mock.patch.object(broker, "post_json", side_effect=post), \
|
|
mock.patch("framebridge_expanded.runtime.adapter_request", side_effect=execute) as adapter, \
|
|
mock.patch.object(node, "time", wraps=time) as clock:
|
|
with self.assertRaisesRegex(broker.BrokerError, "^BROKER_HTTP_REJECTED_400$"):
|
|
node.run(state)
|
|
adapter.assert_called_once()
|
|
clock.sleep.assert_not_called()
|
|
self.assertEqual(result_attempts, [1])
|
|
self.assertEqual(marker.read_text(), "x")
|
|
# A later operator restart may poll, but must never reconstruct/replay
|
|
# the orphaned command from its lost acknowledgement.
|
|
self.assertTrue(node.run(state, once=True)["idle"])
|
|
self.assertEqual(marker.read_text(), "x")
|
|
|
|
def test_gateway_failures_recover_without_replaying_signed_envelope(self):
|
|
state, _ = self.pair("gateway-node")
|
|
original = self.server.RequestHandlerClass.do_POST
|
|
codes, waits, successes = [502, 503, 504], [], []
|
|
|
|
def gateway(handler):
|
|
if codes:
|
|
handler.send_json(codes.pop(0), {"ok": False})
|
|
return
|
|
successes.append(True)
|
|
return original(handler)
|
|
|
|
def pause(seconds):
|
|
if successes:
|
|
raise StopLoop
|
|
waits.append(seconds)
|
|
|
|
with mock.patch.object(self.server.RequestHandlerClass, "do_POST", gateway), \
|
|
mock.patch.object(node, "time", wraps=time) as clock, \
|
|
self.assertLogs(node._LOG, level="WARNING"):
|
|
clock.sleep.side_effect = pause
|
|
with self.assertRaises(StopLoop):
|
|
node.run(state)
|
|
self.assertEqual(waits, [1, 2, 4])
|
|
|
|
def test_auth_and_invalid_json_fail_without_retry(self):
|
|
state, _ = self.pair("reject-node")
|
|
for code, body in [(403, b'{"ok":false}'), (500, b'{"ok":false}'),
|
|
(200, b'not-json'), (200, b'{"ok":false}')]:
|
|
with self.subTest(code=code, body=body):
|
|
attempts = []
|
|
|
|
def reject(handler):
|
|
attempts.append(1)
|
|
handler.send_response(code)
|
|
handler.send_header("Content-Length", str(len(body)))
|
|
handler.end_headers()
|
|
handler.wfile.write(body)
|
|
|
|
with mock.patch.object(self.server.RequestHandlerClass, "do_POST", reject), \
|
|
mock.patch.object(node, "time", wraps=time) as clock:
|
|
with self.assertRaises((broker.BrokerError, node.NodeError)):
|
|
node.run(state)
|
|
clock.sleep.assert_not_called()
|
|
self.assertEqual(len(attempts), 1)
|
|
|
|
def test_tls_and_unknown_transport_causes_are_terminal(self):
|
|
state, _ = self.pair("tls-node")
|
|
for cause in [urllib.error.URLError(ssl.SSLCertVerificationError("untrusted test CA")),
|
|
urllib.error.URLError(socket.gaierror(socket.EAI_NONAME, "invalid host")),
|
|
OSError(errno.EACCES, "denied"), ValueError("invalid response")]:
|
|
with self.subTest(cause=type(cause).__name__), \
|
|
mock.patch("urllib.request.OpenerDirector.open", side_effect=cause) as opened, \
|
|
mock.patch.object(node, "time", wraps=time) as clock:
|
|
with self.assertRaises(broker.BrokerError):
|
|
node.run(state)
|
|
self.assertEqual(opened.call_count, 1)
|
|
clock.sleep.assert_not_called()
|
|
|
|
def test_once_is_fail_fast_and_retry_wait_propagates_cancellation(self):
|
|
state, _ = self.pair("cancel-node")
|
|
attempts = []
|
|
|
|
def fault(handler):
|
|
attempts.append(1)
|
|
self.disconnect(handler)
|
|
|
|
with mock.patch.object(self.server.RequestHandlerClass, "do_POST", fault), \
|
|
mock.patch.object(node, "time", wraps=time) as clock:
|
|
with self.assertRaises(broker.BrokerError):
|
|
node.run(state, once=True)
|
|
clock.sleep.assert_not_called()
|
|
clock.sleep.side_effect = KeyboardInterrupt
|
|
with self.assertLogs(node._LOG, level="WARNING"), self.assertRaises(KeyboardInterrupt):
|
|
node.run(state)
|
|
self.assertEqual(len(attempts), 2)
|
|
clock.sleep.assert_called_once_with(1)
|
|
|
|
def test_real_broker_restart_recovers_same_node_process_and_identity(self):
|
|
state, ident = self.pair("restart-node")
|
|
identity_before = (state / "node/identity.json").read_bytes()
|
|
code = "from framebridge_expanded.node import run; run(state_dir=" + repr(str(state)) + ")"
|
|
child = subprocess.Popen([sys.executable, "-u", "-c", code],
|
|
stdout=subprocess.DEVNULL, stderr=subprocess.PIPE, text=True)
|
|
|
|
def wait_ready():
|
|
deadline = time.monotonic() + 8
|
|
while time.monotonic() < deadline:
|
|
self.assertIsNone(child.poll(), "node exited during recovery")
|
|
if self.server.expanded_store.node_runtime_status(ident)["ready"]:
|
|
return
|
|
threading.Event().wait(0.02)
|
|
self.fail("node heartbeat did not recover")
|
|
|
|
try:
|
|
wait_ready()
|
|
self.server.shutdown()
|
|
self.server.server_close()
|
|
self.thread.join(2)
|
|
with selectors.DefaultSelector() as selector:
|
|
selector.register(child.stderr, selectors.EVENT_READ)
|
|
self.assertTrue(selector.select(10), "child did not report broker interruption")
|
|
self.assertIn("NODE_BROKER_RETRY", child.stderr.readline())
|
|
self.assertIsNone(child.poll())
|
|
self.server = broker.make_server(port=self.port, state_dir=self.central)
|
|
self.assertFalse(self.server.expanded_store.node_runtime_status(ident)["ready"])
|
|
self.thread = threading.Thread(target=self.server.serve_forever, daemon=True)
|
|
self.thread.start()
|
|
wait_ready()
|
|
self.assertEqual((state / "node/identity.json").read_bytes(), identity_before)
|
|
self.assertEqual(broker.list_confirmed_nodes()[0]["node_id"], ident)
|
|
finally:
|
|
child.terminate()
|
|
try:
|
|
child.wait(timeout=3)
|
|
finally:
|
|
if child.poll() is None:
|
|
child.kill()
|
|
child.wait(timeout=3)
|
|
child.stderr.close()
|
|
|
|
def test_sigterm_stops_real_process_during_retry_wait(self):
|
|
state, _ = self.pair("stop-node")
|
|
self.server.shutdown()
|
|
self.server.server_close()
|
|
self.thread.join(2)
|
|
code = "from framebridge_expanded.node import run; run(state_dir=" + repr(str(state)) + ")"
|
|
child = subprocess.Popen([sys.executable, "-u", "-c", code],
|
|
stdout=subprocess.DEVNULL, stderr=subprocess.PIPE, text=True)
|
|
try:
|
|
with selectors.DefaultSelector() as selector:
|
|
selector.register(child.stderr, selectors.EVENT_READ)
|
|
self.assertTrue(selector.select(10), "child did not report retry")
|
|
self.assertIn("NODE_BROKER_RETRY", child.stderr.readline())
|
|
started = time.monotonic()
|
|
child.terminate()
|
|
self.assertEqual(child.wait(timeout=3), -signal.SIGTERM)
|
|
self.assertLess(time.monotonic() - started, 3)
|
|
finally:
|
|
if child.poll() is None:
|
|
child.kill()
|
|
child.wait(timeout=3)
|
|
child.stderr.close()
|
|
|
|
|
|
if __name__ == "__main__":
|
|
unittest.main()
|