feat: suspend and resume an instance without losing its data
An instance you are not using had to be deleted, and deleting it destroys its database. Now it can be put to sleep instead. sudo supaspin suspend N # stops all twelve pods, keeps everything else sudo supaspin resume N # starts them again with the same data Suspending gives the server back, per instance: the 1.875 GiB it had reserved and the 1.5-2.5 GiB it was actually using, its twelve pod slots out of the 110 a node allows, and its two server ports (Postgres 544N, Studio 300N). It keeps: the database and the uploads (the disk images stay mounted), the .env file with its passwords and keys, the HTTPS certificate, which goes on renewing itself while the instance sleeps, and the hostname, which answers "service unavailable" until the instance is resumed. Disk space is not given back - that is the data being kept. How: every service takes its pod count from a `replicaCount` setting, so suspend writes zero into all twelve and re-applies the chart, resume writes one. This goes through the instance's stored settings rather than scaling the running pods by hand, so it survives anything that re-applies the chart later. `supaspin upgrade` now understands a sleeping instance: it stores the new memory limits for it and leaves it asleep instead of waiting for pods that by design do not exist. `supaspin list` reports each instance as running, suspended or partly suspended, and prints no tunnel commands for a sleeping one, because nothing is listening on its ports. `suspend` and `resume` take one instance number and have no "all" form: stopping every instance is not something to type by accident, and resuming them all at once would start 36 pods in one go. Reading the stored settings and handing them back to Helm is now shared by upgrade, suspend and resume. Verified on instance 2 (supa2.didgibot.com): suspended to zero pods, memory reserved on the node fell from 25766 MiB to 23846 MiB (exactly the 1920 MiB it had), ports 3002 and 5442 freed, then resumed to 12/12 pods with 0 restarts, its 55 tables and 11 MB of data intact and the gateway answering again. Tests: 130 pass (17 new). Version 1.2.1 -> 1.3.0. Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
This commit was merged in pull request #7.
This commit is contained in:
Igor
@@ -1105,3 +1105,173 @@ class HostPortRollout(unittest.TestCase):
|
||||
self.assertNotIn("strategy", d["spec"])
|
||||
self.assertEqual(d["spec"]["template"]["spec"]["containers"][0]["ports"][0]["hostPort"],
|
||||
sp.pg_port(1))
|
||||
|
||||
|
||||
class SuspendResume(unittest.TestCase):
|
||||
"""Suspending an instance stops its pods and keeps everything else: the disks, the passwords,
|
||||
the certificate and the hostname. Resuming starts it again with the same data."""
|
||||
|
||||
RUNNING = {
|
||||
"deployment": {
|
||||
"analytics": {"enabled": True,
|
||||
"resources": {"limits": {"memory": "1536Mi"},
|
||||
"requests": {"memory": "128Mi"}}},
|
||||
"db": {"resources": {"limits": {"memory": "2048Mi"},
|
||||
"requests": {"memory": "512Mi"}}},
|
||||
},
|
||||
"secret": {"jwt": {"secret": "keep-me"}, "db": {"password": "keep-me-too"}},
|
||||
"persistence": {"db": {"enabled": True, "size": "2048Mi"}},
|
||||
}
|
||||
|
||||
def setUp(self):
|
||||
self.calls = []
|
||||
self.written = {}
|
||||
self.log = []
|
||||
self.waited = []
|
||||
|
||||
@contextlib.contextmanager
|
||||
def fake_helm(self, stored):
|
||||
P = collections.namedtuple("P", "returncode stdout stderr")
|
||||
|
||||
def helm(*args, check=True):
|
||||
self.calls.append(list(args))
|
||||
if args[0] == "get":
|
||||
return P(0, yaml.safe_dump(stored), "")
|
||||
if args[0] == "upgrade":
|
||||
with open(args[args.index("-f") + 1]) as f:
|
||||
self.written = yaml.safe_load(f)
|
||||
return P(0, "", "")
|
||||
|
||||
with mock.patch.object(sp, "helm", helm), \
|
||||
mock.patch.object(sp, "wait_ready",
|
||||
lambda n, log, **kw: self.waited.append(("ready", n))), \
|
||||
mock.patch.object(sp, "wait_gone",
|
||||
lambda n, log, **kw: self.waited.append(("gone", n))), \
|
||||
mock.patch.object(sp, "STATE_DIR", tempfile.mkdtemp()):
|
||||
yield
|
||||
|
||||
def suspended(self):
|
||||
return sp.set_replicas(yaml.safe_load(yaml.safe_dump(self.RUNNING)), 0)
|
||||
|
||||
# ---- the setting itself ----
|
||||
|
||||
def test_set_replicas_covers_every_service(self):
|
||||
v = sp.set_replicas({}, 0)
|
||||
self.assertEqual(sorted(v["deployment"]), sorted(sp.SERVICES))
|
||||
self.assertTrue(all(d["replicaCount"] == 0 for d in v["deployment"].values()))
|
||||
self.assertTrue(sp.is_suspended(v))
|
||||
|
||||
def test_a_service_missing_the_setting_counts_as_running(self):
|
||||
self.assertFalse(sp.is_suspended(self.RUNNING))
|
||||
self.assertFalse(sp.is_suspended({"deployment": {}}))
|
||||
|
||||
def test_one_service_left_running_is_not_suspended(self):
|
||||
v = sp.set_replicas({}, 0)
|
||||
v["deployment"]["kong"]["replicaCount"] = 1
|
||||
self.assertFalse(sp.is_suspended(v))
|
||||
|
||||
# ---- suspend ----
|
||||
|
||||
def test_suspend_stops_every_service_and_keeps_the_rest(self):
|
||||
with self.fake_helm(self.RUNNING):
|
||||
self.assertTrue(sp.suspend_instance(CFG, 1, self.log.append))
|
||||
for svc in sp.SERVICES:
|
||||
self.assertEqual(self.written["deployment"][svc]["replicaCount"], 0)
|
||||
self.assertEqual(self.written["secret"]["jwt"]["secret"], "keep-me")
|
||||
self.assertEqual(self.written["secret"]["db"]["password"], "keep-me-too")
|
||||
self.assertEqual(self.written["persistence"]["db"]["size"], "2048Mi")
|
||||
# the memory caps are kept as they were: they apply again on resume
|
||||
self.assertEqual(self.written["deployment"]["analytics"]["resources"]["limits"]["memory"],
|
||||
"1536Mi")
|
||||
self.assertEqual(self.waited, [("gone", 1)])
|
||||
|
||||
def test_suspend_waits_for_the_pods_to_be_gone_not_for_them_to_be_ready(self):
|
||||
with self.fake_helm(self.RUNNING):
|
||||
sp.suspend_instance(CFG, 2, self.log.append)
|
||||
self.assertEqual([w[0] for w in self.waited], ["gone"])
|
||||
|
||||
def test_suspending_a_suspended_instance_changes_nothing(self):
|
||||
with self.fake_helm(self.suspended()):
|
||||
self.assertFalse(sp.suspend_instance(CFG, 1, self.log.append))
|
||||
self.assertEqual([c[0] for c in self.calls], ["get"])
|
||||
self.assertEqual(self.waited, [])
|
||||
|
||||
def test_the_values_file_with_the_secrets_is_deleted_again(self):
|
||||
with self.fake_helm(self.RUNNING):
|
||||
sp.suspend_instance(CFG, 1, self.log.append)
|
||||
path = [c[c.index("-f") + 1] for c in self.calls if c[0] == "upgrade"][0]
|
||||
self.assertFalse(os.path.exists(path))
|
||||
|
||||
# ---- resume ----
|
||||
|
||||
def test_resume_starts_every_service_again(self):
|
||||
with self.fake_helm(self.suspended()):
|
||||
self.assertTrue(sp.resume_instance(CFG, 3, self.log.append))
|
||||
for svc in sp.SERVICES:
|
||||
self.assertEqual(self.written["deployment"][svc]["replicaCount"], 1)
|
||||
self.assertFalse(sp.is_suspended(self.written))
|
||||
self.assertEqual(self.waited, [("ready", 3)])
|
||||
|
||||
def test_resuming_a_running_instance_changes_nothing(self):
|
||||
with self.fake_helm(self.RUNNING):
|
||||
self.assertFalse(sp.resume_instance(CFG, 1, self.log.append))
|
||||
self.assertEqual([c[0] for c in self.calls], ["get"])
|
||||
self.assertEqual(self.waited, [])
|
||||
|
||||
def test_a_suspended_instance_keeps_its_data_settings(self):
|
||||
with self.fake_helm(self.suspended()):
|
||||
sp.resume_instance(CFG, 1, self.log.append)
|
||||
self.assertEqual(self.written["persistence"]["db"]["size"], "2048Mi")
|
||||
self.assertEqual(self.written["secret"]["jwt"]["secret"], "keep-me")
|
||||
|
||||
# ---- upgrade must not trip over a suspended instance ----
|
||||
|
||||
def test_upgrade_leaves_a_suspended_instance_suspended_and_does_not_wait(self):
|
||||
with self.fake_helm(self.suspended()):
|
||||
sp.upgrade_instance(CFG, 1, self.log.append)
|
||||
self.assertTrue(sp.is_suspended(self.written)) # not woken up
|
||||
self.assertEqual(self.waited, []) # would never finish
|
||||
self.assertEqual(self.written["deployment"]["analytics"]["resources"]["limits"]["memory"],
|
||||
f"{sp.MEMORY['analytics'][1]}Mi") # new caps stored for the resume
|
||||
|
||||
def test_upgrade_still_waits_for_a_running_instance(self):
|
||||
with self.fake_helm(self.RUNNING):
|
||||
sp.upgrade_instance(CFG, 1, self.log.append)
|
||||
self.assertEqual(self.waited, [("ready", 1)])
|
||||
|
||||
|
||||
class InstanceState(unittest.TestCase):
|
||||
"""What `supaspin list` says about an instance, read from what the cluster is asked to run."""
|
||||
|
||||
def state(self, deployments, statefulsets):
|
||||
def kjson(*args):
|
||||
kind = args[1]
|
||||
counts = deployments if kind == "deployments" else statefulsets
|
||||
return {"items": [{"spec": {"replicas": c}} for c in counts]}
|
||||
|
||||
with mock.patch.object(sp, "kjson", kjson):
|
||||
return sp.instance_state(1)
|
||||
|
||||
def test_running(self):
|
||||
self.assertEqual(self.state([1] * 11, [1]), "running")
|
||||
|
||||
def test_suspended(self):
|
||||
self.assertEqual(self.state([0] * 11, [0]), "suspended")
|
||||
|
||||
def test_the_database_alone_still_running_is_reported_honestly(self):
|
||||
self.assertEqual(self.state([0] * 11, [1]), "partly suspended")
|
||||
|
||||
def test_nothing_there(self):
|
||||
self.assertEqual(self.state([], []), "unknown")
|
||||
|
||||
def test_the_list_shows_the_state_and_no_tunnel_for_a_sleeping_instance(self):
|
||||
rows = [{"n": 1, "host": "supa1.example.com", "pods": "0/0", "state": "suspended",
|
||||
"memory": "n/a", "disk": "db 1/2M", "env": "/data/supabase/1/.env"}]
|
||||
out = io.StringIO()
|
||||
with mock.patch.object(sp, "list_rows", lambda cfg: rows), \
|
||||
contextlib.redirect_stdout(out):
|
||||
sp.print_list(CFG)
|
||||
text = out.getvalue()
|
||||
self.assertIn("suspended", text)
|
||||
self.assertIn("supaspin resume 1", text)
|
||||
self.assertNotIn("ssh -N -L", text)
|
||||
|
||||
Reference in New Issue
Block a user