diff --git a/README.md b/README.md index 2de7054..52cc4a3 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,6 @@ # supaspin -Version 1.2.1 +Version 1.3.0 Create, list, show and delete throwaway [Supabase](https://supabase.com) instances on a single-node [microk8s](https://microk8s.io) server, from a terminal menu. @@ -94,6 +94,8 @@ Supabase Helm chart (0.8.0, Postgres 17.6.1.136) once into `/var/lib/supaspin/ch | Delete instance | asks you to type the number; removes everything (see below) | | Copy instance values into another .env file | puts an instance's `.env` values into your app's `.env`, on this server or on another computer (see below) | | Apply current memory caps to running instances | puts this version's memory limits on instances that already exist, replacing their pods one service at a time; databases, uploads and `.env` files are not touched | +| Suspend instance | stops every pod of one instance and keeps everything else: its memory, processor time, pod slots and two server ports go back to the server, its disks and `.env` file stay | +| Resume instance | starts a suspended instance again, with the data it had | | Mail settings | view and change the saved default mail settings | | Settings | change any first-run setting | @@ -108,9 +110,11 @@ sudo supaspin tunnel N # the tunnel commands, step by step sudo supaspin push-env N TARGET [--port P] [--key FILE] [--dry-run] # see "Copying an instance's values" sudo supaspin delete N [--yes] sudo supaspin upgrade [N] # apply this version's memory caps to running instances +sudo supaspin suspend N # stop an instance, keep its data (see "Suspending an instance") +sudo supaspin resume N # start it again sudo supaspin mail # change the saved default mail settings sudo supaspin setup # change settings -supaspin --version # prints "supaspin 1.2.1" +supaspin --version # prints "supaspin 1.3.0" ``` ### Connecting your app @@ -209,6 +213,39 @@ Creating that instance again restores the saved certificate, so repeated create/ not hit Let's Encrypt's limit of 5 certificates per hostname per week. If a create fails midway, run `supaspin delete N` to clean up. +## Suspending an instance + +An instance you are not using right now does not have to be deleted. `sudo supaspin suspend N` +stops all twelve of its pods and leaves everything else alone; `sudo supaspin resume N` starts +them again with the data they had. + +```bash +sudo supaspin suspend 2 # about half a minute +sudo supaspin resume 2 # a minute or two: the database answers first, the rest wait for it +``` + +What the server gets back while an instance sleeps: + +- the memory it had reserved (1.875 GiB) and everything it was actually using (1.5 to 2.5 GiB), +- its twelve pod slots, out of the 110 a node allows, +- its two ports on the server: Postgres on 544N and Studio on 300N are free for something else. + +What is kept, untouched: + +- the database and the uploaded files - the disk images stay mounted and keep their contents, +- the `.env` file with its passwords and keys, so your app needs no change when the instance is back, +- the HTTPS certificate, which goes on renewing itself while the instance sleeps, +- the hostname and every Kubernetes object. Until you resume it, the hostname answers "service + unavailable" instead of not answering at all. + +Disk space is **not** given back: that is the data you are keeping. `sudo supaspin list` shows the +state of every instance, and a suspended one is listed as `suspended` with no tunnel commands, +because nothing is listening on its ports. + +Suspending is written into the instance's stored settings, not done by hand to the running pods, +so it survives anything that re-applies the chart. `sudo supaspin upgrade` on a suspended instance +stores the new memory limits and leaves it asleep; they take effect when you resume it. + ## Memory per instance | Service | Request | Limit | Highest use measured | diff --git a/VERSION b/VERSION index 6085e94..f0bb29e 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -1.2.1 +1.3.0 diff --git a/supaspin b/supaspin index 102ef32..0b7ead5 100755 --- a/supaspin +++ b/supaspin @@ -8,6 +8,10 @@ Run as root: supaspin (menu) supaspin upgrade [N] Puts this version's memory caps on instances that already run, one service at a time. Without N every instance. Data and .env files are not touched. + supaspin suspend N | resume N + Suspend stops every pod of one instance and gives back its memory, its pod + slots and its two server ports. The disks, the passwords in .env, the + certificate and the hostname stay. Resume starts it again with its data. supaspin push-env N TARGET [--port P] [--key FILE] [--dry-run] TARGET = a .env file on this server (/path/.env) or on another computer (user@host:/path/.env, Windows: user@host:C:/path/.env). Puts the instance's @@ -35,7 +39,7 @@ import tty import yaml -VERSION = "1.2.1" +VERSION = "1.3.0" CONFIG_PATH = os.environ.get("SUPASPIN_CONFIG", "/etc/supaspin/config.yaml") STATE_DIR = os.environ.get("SUPASPIN_STATE", "/var/lib/supaspin") CHART_REPO = "https://supabase-community.github.io/supabase-kubernetes" @@ -208,6 +212,24 @@ def apply_memory(values): return values +def set_replicas(values, count): + """How many copies of each service to run. Zero stops the instance: the pods go away and give + back their memory, their processor time, their pod slots and the two ports they hold on the + server, while the disks, the passwords, the certificate and the hostname stay exactly as they + are. One starts it again.""" + dep = values.setdefault("deployment", {}) + for svc in SERVICES: + dep.setdefault(svc, {})["replicaCount"] = count + return values + + +def is_suspended(values): + """True when the stored settings say to run no copy of any service.""" + dep = values.get("deployment") or {} + counts = [(dep.get(svc) or {}).get("replicaCount", 1) for svc in SERVICES] + return all(c == 0 for c in counts) + + def memory_limits(values): """{service: limit} as the values dictionary has it, for telling before from after.""" dep = values.get("deployment") or {} @@ -807,20 +829,80 @@ def post_render_args(n): "--post-renderer-args", "post-render", "--post-renderer-args", str(n)] -def upgrade_instance(cfg, n, log): - """Bring a running instance up to the memory caps this version of supaspin carries, without - deleting anything. The values Helm kept from the install are read back, the memory block in - them is replaced, and the chart is applied again: the pods are replaced one service at a time - and the database, the uploads and the .env file are untouched.""" +def stored_values(cfg, n): + """The settings Helm kept from the install. They carry every secret of the instance, which is + why supaspin deletes its own copy of them as soon as Helm has read it.""" ns, rel = namespace(n), release(n) - host = cfg["slots"].get(n, "?") - log(f"Instance {n} ({host})") r = helm("get", "values", rel, "-n", ns, "-o", "yaml", check=False) if r.returncode != 0: raise RuntimeError(f"instance {n}: no Helm release {rel} in namespace {ns}\n{r.stderr.strip()}") values = yaml.safe_load(r.stdout) or {} if not values.get("deployment"): - raise RuntimeError(f"instance {n}: the stored values of {rel} look empty; not touching it") + raise RuntimeError(f"instance {n}: the stored settings of {rel} look empty; not touching it") + return values + + +def apply_values(cfg, n, values): + """Hand the settings to Helm, which works out what to change and changes only that.""" + vfile = write_values_file(n, values) + try: + helm("upgrade", release(n), chart_path(cfg), "-n", namespace(n), "-f", vfile, + *post_render_args(n)) + finally: + os.remove(vfile) # holds every secret + + +def wait_gone(n, log, timeout=300): + """Wait until the instance's own pods have stopped.""" + end = time.time() + timeout + while time.time() < end: + _, total, _ = pod_summary(n) + log(f" pods still running: {total}", live=True) + if total == 0: + log("") + return + time.sleep(3) + raise RuntimeError("the pods did not stop in time") + + +def suspend_instance(cfg, n, log): + """Stop every pod of an instance and keep everything else. The database, the uploads, the + passwords in .env, the certificate and the hostname are untouched; the instance answers + nothing until it is resumed.""" + log(f"Suspending instance {n} ({cfg['slots'].get(n, '?')})") + values = stored_values(cfg, n) + if is_suspended(values): + log(" already suspended") + return False + apply_values(cfg, n, set_replicas(values, 0)) + wait_gone(n, log) + log(f" instance {n} is suspended; its disks and its .env file are untouched") + log(f" start it again with: supaspin resume {n}") + return True + + +def resume_instance(cfg, n, log): + """Start a suspended instance again, with the data it had when it was suspended.""" + log(f"Resuming instance {n} ({cfg['slots'].get(n, '?')})") + values = stored_values(cfg, n) + if not is_suspended(values): + log(" already running") + return False + apply_values(cfg, n, set_replicas(values, 1)) + log(" the database answers first; the services that need it retry until it does") + wait_ready(n, log) + log(f" instance {n} is running again: https://{cfg['slots'].get(n, '?')}") + return True + + +def upgrade_instance(cfg, n, log): + """Bring a running instance up to the memory caps this version of supaspin carries, without + deleting anything. The values Helm kept from the install are read back, the memory block in + them is replaced, and the chart is applied again: the pods are replaced one service at a time + and the database, the uploads and the .env file are untouched.""" + log(f"Instance {n} ({cfg['slots'].get(n, '?')})") + values = stored_values(cfg, n) + asleep = is_suspended(values) before = memory_limits(values) after = memory_limits(apply_memory(values)) changed = [svc for svc in SERVICES if before[svc] != after[svc]] @@ -831,11 +913,12 @@ def upgrade_instance(cfg, n, log): # differently (the Studio rollout, the host ports, the gateway routes are all decided # while the manifests are rendered). Helm changes nothing if nothing differs. log(" caps already current; re-applying the chart in case anything else differs") - vfile = write_values_file(n, values) - try: - helm("upgrade", rel, chart_path(cfg), "-n", ns, "-f", vfile, *post_render_args(n)) - finally: - os.remove(vfile) # holds every secret + apply_values(cfg, n, values) + if asleep: + # A suspended instance stays suspended: the new caps are stored and take effect the moment + # it is resumed. Waiting for its pods would wait for ever - by design it has none. + log(f" instance {n} is suspended; the new caps apply when you resume it") + return bool(changed) wait_ready(n, log) log(f" instance {n} is running with the current settings") return bool(changed) @@ -923,12 +1006,29 @@ def disk_use(cfg, n): return ", ".join(parts) +def instance_state(n): + """What the cluster is currently asked to run for this instance: "running", "suspended", + "partly suspended" (some services stopped, which should not happen on its own) or "unknown" + when the instance has no workloads at all.""" + want = [] + for kind in ("deployments", "statefulsets"): + data = kjson("get", kind, "-n", namespace(n), "-l", f"app.kubernetes.io/instance={release(n)}") + for item in (data or {}).get("items", []): + want.append(item["spec"].get("replicas", 1)) + if not want: + return "unknown" + if all(w == 0 for w in want): + return "suspended" + return "partly suspended" if any(w == 0 for w in want) else "running" + + def list_rows(cfg): rows = [] for n in sorted(existing_numbers(cfg)): ready, total, _ = pod_summary(n) env = f"{inst_dir(cfg, n)}/.env" rows.append({"n": n, "host": cfg["slots"].get(n, "?"), "pods": f"{ready}/{total}", + "state": instance_state(n), "memory": memory_use(n), "disk": disk_use(cfg, n), "env": env if os.path.exists(env) else "(none)"}) return rows @@ -1646,8 +1746,12 @@ def print_list(cfg): print("No instances.") return for r in rows: - print(f"{BOLD}{r['n']}{OFF} {r['host']} pods {r['pods']} memory {r['memory']}") + state = r.get("state", "running") + print(f"{BOLD}{r['n']}{OFF} {r['host']} {state} pods {r['pods']} memory {r['memory']}") print(f" disk {r['disk']}\n env {r['env']}") + if state == "suspended": + print(f" start it again with: supaspin resume {r['n']}\n") + continue # the tunnel is pointless while nothing listens on the ports for line in tunnel_guide(cfg, r["n"]): print(f" {line}" if line else "") print() @@ -1658,8 +1762,9 @@ def main_menu(cfg, log): i = menu(f"supaspin {VERSION}", ["Create instance", "List instances", "Show instance", "Delete instance", "Copy instance values into another .env file", "Apply current memory caps to running instances", - "Mail settings", "Settings", "Quit"]) - if i is None or i == 8: + "Suspend instance (stop it, keep its data)", + "Resume instance", "Mail settings", "Settings", "Quit"]) + if i is None or i == 10: return print() try: @@ -1685,9 +1790,17 @@ def main_menu(cfg, log): elif i == 5: upgrade_all(cfg, log) elif i == 6: + n = pick_instance(cfg, "Suspend which instance?") + if n: + suspend_instance(cfg, n, log) + elif i == 7: + n = pick_instance(cfg, "Resume which instance?") + if n: + resume_instance(cfg, n, log) + elif i == 8: cfg["mail"] = mail_dialog(cfg) save_config(cfg) - elif i == 7: + elif i == 9: cfg = setup_dialog(cfg, log) except Exception as e: print(f"{RED}{e}{OFF}") @@ -1738,6 +1851,14 @@ def main(argv): delete_instance(cfg, n, log_line) elif cmd == "upgrade": upgrade_all(cfg, log_line, n) + elif cmd in ("suspend", "resume"): + if n is None: + # No "all" here on purpose: stopping every instance is not something to type by + # accident, and resuming them all at once would start 36 pods in one go. + print(f"supaspin {cmd} N - which instance? ({cmd} takes one number)", + file=sys.stderr) + return 2 + (suspend_instance if cmd == "suspend" else resume_instance)(cfg, n, log_line) elif cmd == "push-env": return push_env_command(cfg, argv[2:]) elif cmd == "mail": diff --git a/tests/test_supaspin.py b/tests/test_supaspin.py index 6773e14..7cea9ab 100644 --- a/tests/test_supaspin.py +++ b/tests/test_supaspin.py @@ -1105,3 +1105,173 @@ class HostPortRollout(unittest.TestCase): self.assertNotIn("strategy", d["spec"]) self.assertEqual(d["spec"]["template"]["spec"]["containers"][0]["ports"][0]["hostPort"], sp.pg_port(1)) + + +class SuspendResume(unittest.TestCase): + """Suspending an instance stops its pods and keeps everything else: the disks, the passwords, + the certificate and the hostname. Resuming starts it again with the same data.""" + + RUNNING = { + "deployment": { + "analytics": {"enabled": True, + "resources": {"limits": {"memory": "1536Mi"}, + "requests": {"memory": "128Mi"}}}, + "db": {"resources": {"limits": {"memory": "2048Mi"}, + "requests": {"memory": "512Mi"}}}, + }, + "secret": {"jwt": {"secret": "keep-me"}, "db": {"password": "keep-me-too"}}, + "persistence": {"db": {"enabled": True, "size": "2048Mi"}}, + } + + def setUp(self): + self.calls = [] + self.written = {} + self.log = [] + self.waited = [] + + @contextlib.contextmanager + def fake_helm(self, stored): + P = collections.namedtuple("P", "returncode stdout stderr") + + def helm(*args, check=True): + self.calls.append(list(args)) + if args[0] == "get": + return P(0, yaml.safe_dump(stored), "") + if args[0] == "upgrade": + with open(args[args.index("-f") + 1]) as f: + self.written = yaml.safe_load(f) + return P(0, "", "") + + with mock.patch.object(sp, "helm", helm), \ + mock.patch.object(sp, "wait_ready", + lambda n, log, **kw: self.waited.append(("ready", n))), \ + mock.patch.object(sp, "wait_gone", + lambda n, log, **kw: self.waited.append(("gone", n))), \ + mock.patch.object(sp, "STATE_DIR", tempfile.mkdtemp()): + yield + + def suspended(self): + return sp.set_replicas(yaml.safe_load(yaml.safe_dump(self.RUNNING)), 0) + + # ---- the setting itself ---- + + def test_set_replicas_covers_every_service(self): + v = sp.set_replicas({}, 0) + self.assertEqual(sorted(v["deployment"]), sorted(sp.SERVICES)) + self.assertTrue(all(d["replicaCount"] == 0 for d in v["deployment"].values())) + self.assertTrue(sp.is_suspended(v)) + + def test_a_service_missing_the_setting_counts_as_running(self): + self.assertFalse(sp.is_suspended(self.RUNNING)) + self.assertFalse(sp.is_suspended({"deployment": {}})) + + def test_one_service_left_running_is_not_suspended(self): + v = sp.set_replicas({}, 0) + v["deployment"]["kong"]["replicaCount"] = 1 + self.assertFalse(sp.is_suspended(v)) + + # ---- suspend ---- + + def test_suspend_stops_every_service_and_keeps_the_rest(self): + with self.fake_helm(self.RUNNING): + self.assertTrue(sp.suspend_instance(CFG, 1, self.log.append)) + for svc in sp.SERVICES: + self.assertEqual(self.written["deployment"][svc]["replicaCount"], 0) + self.assertEqual(self.written["secret"]["jwt"]["secret"], "keep-me") + self.assertEqual(self.written["secret"]["db"]["password"], "keep-me-too") + self.assertEqual(self.written["persistence"]["db"]["size"], "2048Mi") + # the memory caps are kept as they were: they apply again on resume + self.assertEqual(self.written["deployment"]["analytics"]["resources"]["limits"]["memory"], + "1536Mi") + self.assertEqual(self.waited, [("gone", 1)]) + + def test_suspend_waits_for_the_pods_to_be_gone_not_for_them_to_be_ready(self): + with self.fake_helm(self.RUNNING): + sp.suspend_instance(CFG, 2, self.log.append) + self.assertEqual([w[0] for w in self.waited], ["gone"]) + + def test_suspending_a_suspended_instance_changes_nothing(self): + with self.fake_helm(self.suspended()): + self.assertFalse(sp.suspend_instance(CFG, 1, self.log.append)) + self.assertEqual([c[0] for c in self.calls], ["get"]) + self.assertEqual(self.waited, []) + + def test_the_values_file_with_the_secrets_is_deleted_again(self): + with self.fake_helm(self.RUNNING): + sp.suspend_instance(CFG, 1, self.log.append) + path = [c[c.index("-f") + 1] for c in self.calls if c[0] == "upgrade"][0] + self.assertFalse(os.path.exists(path)) + + # ---- resume ---- + + def test_resume_starts_every_service_again(self): + with self.fake_helm(self.suspended()): + self.assertTrue(sp.resume_instance(CFG, 3, self.log.append)) + for svc in sp.SERVICES: + self.assertEqual(self.written["deployment"][svc]["replicaCount"], 1) + self.assertFalse(sp.is_suspended(self.written)) + self.assertEqual(self.waited, [("ready", 3)]) + + def test_resuming_a_running_instance_changes_nothing(self): + with self.fake_helm(self.RUNNING): + self.assertFalse(sp.resume_instance(CFG, 1, self.log.append)) + self.assertEqual([c[0] for c in self.calls], ["get"]) + self.assertEqual(self.waited, []) + + def test_a_suspended_instance_keeps_its_data_settings(self): + with self.fake_helm(self.suspended()): + sp.resume_instance(CFG, 1, self.log.append) + self.assertEqual(self.written["persistence"]["db"]["size"], "2048Mi") + self.assertEqual(self.written["secret"]["jwt"]["secret"], "keep-me") + + # ---- upgrade must not trip over a suspended instance ---- + + def test_upgrade_leaves_a_suspended_instance_suspended_and_does_not_wait(self): + with self.fake_helm(self.suspended()): + sp.upgrade_instance(CFG, 1, self.log.append) + self.assertTrue(sp.is_suspended(self.written)) # not woken up + self.assertEqual(self.waited, []) # would never finish + self.assertEqual(self.written["deployment"]["analytics"]["resources"]["limits"]["memory"], + f"{sp.MEMORY['analytics'][1]}Mi") # new caps stored for the resume + + def test_upgrade_still_waits_for_a_running_instance(self): + with self.fake_helm(self.RUNNING): + sp.upgrade_instance(CFG, 1, self.log.append) + self.assertEqual(self.waited, [("ready", 1)]) + + +class InstanceState(unittest.TestCase): + """What `supaspin list` says about an instance, read from what the cluster is asked to run.""" + + def state(self, deployments, statefulsets): + def kjson(*args): + kind = args[1] + counts = deployments if kind == "deployments" else statefulsets + return {"items": [{"spec": {"replicas": c}} for c in counts]} + + with mock.patch.object(sp, "kjson", kjson): + return sp.instance_state(1) + + def test_running(self): + self.assertEqual(self.state([1] * 11, [1]), "running") + + def test_suspended(self): + self.assertEqual(self.state([0] * 11, [0]), "suspended") + + def test_the_database_alone_still_running_is_reported_honestly(self): + self.assertEqual(self.state([0] * 11, [1]), "partly suspended") + + def test_nothing_there(self): + self.assertEqual(self.state([], []), "unknown") + + def test_the_list_shows_the_state_and_no_tunnel_for_a_sleeping_instance(self): + rows = [{"n": 1, "host": "supa1.example.com", "pods": "0/0", "state": "suspended", + "memory": "n/a", "disk": "db 1/2M", "env": "/data/supabase/1/.env"}] + out = io.StringIO() + with mock.patch.object(sp, "list_rows", lambda cfg: rows), \ + contextlib.redirect_stdout(out): + sp.print_list(CFG) + text = out.getvalue() + self.assertIn("suspended", text) + self.assertIn("supaspin resume 1", text) + self.assertNotIn("ssh -N -L", text)