feat: raise the memory ceilings and add supaspin upgrade
The two logging services were being killed by the kernel in a loop on instance 1 (supa1.didgibot.com): vector 297 times, analytics 36 times, both exit code 137 (out of memory). The old ceilings sat right on top of what the services normally use, so any backlog crossed them. Measured across the three running instances on 2026-09-22: - analytics 612 / 694 / 662 MiB against a 768 MiB ceiling -> 1536 MiB - vector 287 / 290 MiB against 512 MiB -> 1024 MiB - realtime 269 / 287 / 300 MiB against 384 MiB -> 768 MiB - kong 260 / 269 / 265 MiB against 512 MiB -> 768 MiB - studio 203, storage 187, rest 162 MiB against 256 MiB -> 512 MiB - db 163 / 175 / 273 MiB against 2048 MiB -> unchanged Limits only. The requests stay at 128 MiB (512 for db): the server is one machine shared with the production workloads and already has 80% of its memory reserved, so raising the requests would leave production nothing to schedule into. Raising a limit reserves nothing - it only moves the point at which the kernel kills. Per-instance totals go from 5.875 GiB to 9.5 GiB of limit; reserved memory is unchanged at 1.875 GiB. New numbers only reach instances created afterwards, so this also adds `sudo supaspin upgrade [N]`: it reads back the values Helm stored at install time, replaces the memory block in them and applies the chart again. Pods are replaced service by service; databases, uploads, certificates and .env files are untouched. Without a number it does every instance and reports which ones failed. Also available from the menu. Fixes a rollout that could never finish, found by the first upgrade: Studio binds port 3001 (3002, 3003) on the server itself so it can be reached over an SSH tunnel, and one port can be held by one pod at a time. The default rollout starts the replacement pod before stopping the old one, so the replacement never got the port and stayed Pending for ever. The Studio deployment now stops the old pod first. Postgres was never affected: a StatefulSet already replaces its pod that way. The values file that carries the instance's secrets and the post-renderer arguments are now built by shared helpers used by both create and upgrade. Tests: 113 pass (12 new). Version 1.1.2 -> 1.2.0 in VERSION, the VERSION constant and README. Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
This commit was merged in pull request #5.
This commit is contained in:
Igor
+183
-7
@@ -90,14 +90,20 @@ class Values(unittest.TestCase):
|
||||
|
||||
def test_memory_caps_add_up(self):
|
||||
req, lim = sp.total_memory_mib()
|
||||
# db 2048, analytics 768, kong 512, realtime 384, vector 512, the other seven 256 each
|
||||
self.assertEqual(lim, 2048 + 768 + 512 + 384 + 512 + 7 * 256)
|
||||
self.assertEqual(lim / 1024, 5.875)
|
||||
# db 2048, analytics 1536, kong 768, realtime 768, vector 1024, the other seven 512 each
|
||||
self.assertEqual(lim, 2048 + 1536 + 768 + 768 + 1024 + 7 * 512)
|
||||
self.assertEqual(lim / 1024, 9.5)
|
||||
self.assertEqual(req, 512 + 11 * 128) # 1.875 GiB
|
||||
# analytics, realtime and vector were killed for running out of memory at half of production
|
||||
self.assertGreaterEqual(sp.MEMORY["analytics"][1], 768)
|
||||
self.assertGreaterEqual(sp.MEMORY["realtime"][1], 384)
|
||||
self.assertGreaterEqual(sp.MEMORY["vector"][1], 512)
|
||||
# Measured on three running instances on 2026-09-22; the ceiling has to stay clear of the
|
||||
# highest use, or the kernel kills the container in a loop the way it did on instance 1.
|
||||
self.assertGreaterEqual(sp.MEMORY["analytics"][1], 1536) # used 694 MiB, died at 768
|
||||
self.assertGreaterEqual(sp.MEMORY["vector"][1], 1024) # used 290 MiB, died at 512
|
||||
self.assertGreaterEqual(sp.MEMORY["realtime"][1], 768) # used 300 MiB against 384
|
||||
self.assertGreaterEqual(sp.MEMORY["kong"][1], 768) # used 269 MiB against 512
|
||||
self.assertGreaterEqual(sp.DEFAULT_MEMORY[1], 512) # studio used 203 against 256
|
||||
# The requests stay where they were: the node is shared with production and reserving
|
||||
# what the instances really use would leave production nothing to schedule into.
|
||||
self.assertTrue(all(m[0] == 128 for s, m in sp.MEMORY.items() if s != "db"))
|
||||
self.assertEqual(req / 1024, 1.875)
|
||||
for svc in sp.SERVICES:
|
||||
self.assertEqual(self.v["deployment"][svc]["priorityClassName"], "supaspin-low")
|
||||
@@ -929,3 +935,173 @@ class PushEnvCommand(unittest.TestCase):
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
|
||||
|
||||
class Upgrade(unittest.TestCase):
|
||||
"""`supaspin upgrade` puts the current memory caps on an instance that is already running.
|
||||
It must change the caps and nothing else: no volume is resized, no secret is regenerated."""
|
||||
|
||||
STORED = {
|
||||
"deployment": {
|
||||
"analytics": {"enabled": True, "priorityClassName": "supaspin-low",
|
||||
"resources": {"limits": {"memory": "768Mi"},
|
||||
"requests": {"memory": "128Mi"}}},
|
||||
"vector": {"enabled": True, "priorityClassName": "supaspin-low",
|
||||
"resources": {"limits": {"memory": "512Mi"},
|
||||
"requests": {"memory": "128Mi"}}},
|
||||
},
|
||||
"secret": {"jwt": {"secret": "keep-me", "anonKey": "anon"}},
|
||||
"persistence": {"db": {"enabled": True, "size": "2048Mi"}},
|
||||
}
|
||||
|
||||
def setUp(self):
|
||||
self.calls = []
|
||||
self.written = {}
|
||||
self.log = []
|
||||
|
||||
@contextlib.contextmanager
|
||||
def fake_helm(self, stored=None, rc=0):
|
||||
stored = self.STORED if stored is None else stored
|
||||
|
||||
def helm(*args, check=True):
|
||||
self.calls.append(list(args))
|
||||
if args[0] == "get":
|
||||
out = yaml.safe_dump(stored) if stored is not None else ""
|
||||
return collections.namedtuple("P", "returncode stdout stderr")(rc, out, "boom")
|
||||
if args[0] == "upgrade":
|
||||
with open(args[args.index("-f") + 1]) as f:
|
||||
self.written = yaml.safe_load(f)
|
||||
return collections.namedtuple("P", "returncode stdout stderr")(0, "", "")
|
||||
|
||||
with mock.patch.object(sp, "helm", helm), \
|
||||
mock.patch.object(sp, "wait_ready", lambda n, log, **kw: None), \
|
||||
mock.patch.object(sp, "STATE_DIR", tempfile.mkdtemp()):
|
||||
yield
|
||||
|
||||
def test_raises_the_caps_and_keeps_everything_else(self):
|
||||
with self.fake_helm():
|
||||
changed = sp.upgrade_instance(CFG, 1, self.log.append)
|
||||
self.assertTrue(changed)
|
||||
dep = self.written["deployment"]
|
||||
self.assertEqual(dep["analytics"]["resources"]["limits"]["memory"],
|
||||
f"{sp.MEMORY['analytics'][1]}Mi")
|
||||
self.assertEqual(dep["vector"]["resources"]["limits"]["memory"],
|
||||
f"{sp.MEMORY['vector'][1]}Mi")
|
||||
self.assertTrue(dep["analytics"]["enabled"]) # untouched
|
||||
self.assertEqual(self.written["secret"]["jwt"]["secret"], "keep-me")
|
||||
self.assertEqual(self.written["persistence"]["db"]["size"], "2048Mi")
|
||||
|
||||
def test_every_service_gets_a_cap_even_if_the_stored_values_had_none(self):
|
||||
with self.fake_helm():
|
||||
sp.upgrade_instance(CFG, 1, self.log.append)
|
||||
for svc in sp.SERVICES:
|
||||
req, lim = sp.MEMORY.get(svc, sp.DEFAULT_MEMORY)
|
||||
got = self.written["deployment"][svc]["resources"]
|
||||
self.assertEqual(got["limits"]["memory"], f"{lim}Mi")
|
||||
self.assertEqual(got["requests"]["memory"], f"{req}Mi")
|
||||
|
||||
def test_the_values_file_with_the_secrets_is_deleted_again(self):
|
||||
with self.fake_helm():
|
||||
sp.upgrade_instance(CFG, 1, self.log.append)
|
||||
path = [c[c.index("-f") + 1] for c in self.calls if c[0] == "upgrade"][0]
|
||||
self.assertFalse(os.path.exists(path))
|
||||
|
||||
def test_helm_gets_the_post_renderer(self):
|
||||
with self.fake_helm():
|
||||
sp.upgrade_instance(CFG, 2, self.log.append)
|
||||
up = [c for c in self.calls if c[0] == "upgrade"][0]
|
||||
self.assertIn("--post-renderer", up)
|
||||
self.assertEqual(up[up.index("--post-renderer-args") + 1], "post-render")
|
||||
self.assertIn("2", up)
|
||||
|
||||
def test_an_instance_already_at_the_caps_is_still_re_applied(self):
|
||||
# The caps are not the only thing a new version can change: the Studio rollout, the host
|
||||
# ports and the gateway routes are decided while the manifests are rendered, and none of
|
||||
# them shows up in the stored values.
|
||||
stored = {"deployment": sp.apply_memory({})["deployment"]}
|
||||
with self.fake_helm(stored):
|
||||
changed = sp.upgrade_instance(CFG, 1, self.log.append)
|
||||
self.assertFalse(changed) # no cap moved
|
||||
self.assertEqual([c[0] for c in self.calls], ["get", "upgrade"]) # applied anyway
|
||||
|
||||
def test_a_missing_release_is_refused_before_anything_is_written(self):
|
||||
with self.fake_helm(rc=1):
|
||||
with self.assertRaises(RuntimeError):
|
||||
sp.upgrade_instance(CFG, 3, self.log.append)
|
||||
self.assertEqual([c[0] for c in self.calls], ["get"])
|
||||
|
||||
def test_empty_stored_values_are_refused(self):
|
||||
with self.fake_helm({}):
|
||||
with self.assertRaises(RuntimeError):
|
||||
sp.upgrade_instance(CFG, 1, self.log.append)
|
||||
self.assertEqual([c[0] for c in self.calls], ["get"])
|
||||
|
||||
def test_upgrade_all_does_every_instance_and_reports_the_failures(self):
|
||||
done = []
|
||||
|
||||
def one(cfg, n, log):
|
||||
if n == 2:
|
||||
raise RuntimeError("no release")
|
||||
done.append(n)
|
||||
return True
|
||||
|
||||
with mock.patch.object(sp, "upgrade_instance", one), \
|
||||
mock.patch.object(sp, "existing_numbers", lambda cfg: {1, 2, 3}):
|
||||
with self.assertRaises(RuntimeError) as e:
|
||||
sp.upgrade_all(CFG, self.log.append)
|
||||
self.assertEqual(done, [1, 3])
|
||||
self.assertIn("2", str(e.exception))
|
||||
|
||||
def test_upgrade_all_with_a_number_does_only_that_instance(self):
|
||||
done = []
|
||||
with mock.patch.object(sp, "upgrade_instance", lambda cfg, n, log: done.append(n)), \
|
||||
mock.patch.object(sp, "existing_numbers", lambda cfg: {1, 2, 3}):
|
||||
sp.upgrade_all(CFG, self.log.append, 2)
|
||||
self.assertEqual(done, [2])
|
||||
|
||||
|
||||
class HostPortRollout(unittest.TestCase):
|
||||
"""Studio and Postgres bind a port on the server itself, and only one pod can hold a port.
|
||||
The Studio deployment must therefore stop its old pod before starting the new one, or an
|
||||
upgrade leaves the replacement pod Pending for ever."""
|
||||
|
||||
def rendered(self, n=1):
|
||||
sec = sp.generate_secrets()
|
||||
values = sp.instance_values(CFG, n, sec, MAIL, template())
|
||||
with mock.patch.object(sp, "helm"), mock.patch.object(sp, "kubectl"):
|
||||
text = yaml.safe_dump_all([
|
||||
{"apiVersion": "apps/v1", "kind": "Deployment",
|
||||
"metadata": {"name": f"{sp.fullname(n)}-studio"},
|
||||
"spec": {"strategy": {"type": "RollingUpdate",
|
||||
"rollingUpdate": {"maxSurge": "25%"}},
|
||||
"template": {"spec": {"containers": [
|
||||
{"name": "studio", "ports": [{"containerPort": 3000}]}]}}}},
|
||||
{"apiVersion": "apps/v1", "kind": "StatefulSet",
|
||||
"metadata": {"name": f"{sp.fullname(n)}-db"},
|
||||
"spec": {"template": {"spec": {"containers": [
|
||||
{"name": "db", "ports": [{"containerPort": 5432}]}]}}}},
|
||||
{"apiVersion": "apps/v1", "kind": "Deployment",
|
||||
"metadata": {"name": f"{sp.fullname(n)}-rest"},
|
||||
"spec": {"strategy": {"type": "RollingUpdate"},
|
||||
"template": {"spec": {"containers": [
|
||||
{"name": "rest", "ports": [{"containerPort": 3000}]}]}}}},
|
||||
])
|
||||
out = list(yaml.safe_load_all(sp.post_render(text, n, CFG["lan_ip"])))
|
||||
return {d["metadata"]["name"]: d for d in out}
|
||||
|
||||
def test_studio_stops_the_old_pod_before_starting_the_new_one(self):
|
||||
d = self.rendered()[f"{sp.fullname(1)}-studio"]
|
||||
self.assertEqual(d["spec"]["strategy"], {"type": "Recreate"})
|
||||
port = d["spec"]["template"]["spec"]["containers"][0]["ports"][0]
|
||||
self.assertEqual(port["hostPort"], sp.studio_port(1))
|
||||
self.assertEqual(port["hostIP"], CFG["lan_ip"])
|
||||
|
||||
def test_a_deployment_without_a_host_port_keeps_the_normal_rollout(self):
|
||||
d = self.rendered()[f"{sp.fullname(1)}-rest"]
|
||||
self.assertEqual(d["spec"]["strategy"]["type"], "RollingUpdate")
|
||||
|
||||
def test_the_database_keeps_its_host_port_and_gains_no_strategy(self):
|
||||
d = self.rendered()[f"{sp.fullname(1)}-db"]
|
||||
self.assertNotIn("strategy", d["spec"])
|
||||
self.assertEqual(d["spec"]["template"]["spec"]["containers"][0]["ports"][0]["hostPort"],
|
||||
sp.pg_port(1))
|
||||
|
||||
Reference in New Issue
Block a user