fix: delete analytics logs older than 2 days and give vector a 512 MiB cap
- every instance gets a pg_cron job (set up on create, upgrade and resume) that deletes _analytics.log_events_* rows older than 2 days and runs the first cleanup right away; Logflare never deletes stored logs, and instance 3's 2 GB db image died 100% full of them on 2026-10-06, after which analytics 500'd every delivery and vector was OOM-killed by its undeliverable backlog every few minutes - vector's memory limit back to 512 MiB, the ceiling the production supabase charts carry; the retention job bounds its backlog, so 1024 MiB is not needed anymore - version 1.3.2
This commit is contained in:
+21
-6
@@ -90,14 +90,17 @@ class Values(unittest.TestCase):
|
||||
|
||||
def test_memory_caps_add_up(self):
|
||||
req, lim = sp.total_memory_mib()
|
||||
# db 2048, analytics 1536, kong 768, realtime 768, vector 1024, the other seven 512 each
|
||||
self.assertEqual(lim, 2048 + 1536 + 768 + 768 + 1024 + 7 * 512)
|
||||
self.assertEqual(lim / 1024, 9.5)
|
||||
# db 2048, analytics 1536, kong 768, realtime 768, vector 512, the other seven 512 each
|
||||
self.assertEqual(lim, 2048 + 1536 + 768 + 768 + 512 + 7 * 512)
|
||||
self.assertEqual(lim / 1024, 9.0)
|
||||
self.assertEqual(req, 512 + 11 * 128) # 1.875 GiB
|
||||
# Measured on three running instances on 2026-09-22; the ceiling has to stay clear of the
|
||||
# highest use, or the kernel kills the container in a loop the way it did on instance 1.
|
||||
self.assertGreaterEqual(sp.MEMORY["analytics"][1], 1536) # used 694 MiB, died at 768
|
||||
self.assertGreaterEqual(sp.MEMORY["vector"][1], 1024) # used 290 MiB, died at 512
|
||||
self.assertGreaterEqual(sp.MEMORY["vector"][1], 512) # used 290 MiB; the nightly log
|
||||
# retention keeps its delivery backlog from growing past this the way it did on
|
||||
# 2026-10-06, when instance 3's full db image killed analytics and vector filled with
|
||||
# the undeliverable backlog every few minutes against 1024 MiB.
|
||||
self.assertGreaterEqual(sp.MEMORY["realtime"][1], 768) # used 300 MiB against 384
|
||||
self.assertGreaterEqual(sp.MEMORY["kong"][1], 768) # used 269 MiB against 512
|
||||
self.assertGreaterEqual(sp.DEFAULT_MEMORY[1], 512) # studio used 203 against 256
|
||||
@@ -128,6 +131,15 @@ class Values(unittest.TestCase):
|
||||
self.assertEqual(set(sp.VOLUMES), {"db", "storage", "functions", "snippets", "deno", "pgsodium"})
|
||||
self.assertTrue(all(p[v]["storageClassName"] == "supaspin-local-sc" for v in sp.VOLUMES))
|
||||
|
||||
def test_log_retention_sql(self):
|
||||
# Logflare stores logs in _analytics.log_events_* and never deletes them; the job deletes
|
||||
# rows older than LOG_RETENTION and is scheduled under a fixed name so re-runs overwrite.
|
||||
self.assertIn("schemaname = '_analytics'", sp.RETENTION_SQL)
|
||||
self.assertIn("like 'log_events\\_%'", sp.RETENTION_SQL)
|
||||
self.assertIn(f"interval ''{sp.LOG_RETENTION}''", sp.RETENTION_SQL)
|
||||
self.assertIn(f"'{sp.LOG_RETENTION_JOB}'", sp.RETENTION_SQL)
|
||||
self.assertIn("cron.schedule", sp.RETENTION_SQL)
|
||||
|
||||
def test_imgproxy_reads_the_storage_volume(self):
|
||||
img = self.v["persistence"]["imgproxy"]
|
||||
self.assertEqual(img["existingClaim"], "supa2-supabase-storage")
|
||||
@@ -975,6 +987,7 @@ class Upgrade(unittest.TestCase):
|
||||
|
||||
with mock.patch.object(sp, "helm", helm), \
|
||||
mock.patch.object(sp, "wait_ready", lambda n, log, **kw: None), \
|
||||
mock.patch.object(sp, "setup_log_retention", lambda cfg, n, log: None), \
|
||||
mock.patch.object(sp, "STATE_DIR", tempfile.mkdtemp()):
|
||||
yield
|
||||
|
||||
@@ -1147,6 +1160,8 @@ class SuspendResume(unittest.TestCase):
|
||||
lambda n, log, **kw: self.waited.append(("ready", n))), \
|
||||
mock.patch.object(sp, "wait_gone",
|
||||
lambda n, log, **kw: self.waited.append(("gone", n))), \
|
||||
mock.patch.object(sp, "setup_log_retention",
|
||||
lambda cfg, n, log: self.waited.append(("retention", n))), \
|
||||
mock.patch.object(sp, "STATE_DIR", tempfile.mkdtemp()):
|
||||
yield
|
||||
|
||||
@@ -1210,7 +1225,7 @@ class SuspendResume(unittest.TestCase):
|
||||
for svc in sp.SERVICES:
|
||||
self.assertEqual(self.written["deployment"][svc]["replicaCount"], 1)
|
||||
self.assertFalse(sp.is_suspended(self.written))
|
||||
self.assertEqual(self.waited, [("ready", 3)])
|
||||
self.assertEqual(self.waited, [("ready", 3), ("retention", 3)])
|
||||
|
||||
def test_resuming_a_running_instance_changes_nothing(self):
|
||||
with self.fake_helm(self.RUNNING):
|
||||
@@ -1237,7 +1252,7 @@ class SuspendResume(unittest.TestCase):
|
||||
def test_upgrade_still_waits_for_a_running_instance(self):
|
||||
with self.fake_helm(self.RUNNING):
|
||||
sp.upgrade_instance(CFG, 1, self.log.append)
|
||||
self.assertEqual(self.waited, [("ready", 1)])
|
||||
self.assertEqual(self.waited, [("ready", 1), ("retention", 1)])
|
||||
|
||||
|
||||
class InstanceState(unittest.TestCase):
|
||||
|
||||
Reference in New Issue
Block a user