fix: delete analytics logs older than 2 days and give vector a 512 MiB cap

- every instance gets a pg_cron job (set up on create, upgrade and resume)
  that deletes _analytics.log_events_* rows older than 2 days and runs the
  first cleanup right away; Logflare never deletes stored logs, and instance
  3's 2 GB db image died 100% full of them on 2026-10-06, after which
  analytics 500'd every delivery and vector was OOM-killed by its
  undeliverable backlog every few minutes
- vector's memory limit back to 512 MiB, the ceiling the production
  supabase charts carry; the retention job bounds its backlog, so 1024 MiB
  is not needed anymore
- version 1.3.2
This commit is contained in:
Igor K.
2026-10-06 20:28:29 +03:00
parent 4bcdf6907c
commit a923839c47
4 changed files with 89 additions and 22 deletions
+21 -6
View File
@@ -90,14 +90,17 @@ class Values(unittest.TestCase):
def test_memory_caps_add_up(self):
req, lim = sp.total_memory_mib()
# db 2048, analytics 1536, kong 768, realtime 768, vector 1024, the other seven 512 each
self.assertEqual(lim, 2048 + 1536 + 768 + 768 + 1024 + 7 * 512)
self.assertEqual(lim / 1024, 9.5)
# db 2048, analytics 1536, kong 768, realtime 768, vector 512, the other seven 512 each
self.assertEqual(lim, 2048 + 1536 + 768 + 768 + 512 + 7 * 512)
self.assertEqual(lim / 1024, 9.0)
self.assertEqual(req, 512 + 11 * 128) # 1.875 GiB
# Measured on three running instances on 2026-09-22; the ceiling has to stay clear of the
# highest use, or the kernel kills the container in a loop the way it did on instance 1.
self.assertGreaterEqual(sp.MEMORY["analytics"][1], 1536) # used 694 MiB, died at 768
self.assertGreaterEqual(sp.MEMORY["vector"][1], 1024) # used 290 MiB, died at 512
self.assertGreaterEqual(sp.MEMORY["vector"][1], 512) # used 290 MiB; the nightly log
# retention keeps its delivery backlog from growing past this the way it did on
# 2026-10-06, when instance 3's full db image killed analytics and vector filled with
# the undeliverable backlog every few minutes against 1024 MiB.
self.assertGreaterEqual(sp.MEMORY["realtime"][1], 768) # used 300 MiB against 384
self.assertGreaterEqual(sp.MEMORY["kong"][1], 768) # used 269 MiB against 512
self.assertGreaterEqual(sp.DEFAULT_MEMORY[1], 512) # studio used 203 against 256
@@ -128,6 +131,15 @@ class Values(unittest.TestCase):
self.assertEqual(set(sp.VOLUMES), {"db", "storage", "functions", "snippets", "deno", "pgsodium"})
self.assertTrue(all(p[v]["storageClassName"] == "supaspin-local-sc" for v in sp.VOLUMES))
def test_log_retention_sql(self):
# Logflare stores logs in _analytics.log_events_* and never deletes them; the job deletes
# rows older than LOG_RETENTION and is scheduled under a fixed name so re-runs overwrite.
self.assertIn("schemaname = '_analytics'", sp.RETENTION_SQL)
self.assertIn("like 'log_events\\_%'", sp.RETENTION_SQL)
self.assertIn(f"interval ''{sp.LOG_RETENTION}''", sp.RETENTION_SQL)
self.assertIn(f"'{sp.LOG_RETENTION_JOB}'", sp.RETENTION_SQL)
self.assertIn("cron.schedule", sp.RETENTION_SQL)
def test_imgproxy_reads_the_storage_volume(self):
img = self.v["persistence"]["imgproxy"]
self.assertEqual(img["existingClaim"], "supa2-supabase-storage")
@@ -975,6 +987,7 @@ class Upgrade(unittest.TestCase):
with mock.patch.object(sp, "helm", helm), \
mock.patch.object(sp, "wait_ready", lambda n, log, **kw: None), \
mock.patch.object(sp, "setup_log_retention", lambda cfg, n, log: None), \
mock.patch.object(sp, "STATE_DIR", tempfile.mkdtemp()):
yield
@@ -1147,6 +1160,8 @@ class SuspendResume(unittest.TestCase):
lambda n, log, **kw: self.waited.append(("ready", n))), \
mock.patch.object(sp, "wait_gone",
lambda n, log, **kw: self.waited.append(("gone", n))), \
mock.patch.object(sp, "setup_log_retention",
lambda cfg, n, log: self.waited.append(("retention", n))), \
mock.patch.object(sp, "STATE_DIR", tempfile.mkdtemp()):
yield
@@ -1210,7 +1225,7 @@ class SuspendResume(unittest.TestCase):
for svc in sp.SERVICES:
self.assertEqual(self.written["deployment"][svc]["replicaCount"], 1)
self.assertFalse(sp.is_suspended(self.written))
self.assertEqual(self.waited, [("ready", 3)])
self.assertEqual(self.waited, [("ready", 3), ("retention", 3)])
def test_resuming_a_running_instance_changes_nothing(self):
with self.fake_helm(self.RUNNING):
@@ -1237,7 +1252,7 @@ class SuspendResume(unittest.TestCase):
def test_upgrade_still_waits_for_a_running_instance(self):
with self.fake_helm(self.RUNNING):
sp.upgrade_instance(CFG, 1, self.log.append)
self.assertEqual(self.waited, [("ready", 1)])
self.assertEqual(self.waited, [("ready", 1), ("retention", 1)])
class InstanceState(unittest.TestCase):