fix: reach the log-retention job into _supabase over dblink #10

Merged
igor merged 1 commits from fix/log-retention-pg-cron-dblink into main 2026-10-06 17:34:48 +00:00
4 changed files with 37 additions and 15 deletions
Showing only changes of commit 2c6e383617 - Show all commits
+5 -3
View File
@@ -1,6 +1,6 @@
# supaspin # supaspin
Version 1.3.2 Version 1.3.3
Create, list, show and delete throwaway [Supabase](https://supabase.com) instances on a Create, list, show and delete throwaway [Supabase](https://supabase.com) instances on a
single-node [microk8s](https://microk8s.io) server, from a terminal menu. single-node [microk8s](https://microk8s.io) server, from a terminal menu.
@@ -291,8 +291,10 @@ it receives in the `_supabase` database and never deletes them, so the 2 GB db i
every delivery with an error, and vector was killed against its 1024 MiB ceiling every few every delivery with an error, and vector was killed against its 1024 MiB ceiling every few
minutes with the undeliverable backlog. The fix is log retention: every instance gets a pg_cron minutes with the undeliverable backlog. The fix is log retention: every instance gets a pg_cron
job (set up on create, upgrade and resume) that deletes analytics logs older than **2 days** every job (set up on create, upgrade and resume) that deletes analytics logs older than **2 days** every
night. That bounds both the db image and vector's backlog, so vector is back to the 512 MiB night. pg_cron in this image only runs jobs from the `postgres` database, so the job lives there
ceiling the production supabase charts carry. and reaches the log tables in `_supabase` over a local dblink connection. That bounds both the
db image and vector's backlog, so vector is back to the 512 MiB ceiling the production supabase
charts carry.
Changing the numbers in this table only affects instances created afterwards. To put them on Changing the numbers in this table only affects instances created afterwards. To put them on
instances that already run, use `sudo supaspin upgrade`: it reads back the values Helm stored when instances that already run, use `sudo supaspin upgrade`: it reads back the values Helm stored when
+1 -1
View File
@@ -1 +1 @@
1.3.2 1.3.3
+25 -9
View File
@@ -39,7 +39,7 @@ import tty
import yaml import yaml
VERSION = "1.3.2" VERSION = "1.3.3"
CONFIG_PATH = os.environ.get("SUPASPIN_CONFIG", "/etc/supaspin/config.yaml") CONFIG_PATH = os.environ.get("SUPASPIN_CONFIG", "/etc/supaspin/config.yaml")
STATE_DIR = os.environ.get("SUPASPIN_STATE", "/var/lib/supaspin") STATE_DIR = os.environ.get("SUPASPIN_STATE", "/var/lib/supaspin")
CHART_REPO = "https://supabase-community.github.io/supabase-kubernetes" CHART_REPO = "https://supabase-community.github.io/supabase-kubernetes"
@@ -88,24 +88,38 @@ DEFAULT_MEMORY = (128, 512)
# Analytics logs: deleted from every instance's _supabase database when older than this. Logflare # Analytics logs: deleted from every instance's _supabase database when older than this. Logflare
# stores logs in _analytics.log_events_* tables and never deletes them; without this job the # stores logs in _analytics.log_events_* tables and never deletes them; without this job the
# capped db image fills up and the database dies (instance 3, 2026-10-06). # capped db image fills up and the database dies (instance 3, 2026-10-06).
# pg_cron in the supabase image only runs jobs from the database named in cron.database_name,
# which the image pins to `postgres`, while the logs live in `_supabase`. The job therefore runs
# in `postgres` and reaches the logs over a local dblink connection, which needs no password: the
# image trusts local socket connections for supabase_admin.
LOG_RETENTION = "2 days" LOG_RETENTION = "2 days"
LOG_RETENTION_JOB = "supaspin-log-retention" LOG_RETENTION_JOB = "supaspin-log-retention"
RETENTION_SQL = f""" RETENTION_SQL = f"""
create extension if not exists pg_cron; create extension if not exists pg_cron;
create or replace function _analytics.delete_old_logs() returns void language plpgsql as $fn$ create extension if not exists dblink;
create or replace function supaspin_delete_old_logs() returns void
language plpgsql as $fn$
declare declare
conn text := 'supaspin_lr_' || substr(md5(clock_timestamp()::text), 1, 8);
t text; t text;
begin begin
perform dblink_connect(conn, 'dbname=_supabase user=supabase_admin');
for t in for t in
select tablename from pg_tables select tablename from dblink(conn,
where schemaname = '_analytics' and tablename like 'log_events\\_%' 'select tablename from pg_tables where schemaname = ''_analytics'' and tablename like ''log_events\\_%''')
as log_tables(tablename text)
loop loop
execute format('delete from _analytics.%I where timestamp < now() - interval ''{LOG_RETENTION}''', t); perform dblink_exec(conn,
format('delete from _analytics.%I where timestamp < now() - interval ''{LOG_RETENTION}''', t));
end loop; end loop;
perform dblink_disconnect(conn);
exception when others then
begin perform dblink_disconnect(conn); exception when others then null; end;
raise;
end $fn$; end $fn$;
select _analytics.delete_old_logs(); select supaspin_delete_old_logs();
delete from cron.job where jobname = '{LOG_RETENTION_JOB}'; delete from cron.job where jobname = '{LOG_RETENTION_JOB}';
select cron.schedule('{LOG_RETENTION_JOB}', '17 3 * * *', 'select _analytics.delete_old_logs();'); select cron.schedule('{LOG_RETENTION_JOB}', '17 3 * * *', 'select supaspin_delete_old_logs()');
""".strip() """.strip()
DEFAULT_MAIL = {"host": "smtp.example.com", "port": 587, "user": "", "password": "", DEFAULT_MAIL = {"host": "smtp.example.com", "port": 587, "user": "", "password": "",
@@ -800,8 +814,10 @@ def setup_log_retention(cfg, n, log):
"""Give the instance a pg_cron job that deletes analytics logs older than LOG_RETENTION every """Give the instance a pg_cron job that deletes analytics logs older than LOG_RETENTION every
night. Logflare stores the logs it receives in the _supabase database and never deletes them; night. Logflare stores the logs it receives in the _supabase database and never deletes them;
without this job the capped db image fills up and the database dies (instance 3 on without this job the capped db image fills up and the database dies (instance 3 on
2026-10-06). Runs the first cleanup right away, so an upgrade also frees a full disk. Runs 2026-10-06). The job runs in the postgres database, the only one pg_cron accepts in this
after create, upgrade and resume; scheduling under a fixed job name makes it idempotent.""" image, and deletes in _supabase over a local dblink connection. Runs the first cleanup right
away, so an upgrade also frees a full disk. Runs after create, upgrade and resume;
scheduling under a fixed job name makes it idempotent."""
log(f" log retention: deleting analytics logs older than {LOG_RETENTION}, every night") log(f" log retention: deleting analytics logs older than {LOG_RETENTION}, every night")
kubectl("-n", namespace(n), "exec", f"{fullname(n)}-db-0", "-c", "supabase-db", "--", kubectl("-n", namespace(n), "exec", f"{fullname(n)}-db-0", "-c", "supabase-db", "--",
"psql", "-U", "supabase_admin", "-d", "_supabase", "-v", "ON_ERROR_STOP=1", "psql", "-U", "supabase_admin", "-d", "_supabase", "-v", "ON_ERROR_STOP=1",
+6 -2
View File
@@ -134,11 +134,15 @@ class Values(unittest.TestCase):
def test_log_retention_sql(self): def test_log_retention_sql(self):
# Logflare stores logs in _analytics.log_events_* and never deletes them; the job deletes # Logflare stores logs in _analytics.log_events_* and never deletes them; the job deletes
# rows older than LOG_RETENTION and is scheduled under a fixed name so re-runs overwrite. # rows older than LOG_RETENTION and is scheduled under a fixed name so re-runs overwrite.
self.assertIn("schemaname = '_analytics'", sp.RETENTION_SQL) # pg_cron in the supabase image only runs jobs from the `postgres` database, so the job
self.assertIn("like 'log_events\\_%'", sp.RETENTION_SQL) # reaches the log tables in `_supabase` over a local dblink connection (trusted there).
self.assertIn("schemaname = ''_analytics''", sp.RETENTION_SQL)
self.assertIn("''log_events\\_%''", sp.RETENTION_SQL)
self.assertIn(f"interval ''{sp.LOG_RETENTION}''", sp.RETENTION_SQL) self.assertIn(f"interval ''{sp.LOG_RETENTION}''", sp.RETENTION_SQL)
self.assertIn(f"'{sp.LOG_RETENTION_JOB}'", sp.RETENTION_SQL) self.assertIn(f"'{sp.LOG_RETENTION_JOB}'", sp.RETENTION_SQL)
self.assertIn("cron.schedule", sp.RETENTION_SQL) self.assertIn("cron.schedule", sp.RETENTION_SQL)
self.assertIn("dblink_connect", sp.RETENTION_SQL)
self.assertIn("user=supabase_admin", sp.RETENTION_SQL)
def test_imgproxy_reads_the_storage_volume(self): def test_imgproxy_reads_the_storage_volume(self):
img = self.v["persistence"]["imgproxy"] img = self.v["persistence"]["imgproxy"]