diff --git a/news/inn-current/Makefile b/news/inn-current/Makefile index 007f6f54bd5a..565b417897a2 100644 --- a/news/inn-current/Makefile +++ b/news/inn-current/Makefile @@ -1,22 +1,22 @@ PORTNAME= inn-current PORTVERSION= ${SNAPSHOT} -PORTREVISION= 6 +PORTREVISION= 0 MASTER_SITE_SUBDIR= inn/snapshots DISTNAME= inn-${BRANCH}-${SNAPSHOT} BRANCH= 2.8 -SNAPSHOT= 20260419 +SNAPSHOT= 20260527 CONFLICTS= inn-2.[0-9]* CPE_PRODUCT= inn INNLIB_LONG= 9.0.2 LIBVER_LONG= 3.0.9 LIBSTORAGEVER_LONG= 3.1.3 MASTERDIR= ${.CURDIR}/../inn PKGDIR= ${.CURDIR} DISTINFO_FILE= ${.CURDIR}/distinfo .include "${MASTERDIR}/Makefile" diff --git a/news/inn-current/distinfo b/news/inn-current/distinfo index 6750ff7add57..e91f36666838 100644 --- a/news/inn-current/distinfo +++ b/news/inn-current/distinfo @@ -1,3 +1,3 @@ -TIMESTAMP = 1778145188 -SHA256 (inn-2.8-20260419.tar.gz) = 6c5ac86d213445f656d50483e68adfeaf2a9cf35bece5c90acfc7405a0d7df4d -SIZE (inn-2.8-20260419.tar.gz) = 2835406 +TIMESTAMP = 1780115559 +SHA256 (inn-2.8-20260527.tar.gz) = 2f73171eeb8b3412bf41199ea37beee2e1e5ec3bbb6bd2bdf879cd05bf66099e +SIZE (inn-2.8-20260527.tar.gz) = 2897821 diff --git a/news/inn-current/files/gitignore b/news/inn-current/files/gitignore deleted file mode 100644 index 4fc5e98b04a9..000000000000 --- a/news/inn-current/files/gitignore +++ /dev/null @@ -1,252 +0,0 @@ -# Ignore list for Git. -*- conf -*- - -/config.cache -/config.log -/config.status -/config.status.lineno -/configure -/configure.lineno -/HACKING -/inn-* -/INSTALL -/libtool -/LIST.* -/Makefile.global -/NEWS -/README -/README.* -/snapshot.log -/authprogs/auth_krb5 -/authprogs/ckpasswd -/authprogs/domain -/authprogs/ident -/authprogs/radius -/autom4te.cache/ -/backends/actmerge -/backends/actsync -/backends/actsyncd -/backends/archive -/backends/batcher -/backends/buffchan -/backends/cvtbatch -/backends/delayer -/backends/innbind -/backends/inndf -/backends/innxbatch -/backends/innxmit -/backends/mod-active -/backends/news2mail -/backends/ninpaths -/backends/nntpget -/backends/nntpsend -/backends/overchan -/backends/send-ihave -/backends/send-nntp -/backends/send-uucp -/backends/sendinpaths -/backends/sendxbatches -/backends/shlock -/backends/shrinkfile -/contrib/analyze-traffic -/contrib/archivegz -/contrib/auth_pass -/contrib/authmysql -/contrib/backlogstat -/contrib/cleannewsgroups -/contrib/count_overview -/contrib/expirectl -/contrib/findreadgroups -/contrib/fixhist -/contrib/innconfcheck -/contrib/makeexpctl -/contrib/makestorconf -/contrib/mkbuf -/contrib/mlockfile -/contrib/newsresp -/contrib/nnrp.access2readers.conf -/contrib/ovsqlite-dump -/contrib/ovsqlite-undump -/contrib/pullart -/contrib/reset-cnfs -/contrib/respool -/contrib/stathist -/contrib/thdexpire -/contrib/tunefeed -/control/controlbatch -/control/controlchan -/control/docheckgroups -/control/gencancel -/control/perl-nocem -/control/pgpverify -/doc/checklist -/doc/external-auth -/doc/hook-perl -/doc/hook-python -# Assume we no longer add rough man pages without POD, -# so that there is no need to exclude existing man pages -# waiting to be converted to POD. -/doc/man/*.1 -/doc/man/*.3 -/doc/man/*.3pm -/doc/man/*.5 -/doc/man/*.8 -/expire/convdate -/expire/expire -/expire/expireover -/expire/expirerm -/expire/fastrm -/expire/grephistory -/expire/makedbz -/expire/makehistory -/expire/prunehistory -/frontends/bunbatch -/frontends/c7unbatch -/frontends/cnfsheadconf -/frontends/cnfsstat -/frontends/ctlinnd -/frontends/decode -/frontends/encode -/frontends/feedone -/frontends/getlist -/frontends/gunbatch -/frontends/inews -/frontends/innconfval -/frontends/mailpost -/frontends/ovdb_init -/frontends/ovdb_monitor -/frontends/ovdb_server -/frontends/ovdb_stat -/frontends/pullnews -/frontends/rnews -/frontends/scanspool -/frontends/sm -/frontends/sys2nf -/history/Make.methods -/history/buildconfig -/history/hismethods.c -/history/hismethods.h -/include/config.h -/include/config.h.in -/include/inn/paths.h -/include/inn/portable-getaddrinfo.h -/include/inn/portable-getnameinfo.h -/include/inn/portable-macros.h -/include/inn/portable-socket.h -/include/inn/portable-stdbool.h -/include/inn/system.h -/include/inn/version.h -/innd/innd -/innd/tinyleaf -/innfeed/config_l.c -/innfeed/config_y.c -/innfeed/config_y.h -/innfeed/imapfeed -/innfeed/innfeed -/innfeed/procbatch -/nnrpd/nnrpd -/perl/INN/Config.pm -/perl/INN/Utils/Shlock.pm -/samples/buffindexed.conf -/samples/inn.conf -/samples/innreport.conf -/samples/newsfeeds -/samples/nnrpd_access.pl -/samples/nnrpd_auth.pl -/scripts/inncheck -/scripts/innmail -/scripts/innreport -/scripts/innshellvars -/scripts/innshellvars.pl -/scripts/innshellvars.tcl -/scripts/innstat -/scripts/innupgrade -/scripts/innwatch -/scripts/news.daily -/scripts/rc.news -/scripts/scanlogs -/scripts/simpleftp -/scripts/tally.control -/scripts/writelog -/site/* -!/site/Makefile -!/site/getsafe.sh -/storage/Make.methods -/storage/buildconfig -/storage/methods.c -/storage/methods.h -/storage/ovmethods.c -/storage/ovmethods.h -/storage/buffindexed/buffindexed_d -/storage/ovsqlite/ovsqlite-server -/storage/ovsqlite/ovsqlite-util -/storage/ovsqlite/sql-init.c -/storage/ovsqlite/sql-init.h -/storage/ovsqlite/sql-main.c -/storage/ovsqlite/sql-main.h -/storage/ovsqlite/sqlite-helper-gen -/storage/tradindexed/tdx-util -/support/fixconfig -/support/fixscript -/tests/runtests -/tests/authprogs/ident.t -/tests/clients/server-list -/tests/docs/pod.t -/tests/innd/artparse.t -/tests/innd/chan.t -/tests/lib/artnumber.t -/tests/lib/asprintf.t -/tests/lib/buffer.t -/tests/lib/canlock.t -/tests/lib/concat.t -/tests/lib/conffile.t -/tests/lib/confparse.t -/tests/lib/daemon.t -/tests/lib/date.t -/tests/lib/dispatch.t -/tests/lib/fdflag.t -/tests/lib/getaddrinfo.t -/tests/lib/getnameinfo.t -/tests/lib/hash.t -/tests/lib/hashtab.t -/tests/lib/headers.t -/tests/lib/hex.t -/tests/lib/inet_aton.t -/tests/lib/inet_ntoa.t -/tests/lib/inet_ntop.t -/tests/lib/innconf.t -/tests/lib/list.t -/tests/lib/md5.t -/tests/lib/messageid.t -/tests/lib/messages.t -/tests/lib/mkstemp.t -/tests/lib/pread.t -/tests/lib/pwrite.t -/tests/lib/qio.t -/tests/lib/reallocarray.t -/tests/lib/setenv.t -/tests/lib/snprintf.t -/tests/lib/strlcat.t -/tests/lib/strlcpy.t -/tests/lib/tst.t -/tests/lib/uwildmat.t -/tests/lib/vector.t -/tests/lib/wire.t -/tests/lib/xmalloc -/tests/lib/xwrite.t -/tests/lib/network/addr-ipv4.t -/tests/lib/network/addr-ipv6.t -/tests/lib/network/client.t -/tests/lib/network/server.t -/tests/nnrpd/auth-ext.t -/tests/overview/api.t -/tests/overview/buffindexed.t -/tests/overview/tradindexed.t -/tests/overview/xref.t -/tests/perl/minimum-version.t -/tests/util/innbind.t -**/.libs/ -**/.pure/ -*.a -*.la -*.lo -*.o diff --git a/news/inn-current/files/pr337.patch b/news/inn-current/files/pr337.patch deleted file mode 100644 index 19d84e1d7897..000000000000 --- a/news/inn-current/files/pr337.patch +++ /dev/null @@ -1,87 +0,0 @@ -From 93a36682f7ba9d013d07859882c2b8c501da249e Mon Sep 17 00:00:00 2001 -From: Kevin Bowling -Date: Wed, 29 Apr 2026 13:54:21 -0700 -Subject: [PATCH] ovsqlite: add mmapsize configuration - ---- - doc/pod/ovsqlite.pod | 11 +++++++++++ - samples/ovsqlite.conf | 7 +++++++ - storage/ovsqlite/ovsqlite-server.c | 11 +++++++++++ - 3 files changed, 29 insertions(+) - -diff --git doc/pod/ovsqlite.pod doc/pod/ovsqlite.pod -index 96a6b3165..3090d5bcb 100644 ---- doc/pod/ovsqlite.pod -+++ doc/pod/ovsqlite.pod -@@ -67,6 +67,17 @@ is consulted only when creating a new database. Enabling compression - saves about S<55 %> of disk space on standard overview data. The default - value is false. - -+=item I -+ -+The maximum number of bytes of the database file that SQLite will -+memory-map. When set, SQLite reads data directly from the operating -+system's page cache via mmap instead of copying it into its own page -+cache, which can significantly improve read performance for large -+databases. A good starting value is the size of the database file -+or the amount of available memory, whichever is smaller. For example, -+a value of C<4294967296> allows up to S<4 GB> to be memory-mapped. -+The default value of 0 disables memory-mapped I/O. -+ - =item I - - The SQLite database page size in bytes. Must be a power of 2, minimum 512, -diff --git samples/ovsqlite.conf samples/ovsqlite.conf -index cba4f7277..a2e2e204f 100644 ---- samples/ovsqlite.conf -+++ samples/ovsqlite.conf -@@ -10,6 +10,13 @@ - # The default value is false. - #compress: false - -+# The maximum number of bytes of the database file that SQLite will -+# memory-map. When set, SQLite reads data via mmap instead of copying -+# into its own page cache, which can improve read performance for large -+# databases. A good value is the database file size or available memory, -+# whichever is smaller. Set to 0 (the default) to disable. -+#mmapsize: 0 -+ - # The SQLite database page size in bytes. - # Must be a power of 2, minimum 512, maximum 65536. - # Appropriate values include the virtual memory page size and the -diff --git storage/ovsqlite/ovsqlite-server.c storage/ovsqlite/ovsqlite-server.c -index f04df77e7..afc20e46b 100644 ---- storage/ovsqlite/ovsqlite-server.c -+++ storage/ovsqlite/ovsqlite-server.c -@@ -119,6 +119,7 @@ static sql_main_t sql_main; - static bool use_compression; - static unsigned long pagesize; - static unsigned long cachesize; -+static unsigned long mmapsize; - static struct timeval transaction_time_limit = {10, 0}; - static unsigned long transaction_row_limit = 10000; - -@@ -386,6 +387,7 @@ load_config(void) - config_param_boolean(top, "compress", &use_compression); - config_param_unsigned_number(top, "pagesize", &pagesize); - config_param_unsigned_number(top, "cachesize", &cachesize); -+ config_param_unsigned_number(top, "mmapsize", &mmapsize); - if (config_param_real(top, "transtimelimit", &timelimit)) { - transaction_time_limit.tv_sec = (long) timelimit; - transaction_time_limit.tv_usec = -@@ -656,6 +658,15 @@ open_db(void) - sqlite3_free(errmsg); - } - } -+ if (mmapsize) { -+ snprintf(sqltext, sizeof sqltext, "pragma mmap_size = %lu;", -+ mmapsize); -+ status = sqlite3_exec(connection, sqltext, 0, NULL, &errmsg); -+ if (status != SQLITE_OK) { -+ warn("cannot set mmap size: %s", errmsg); -+ sqlite3_free(errmsg); -+ } -+ } - } - - static void diff --git a/news/inn-current/files/pr338.patch b/news/inn-current/files/pr338.patch deleted file mode 100644 index 5ff4a03632e4..000000000000 --- a/news/inn-current/files/pr338.patch +++ /dev/null @@ -1,2883 +0,0 @@ -From cf4a8ecac2f3149505f9a55cdd95383e9b3f725b Mon Sep 17 00:00:00 2001 -From: Kevin Bowling -Date: Wed, 29 Apr 2026 13:47:42 -0700 -Subject: [PATCH 1/7] ovsqlite: Add WAL mode configuration - ---- - doc/pod/ovsqlite.pod | 21 ++++++++++++++++++++- - samples/ovsqlite.conf | 10 ++++++++++ - storage/ovsqlite/ovsqlite-server.c | 21 +++++++++++++++++++-- - storage/ovsqlite/sql-main.sql | 3 +++ - 4 files changed, 52 insertions(+), 3 deletions(-) - -diff --git doc/pod/ovsqlite.pod doc/pod/ovsqlite.pod -index 96a6b3165..797e2c6a8 100644 ---- doc/pod/ovsqlite.pod -+++ doc/pod/ovsqlite.pod -@@ -39,7 +39,8 @@ Plan on needing at least S<0.93 KB> for every article in your spool - need at least S<4.65 GB> of disk space for ovsqlite. With compression - enabled, this estimate changes to S<0.4 KB> per article, so you'll - need at least S<2 GB> of disk space for 5 million articles. Plus, --you'll need additional space for transaction logs (a few MB). -+you'll need additional space for transaction logs (a few MB, or up to -+a few hundred MB temporarily with I enabled). - - =head1 CONFIGURATION - -@@ -67,6 +68,24 @@ is consulted only when creating a new database. Enabling compression - saves about S<55 %> of disk space on standard overview data. The default - value is false. - -+=item I -+ -+If this parameter is true, ovsqlite will use SQLite's WAL (Write-Ahead -+Logging) journal mode instead of the default PERSIST mode. WAL mode -+allows concurrent readers and writers without blocking, which significantly -+improves read performance when multiple B processes access the -+overview database simultaneously. It also sets C to -+C, which is safe with WAL and reduces unnecessary fsync overhead. -+ -+On shutdown, the WAL is checkpointed back into the main database file -+and truncated. -+ -+B WAL mode requires shared-memory support and does not work on -+network filesystems such as NFS. Do not enable this parameter if -+I is on a network filesystem. -+ -+The default value is false. -+ - =item I - - The SQLite database page size in bytes. Must be a power of 2, minimum 512, -diff --git samples/ovsqlite.conf samples/ovsqlite.conf -index cba4f7277..6912e8bf2 100644 ---- samples/ovsqlite.conf -+++ samples/ovsqlite.conf -@@ -10,6 +10,16 @@ - # The default value is false. - #compress: false - -+# WAL mode: if true, use SQLite's Write-Ahead Logging journal mode -+# instead of the default PERSIST mode. WAL mode allows concurrent -+# readers and writers without blocking, improving read performance -+# when multiple nnrpd processes access the overview database -+# simultaneously. Also sets synchronous to NORMAL, which is safe -+# with WAL and reduces unnecessary fsync overhead. -+# Note: WAL mode does not work on network filesystems such as NFS. -+# The default value is false. -+#walmode: false -+ - # The SQLite database page size in bytes. - # Must be a power of 2, minimum 512, maximum 65536. - # Appropriate values include the virtual memory page size and the -diff --git storage/ovsqlite/ovsqlite-server.c storage/ovsqlite/ovsqlite-server.c -index f04df77e7..5bc4aa8d9 100644 ---- storage/ovsqlite/ovsqlite-server.c -+++ storage/ovsqlite/ovsqlite-server.c -@@ -117,6 +117,7 @@ static sqlite3 *connection; - static sql_main_t sql_main; - - static bool use_compression; -+static bool use_wal; - static unsigned long pagesize; - static unsigned long cachesize; - static struct timeval transaction_time_limit = {10, 0}; -@@ -384,6 +385,7 @@ load_config(void) - free(path); - if (top) { - config_param_boolean(top, "compress", &use_compression); -+ config_param_boolean(top, "walmode", &use_wal); - config_param_unsigned_number(top, "pagesize", &pagesize); - config_param_unsigned_number(top, "cachesize", &cachesize); - if (config_param_real(top, "transtimelimit", &timelimit)) { -@@ -656,13 +658,28 @@ open_db(void) - sqlite3_free(errmsg); - } - } -+ if (use_wal) { -+ status = sqlite3_exec(connection, -+ "pragma journal_mode = 'WAL';" -+ "pragma synchronous = 'NORMAL';", -+ 0, NULL, &errmsg); -+ if (status != SQLITE_OK) { -+ warn("cannot enable WAL mode: %s", errmsg); -+ sqlite3_free(errmsg); -+ } -+ } - } - - static void - close_db(void) - { -- sqlite3_step(sql_main.delete_journal); -- sqlite3_reset(sql_main.delete_journal); -+ if (use_wal) { -+ sqlite3_step(sql_main.checkpoint_wal); -+ sqlite3_reset(sql_main.checkpoint_wal); -+ } else { -+ sqlite3_step(sql_main.delete_journal); -+ sqlite3_reset(sql_main.delete_journal); -+ } - sqlite_helper_term(&sql_main_helper, (sqlite3_stmt **) &sql_main); - sqlite3_close_v2(connection); - connection = NULL; -diff --git storage/ovsqlite/sql-main.sql storage/ovsqlite/sql-main.sql -index ff73f1b0e..ab5495a49 100644 ---- storage/ovsqlite/sql-main.sql -+++ storage/ovsqlite/sql-main.sql -@@ -40,6 +40,9 @@ rollback to savepoint article_group; - -- .delete_journal - pragma journal_mode = 'DELETE'; - -+-- .checkpoint_wal -+pragma wal_checkpoint(TRUNCATE); -+ - -- .add_group - insert into groupinfo (groupname, flag_alias, low, high) - values(?1, ?2, ?3, ?4); - -From 0d84255b417c89219225f0a034fd4c068d9749a0 Mon Sep 17 00:00:00 2001 -From: Kevin Bowling -Date: Sat, 2 May 2026 03:36:19 -0700 -Subject: [PATCH 2/7] ovsqlite: Add direct reader mode for nnrpd with WAL - -When WAL mode is enabled, nnrpd processes now open the overview database -directly with SQLITE_OPEN_READONLY instead of routing reads through -ovsqlite-server. This eliminates the IPC round-trip and the -single-threaded server bottleneck for read operations, allowing read -performance to scale with the number of concurrent readers. - -Direct reader mode activates automatically when walmode is true in -ovsqlite.conf AND the database is actually in WAL mode (verified via -pragma journal_mode). Without WAL, behavior is unchanged -- readers -connect to ovsqlite-server as before to avoid EXCLUSIVE lock contention. - -Key changes: -- ovsqlite.c: direct_open() opens the database read-only with schema - version validation, journal mode verification, decompression support, - and goto-based cleanup. Read functions (groupstats, getartinfo, - search) branch on direct_reader flag. Write operations return errors - in direct reader mode. Falls back to server path gracefully. -- ovsqlite-server.c: WAL checkpoint at shutdown uses PASSIVE first - (non-blocking, safe with readers connected) then TRUNCATE only if no - readers hold pins. Prevents server shutdown from hanging. -- ovsqlite.c: Fix pre-existing buffer leak in ovsqlite_close(): - request and response buffers from server_connect() were never freed. -- sql-read.sql: read-only prepared statements with busy_timeout=10000 - and query_only=1. -- New readercachesize config parameter (default 8 MB per reader). - -Includes unit tests (WAL gating, write rejection) and integration tests -that start a real ovsqlite-server, write compressed data through it, -kill the server, and verify the direct reader can read it back correctly -including a server restart cycle. ---- - .gitignore | 5 + - MANIFEST | 7 + - Makefile.global.in | 11 +- - doc/pod/ovsqlite.pod | 36 +- - samples/ovsqlite.conf | 14 +- - storage/Makefile | 4 + - storage/ovsqlite/ovmethod.config | 6 +- - storage/ovsqlite/ovmethod.mk | 7 +- - storage/ovsqlite/ovsqlite-server.c | 22 +- - storage/ovsqlite/ovsqlite.c | 760 +++++++++++++++++++++++++++-- - storage/ovsqlite/sql-read.sql | 40 ++ - support/mkmanifest | 3 + - tests/Makefile | 14 +- - tests/TESTS | 2 + - tests/overview/ovsqlite-integ.t | 215 ++++++++ - tests/overview/ovsqlite-read-t.c | 424 ++++++++++++++++ - tests/overview/ovsqlite-t.c | 186 +++++++ - tests/overview/ovsqlite-write-t.c | 144 ++++++ - 18 files changed, 1840 insertions(+), 60 deletions(-) - create mode 100644 storage/ovsqlite/sql-read.sql - create mode 100755 tests/overview/ovsqlite-integ.t - create mode 100644 tests/overview/ovsqlite-read-t.c - create mode 100644 tests/overview/ovsqlite-t.c - create mode 100644 tests/overview/ovsqlite-write-t.c - -diff --git .gitignore .gitignore -index 4fc5e98b0..abce0ef2b 100644 ---- .gitignore -+++ .gitignore -@@ -183,6 +183,8 @@ - /storage/ovsqlite/sql-init.h - /storage/ovsqlite/sql-main.c - /storage/ovsqlite/sql-main.h -+/storage/ovsqlite/sql-read.c -+/storage/ovsqlite/sql-read.h - /storage/ovsqlite/sqlite-helper-gen - /storage/tradindexed/tdx-util - /support/fixconfig -@@ -240,6 +242,9 @@ - /tests/nnrpd/auth-ext.t - /tests/overview/api.t - /tests/overview/buffindexed.t -+/tests/overview/ovsqlite.t -+/tests/overview/ovsqlite-read.t -+/tests/overview/ovsqlite-write.t - /tests/overview/tradindexed.t - /tests/overview/xref.t - /tests/perl/minimum-version.t -diff --git MANIFEST MANIFEST -index 094b0a020..39889beb6 100644 ---- MANIFEST -+++ MANIFEST -@@ -791,6 +791,9 @@ storage/ovsqlite/sql-init.sql SQLite code for database setup - storage/ovsqlite/sql-main.c Generated daily operation implementation - storage/ovsqlite/sql-main.h Generated daily operation interface - storage/ovsqlite/sql-main.sql SQLite code for daily operation -+storage/ovsqlite/sql-read.c Generated read-only query implementation -+storage/ovsqlite/sql-read.h Generated read-only query interface -+storage/ovsqlite/sql-read.sql SQLite code for direct reader queries - storage/ovsqlite/sqlite-helper-gen.in Package SQLite code for convenient use - storage/ovsqlite/sqlite-helper.c SQLite code package implementation - storage/ovsqlite/sqlite-helper.h SQLite code package interface -@@ -991,6 +994,10 @@ tests/overview Test suite for overview (Directory) - tests/overview/api-t.c Basic tests for overview API - tests/overview/overchan.t Tests for backends/overchan - tests/overview/overview-t.c Basic tests for overview methods -+tests/overview/ovsqlite-integ.t Integration test for ovsqlite direct reader -+tests/overview/ovsqlite-read-t.c Direct reader verification for integration test -+tests/overview/ovsqlite-t.c Unit tests for ovsqlite direct reader -+tests/overview/ovsqlite-write-t.c Writer helper for ovsqlite integration test - tests/overview/xref-t.c Test storing overview data by Xref - tests/perl Test suite for Perl scripts (Directory) - tests/perl/minimum-version.t.in Tests for not too-new features of Perl -diff --git Makefile.global.in Makefile.global.in -index d4dbe5de4..a14283d5d 100644 ---- Makefile.global.in -+++ Makefile.global.in -@@ -87,11 +87,12 @@ BDB_LDFLAGS = @BDB_LDFLAGS@ $(ZLIB_LDFLAGS) - BDB_LIBS = @BDB_LIBS@ $(ZLIB_LIBS) - - ## SQLite support. Additional flags and libraries used when compiling or --## linking code that contains SQLite support. -+## linking code that contains SQLite support. zlib is bundled in because -+## ovsqlite direct readers need decompression support. - --SQLITE3_CPPFLAGS = @SQLITE3_CPPFLAGS@ --SQLITE3_LDFLAGS = @SQLITE3_LDFLAGS@ --SQLITE3_LIBS = @SQLITE3_LIBS@ -+SQLITE3_CPPFLAGS = @SQLITE3_CPPFLAGS@ $(ZLIB_CPPFLAGS) -+SQLITE3_LDFLAGS = @SQLITE3_LDFLAGS@ $(ZLIB_LDFLAGS) -+SQLITE3_LIBS = @SQLITE3_LIBS@ $(ZLIB_LIBS) - - ## INN libraries. Nearly all INN programs are linked with libinn, and any - ## INN program that reads from or writes to article storage or overview is -@@ -101,7 +102,7 @@ SQLITE3_LIBS = @SQLITE3_LIBS@ - LIBINN = $(abs_builddir)/lib/libinn$(LIBSUFFIX).$(EXTLIB) - LIBHIST = $(abs_builddir)/history/libinnhist$(LIBSUFFIX).$(EXTLIB) - LIBSTORAGE = $(abs_builddir)/storage/libinnstorage$(LIBSUFFIX).$(EXTLIB) --STORAGE_LIBS = $(BDB_LDFLAGS) $(BDB_LIBS) -+STORAGE_LIBS = $(BDB_LDFLAGS) $(BDB_LIBS) $(SQLITE3_LDFLAGS) $(SQLITE3_LIBS) - - DBM_CPPFLAGS = @DBM_CPPFLAGS@ - DBM_LIBS = @DBM_LIBS@ -diff --git doc/pod/ovsqlite.pod doc/pod/ovsqlite.pod -index 797e2c6a8..8d1a1957e 100644 ---- doc/pod/ovsqlite.pod -+++ doc/pod/ovsqlite.pod -@@ -57,8 +57,9 @@ configuration file: - - =item I - --The SQLite in-memory page cache size in kilobytes. The default value is --left up to the SQLite library and seems to be stable at S<2000 KB>. -+The SQLite in-memory page cache size in kilobytes for the B -+write daemon. The default value is left up to the SQLite library and seems -+to be stable at S<2000 KB>. - - =item I - -@@ -86,6 +87,22 @@ I is on a network filesystem. - - The default value is false. - -+=item I -+ -+The SQLite in-memory page cache size in kilobytes for each B reader -+process. When I is enabled, B processes open the overview -+database directly for read-only access, bypassing B. -+This eliminates the IPC overhead and allows reads to scale with the number -+of reader processes. Each B process gets its own cache, so this -+value should typically be smaller than I. The operating system's -+file cache (or ZFS ARC) serves as a shared second-level cache across all -+readers. -+ -+Direct reader mode is only activated when I is enabled. Without -+WAL, writers take exclusive locks that would block direct readers during -+commits, so the server path is used instead. The default value -+is S<8000 KB> (about S<8 MB>). -+ - =item I - - The SQLite database page size in bytes. Must be a power of 2, minimum 512, -@@ -121,10 +138,17 @@ change much on fast systems. - - =head1 RUNNING - --All overview database access goes through the B daemon. --For ordinary operation, B will start and stop it automatically. --If you want to touch the overview database while B isn't running, you'll --have to start B manually first. See ovsqlite-server(8). -+Write operations (article ingest, expiration) go through the -+B daemon. For ordinary operation, B will start -+and stop it automatically. If you want to touch the overview database while -+B isn't running, you'll have to start B manually -+first. See ovsqlite-server(8). -+ -+When I is enabled, B reader processes open the overview -+database directly with read-only access, bypassing B -+for read operations. This eliminates the IPC round-trip and server -+serialization bottleneck, allowing read performance to scale with the -+number of concurrent readers. - - =head1 HISTORY - -diff --git samples/ovsqlite.conf samples/ovsqlite.conf -index 6912e8bf2..319b6d656 100644 ---- samples/ovsqlite.conf -+++ samples/ovsqlite.conf -@@ -29,11 +29,23 @@ - # between versions. - #pagesize: 4096 - --# The SQLite in-memory page cache size in kilobytes. -+# The SQLite in-memory page cache size in kilobytes for the -+# ovsqlite-server write daemon. - # The default value is left up to the SQLite library and seems to be - # stable at 2000 KB. - #cachesize: 2000 - -+# The SQLite in-memory page cache size in kilobytes for each nnrpd -+# reader process. When WAL mode is enabled, nnrpd processes open the -+# database directly for read-only access, bypassing ovsqlite-server. -+# Each nnrpd process gets its own cache, so this value should be -+# smaller than cachesize. The operating system's file cache (or ZFS -+# ARC) serves as a shared second-level cache across all readers. -+# The mmapsize parameter in this file can also be used to share -+# physical memory pages across reader processes. -+# The default value is 8000 KB (about 8 MB). -+#readercachesize: 8000 -+ - # The maximum number of article rows that can be inserted or deleted - # in a single SQL transaction. - # The default value is 10000 articles. -diff --git storage/Makefile storage/Makefile -index fa53c0007..64d9e9608 100644 ---- storage/Makefile -+++ storage/Makefile -@@ -60,6 +60,10 @@ maintclean: distclean - rm -f Make.methods methods.c methods.h ovmethods.c ovmethods.h - rm -f $(RM_MAINTCLEAN) - -+# ovsqlite.c includes sql-read.h which is generated from sql-read.sql. -+# Ensure the header exists before compiling any objects. -+$(OBJECTS) $(LOBJECTS): ovsqlite/sql-read.h -+ - libinnstorage.la: $(OBJECTS) $(LOBJECTS) $(LIBINN) - $(LIBLD) $(LDFLAGS) -o $@ $(LOBJECTS) \ - $(LIBINN) $(STORAGE_LIBS) $(LIBS) \ -diff --git storage/ovsqlite/ovmethod.config storage/ovsqlite/ovmethod.config -index 57b932ebb..5ba6f5612 100644 ---- storage/ovsqlite/ovmethod.config -+++ storage/ovsqlite/ovmethod.config -@@ -1,7 +1,7 @@ - name = ovsqlite - number = 5 --sources = ovsqlite.c ovsqlite-private.c --extra-sources = ovsqlite-server.c sql-main.c sql-init.c sqlite-helper.c -+sources = ovsqlite.c ovsqlite-private.c sqlite-helper.c sql-read.c -+extra-sources = ovsqlite-server.c sql-main.c sql-init.c - programs = ovsqlite-server ovsqlite-util - clean = sqlite-helper-gen --maintclean = sql-init.c sql-init.h sql-main.c sql-main.h -+maintclean = sql-init.c sql-init.h sql-main.c sql-main.h sql-read.c sql-read.h -diff --git storage/ovsqlite/ovmethod.mk storage/ovsqlite/ovmethod.mk -index 009d3d259..6fc0f0044 100644 ---- storage/ovsqlite/ovmethod.mk -+++ storage/ovsqlite/ovmethod.mk -@@ -1,5 +1,5 @@ - OVSQLITEOBJECTS = ovsqlite/ovsqlite-server.o ovsqlite/sql-main.o \ -- ovsqlite/sql-init.o ovsqlite/sqlite-helper.o \ -+ ovsqlite/sql-init.o \ - ovsqlite/ovsqlite-private.o - OVSQLITELOBJECTS = $(OVSQLITEOBJECTS:.o=.lo) - -@@ -24,3 +24,8 @@ ovsqlite/sql-init.c: ovsqlite/sql-init.sql ovsqlite/sqlite-helper-gen - - ovsqlite/sql-init.h: ovsqlite/sql-init.c ; - -+ovsqlite/sql-read.c: ovsqlite/sql-read.sql ovsqlite/sqlite-helper-gen -+ ovsqlite/sqlite-helper-gen ovsqlite/sql-read.sql -+ -+ovsqlite/sql-read.h: ovsqlite/sql-read.c ; -+ -diff --git storage/ovsqlite/ovsqlite-server.c storage/ovsqlite/ovsqlite-server.c -index 5bc4aa8d9..dc78fbb33 100644 ---- storage/ovsqlite/ovsqlite-server.c -+++ storage/ovsqlite/ovsqlite-server.c -@@ -674,8 +674,26 @@ static void - close_db(void) - { - if (use_wal) { -- sqlite3_step(sql_main.checkpoint_wal); -- sqlite3_reset(sql_main.checkpoint_wal); -+ int wal_frames, checkpointed; -+ int rc; -+ -+ /* Use PASSIVE checkpoint first — it never blocks and is safe even -+ * when direct reader processes (nnrpd) still have the database open. -+ * The prepared statement uses TRUNCATE which would hang waiting for -+ * all readers to disconnect (busy_timeout is ~31 years). */ -+ rc = sqlite3_wal_checkpoint_v2(connection, NULL, -+ SQLITE_CHECKPOINT_PASSIVE, -+ &wal_frames, &checkpointed); -+ if (rc == SQLITE_OK && wal_frames == checkpointed) { -+ /* All frames checkpointed (no readers holding pins). -+ * Safe to truncate the WAL file for a clean shutdown. */ -+ sqlite3_wal_checkpoint_v2(connection, NULL, -+ SQLITE_CHECKPOINT_TRUNCATE, -+ NULL, NULL); -+ } else if (rc == SQLITE_OK && wal_frames > checkpointed) { -+ notice("WAL checkpoint: %d/%d frames checkpointed" -+ " (readers still connected)", checkpointed, wal_frames); -+ } - } else { - sqlite3_step(sql_main.delete_journal); - sqlite3_reset(sql_main.delete_journal); -diff --git storage/ovsqlite/ovsqlite.c storage/ovsqlite/ovsqlite.c -index 81d4811de..5d6621570 100644 ---- storage/ovsqlite/ovsqlite.c -+++ storage/ovsqlite/ovsqlite.c -@@ -6,6 +6,11 @@ - ** - ** Various bug fixes, code and documentation improvements since then - ** in 2021-2024, 2026. -+** -+** Direct reader mode added in 2026 to allow nnrpd processes to read the -+** overview database directly via SQLite, bypassing ovsqlite-server for -+** read-only access. This improves read performance by eliminating the -+** IPC round-trip and server serialization bottleneck. - */ - - #include "ovsqlite.h" -@@ -22,6 +27,7 @@ - # endif - - # include "conffile.h" -+# include "inn/confparse.h" - # include "inn/fdflag.h" - # include "inn/innconf.h" - # include "inn/libinn.h" -@@ -30,6 +36,17 @@ - - # include "../ovinterface.h" - -+# include -+# include "sql-read.h" -+# include "sqlite-helper.h" -+ -+# ifdef HAVE_ZLIB -+# define USE_DICTIONARY 1 -+# include -+# endif -+ -+# define OVSQLITE_DB_FILE "ovsqlite.db" -+ - typedef struct handle_t { - uint8_t buffer[SEARCHSPACE]; - uint64_t low; -@@ -46,6 +63,7 @@ typedef struct handle_t { - char groupname[1]; - } handle_t; - -+/* Server connection state (used for write mode and legacy read mode). */ - static int sock = -1; - static buffer_t *request; - static buffer_t *response; -@@ -54,6 +72,360 @@ static buffer_t *response; - static ovsqlite_port port; - # endif - -+/* Direct reader state (used when mode is OV_READ only). */ -+static bool direct_reader = false; -+static sqlite3 *read_connection = NULL; -+static sql_read_t sql_read; -+static bool reader_use_compression = false; -+ -+# ifdef HAVE_ZLIB -+ -+/* clang-format off */ -+static uint32_t const pack_length_bias[5] = -+{ -+ 0, -+ 0x80, -+ 0x4080, -+ 0x204080, -+ 0x10204080, -+}; -+/* clang-format on */ -+ -+static z_stream reader_inflation; -+static buffer_t *reader_flate; -+ -+# ifdef USE_DICTIONARY -+ -+static char reader_dictionary[0x8000]; -+static unsigned int reader_basedict_len; -+ -+# endif /* USE_DICTIONARY */ -+ -+static uint32_t -+reader_unpack_length(z_stream *stream) -+{ -+ uint8_t *walk; -+ unsigned int c, lenlen, n; -+ uint32_t length; -+ -+ if (stream->avail_in <= 0) -+ return ~0U; -+ walk = stream->next_in; -+ c = *walk++; -+ lenlen = 1; -+ while (c & (1U << (8 - lenlen))) -+ lenlen++; -+ if (lenlen > 5 || lenlen > stream->avail_in) -+ return ~0U; -+ length = c & ~(~0U << (8 - lenlen)); -+ for (n = lenlen - 1; n > 0; n--) -+ length = (length << 8) | *walk++; -+ length += pack_length_bias[lenlen - 1]; -+ stream->next_in = walk; -+ stream->avail_in -= lenlen; -+ return length; -+} -+ -+# ifdef USE_DICTIONARY -+ -+static unsigned int -+reader_make_dict(char const *groupname, int groupname_len, uint64_t artnum) -+{ -+ sqlite3_snprintf(sizeof reader_dictionary - reader_basedict_len, -+ reader_dictionary + reader_basedict_len, -+ "%.*s:%llu\r\n", groupname_len, groupname, artnum); -+ return reader_basedict_len -+ + strlen(reader_dictionary + reader_basedict_len); -+} -+ -+# endif /* USE_DICTIONARY */ -+ -+# endif /* HAVE_ZLIB */ -+ -+ -+/* -+** Decompress an overview blob read directly from SQLite. -+** Returns the decompressed data in reader_flate, or NULL on error. -+** On success, *out_len is set to the decompressed length. -+*/ -+# ifdef HAVE_ZLIB -+static uint8_t * -+reader_decompress(uint8_t const *overview, uint32_t overview_len, -+ char const *groupname, int groupname_len, uint64_t artnum, -+ uint32_t *out_len) -+{ -+ uint32_t raw_len; -+ int status; -+ -+ reader_inflation.next_in = (uint8_t *) overview; -+ reader_inflation.avail_in = overview_len; -+ -+ raw_len = reader_unpack_length(&reader_inflation); -+ if (raw_len > MAX_OVDATA_SIZE) -+ return NULL; -+ if (raw_len > 0) { -+ buffer_resize(reader_flate, raw_len); -+ reader_inflation.next_out = (uint8_t *) reader_flate->data; -+ reader_inflation.avail_out = raw_len; -+ status = inflate(&reader_inflation, Z_FINISH); -+# ifdef USE_DICTIONARY -+ if (status == Z_NEED_DICT) { -+ status = inflateSetDictionary( -+ &reader_inflation, (uint8_t *) reader_dictionary, -+ reader_make_dict(groupname, groupname_len, artnum)); -+ if (status == Z_OK) -+ status = inflate(&reader_inflation, Z_FINISH); -+ } -+# endif -+ reader_flate->left = -+ (char *) reader_inflation.next_out - reader_flate->data; -+ reader_inflation.next_in = NULL; -+ reader_inflation.avail_in = 0; -+ inflateReset(&reader_inflation); -+ if (status != Z_STREAM_END || reader_inflation.avail_out > 0) -+ return NULL; -+ *out_len = reader_flate->left; -+ return (uint8_t *) reader_flate->data; -+ } else { -+ /* Compression didn't save space; data stored uncompressed after -+ * the zero length marker. */ -+ *out_len = reader_inflation.avail_in; -+ return reader_inflation.next_in; -+ } -+} -+# endif /* HAVE_ZLIB */ -+ -+ -+/* -+** Helper callback for pragma queries that return a single text value. -+*/ -+static int -+pragma_callback(void *data, int ncols, char **values, char **names UNUSED) -+{ -+ char **result = data; -+ -+ if (ncols > 0 && values[0]) -+ *result = xstrdup(values[0]); -+ return 0; -+} -+ -+/* -+** Helper to read an integer value from the misc table. -+*/ -+static bool -+read_misc_int(char const *key, int *value) -+{ -+ int status; -+ -+ sqlite3_bind_text(sql_read.getmisc, 1, key, -1, SQLITE_STATIC); -+ status = sqlite3_step(sql_read.getmisc); -+ if (status != SQLITE_ROW) { -+ warn("ovsqlite: cannot read '%s' from database: %s", key, -+ sqlite3_errmsg(read_connection)); -+ sqlite3_reset(sql_read.getmisc); -+ sqlite3_clear_bindings(sql_read.getmisc); -+ return false; -+ } -+ *value = sqlite3_column_int(sql_read.getmisc, 0); -+ sqlite3_reset(sql_read.getmisc); -+ sqlite3_clear_bindings(sql_read.getmisc); -+ return true; -+} -+ -+/* -+** Open the overview database directly for read-only access. -+*/ -+static bool -+direct_open(void) -+{ -+ char *path; -+ char *confpath; -+ struct config_group *top; -+ int status; -+ char *errmsg; -+ char *journal_mode = NULL; -+ bool use_wal = false; -+ unsigned long reader_cachesize = 8000; /* default 8 MB */ -+ int version; -+ int compress_flag; -+ char sqltext[64]; -+ bool have_inflate = false; -+ bool have_stmts = false; -+ -+ /* Load config. Direct reader mode requires WAL; without WAL, writers -+ * take EXCLUSIVE locks that block all other connections during commit. -+ * The server path avoids this because reads and writes share a single -+ * connection. */ -+ confpath = concatpath(innconf->pathetc, "ovsqlite.conf"); -+ top = config_parse_file(confpath); -+ free(confpath); -+ if (top) { -+ config_param_boolean(top, "walmode", &use_wal); -+ config_param_unsigned_number(top, "readercachesize", -+ &reader_cachesize); -+ config_free(top); -+ } -+ if (!use_wal) -+ return false; -+ -+ /* Open database read-only. Suppress warning for CANTOPEN since the -+ * database may not exist yet during initial setup. */ -+ path = concatpath(innconf->pathoverview, OVSQLITE_DB_FILE); -+ status = sqlite3_open_v2(path, &read_connection, SQLITE_OPEN_READONLY, -+ NULL); -+ free(path); -+ if (status != SQLITE_OK) { -+ if (status != SQLITE_CANTOPEN) -+ warn("ovsqlite: cannot open database for reading: %s", -+ sqlite3_errstr(status)); -+ read_connection = NULL; -+ return false; -+ } -+ sqlite3_extended_result_codes(read_connection, 1); -+ -+ /* Verify the database is actually in WAL mode, not just the config. -+ * The writer sets WAL mode; if it hasn't run yet or the database was -+ * created without WAL, fall back to the server path. */ -+ status = sqlite3_exec(read_connection, "pragma journal_mode;", -+ pragma_callback, &journal_mode, NULL); -+ if (status != SQLITE_OK || !journal_mode -+ || strcmp(journal_mode, "wal") != 0) { -+ free(journal_mode); -+ sqlite3_close_v2(read_connection); -+ read_connection = NULL; -+ return false; -+ } -+ free(journal_mode); -+ -+ /* Prepare read-only statements. */ -+ status = -+ sqlite_helper_init(&sql_read_helper, (sqlite3_stmt **) &sql_read, -+ read_connection, SQLITE_PREPARE_PERSISTENT, -+ &errmsg); -+ if (status != SQLITE_OK) { -+ warn("ovsqlite: cannot set up read session: %s", errmsg); -+ sqlite3_free(errmsg); -+ goto fail; -+ } -+ have_stmts = true; -+ -+ /* Set cache size. */ -+ if (reader_cachesize) { -+ snprintf(sqltext, sizeof sqltext, "pragma cache_size = -%lu;", -+ reader_cachesize); -+ status = sqlite3_exec(read_connection, sqltext, 0, NULL, &errmsg); -+ if (status != SQLITE_OK) { -+ warn("ovsqlite: cannot set reader cache size: %s", errmsg); -+ sqlite3_free(errmsg); -+ } -+ } -+ -+ /* Validate schema version. */ -+ if (!read_misc_int("version", &version)) -+ goto fail; -+ if (version != OVSQLITE_SCHEMA_VERSION) { -+ warn("ovsqlite: incompatible database schema %d (expected %d)", -+ version, OVSQLITE_SCHEMA_VERSION); -+ goto fail; -+ } -+ -+ /* Check compression setting. */ -+ if (!read_misc_int("compress", &compress_flag)) -+ goto fail; -+ reader_use_compression = compress_flag; -+ -+# ifdef HAVE_ZLIB -+ if (reader_use_compression) { -+ void const *dict; -+ size_t size; -+ -+ reader_inflation.zalloc = Z_NULL; -+ reader_inflation.zfree = Z_NULL; -+ reader_inflation.opaque = Z_NULL; -+ reader_inflation.next_in = Z_NULL; -+ reader_inflation.avail_in = 0; -+ status = inflateInit(&reader_inflation); -+ if (status != Z_OK) { -+ warn("ovsqlite: cannot set up decompression"); -+ goto fail; -+ } -+ have_inflate = true; -+ -+# ifdef USE_DICTIONARY -+ sqlite3_bind_text(sql_read.getmisc, 1, "basedict", -1, -+ SQLITE_STATIC); -+ status = sqlite3_step(sql_read.getmisc); -+ if (status != SQLITE_ROW) { -+ warn("ovsqlite: cannot load compression dictionary: %s", -+ sqlite3_errmsg(read_connection)); -+ sqlite3_reset(sql_read.getmisc); -+ sqlite3_clear_bindings(sql_read.getmisc); -+ goto fail; -+ } -+ dict = sqlite3_column_blob(sql_read.getmisc, 0); -+ size = sqlite3_column_bytes(sql_read.getmisc, 0); -+ if (!dict || size >= sizeof reader_dictionary) { -+ warn("ovsqlite: invalid compression dictionary in database"); -+ sqlite3_reset(sql_read.getmisc); -+ sqlite3_clear_bindings(sql_read.getmisc); -+ goto fail; -+ } -+ memcpy(reader_dictionary, dict, size); -+ reader_basedict_len = size; -+ sqlite3_reset(sql_read.getmisc); -+ sqlite3_clear_bindings(sql_read.getmisc); -+# endif /* USE_DICTIONARY */ -+ -+ reader_flate = buffer_new(); -+ } -+# else /* ! HAVE_ZLIB */ -+ if (reader_use_compression) { -+ warn("ovsqlite: database uses compression but INN was not built" -+ " with zlib"); -+ goto fail; -+ } -+# endif /* ! HAVE_ZLIB */ -+ -+ direct_reader = true; -+ return true; -+ -+fail: -+# ifdef HAVE_ZLIB -+ if (have_inflate) -+ inflateEnd(&reader_inflation); -+# endif -+ if (have_stmts) -+ sqlite_helper_term(&sql_read_helper, (sqlite3_stmt **) &sql_read); -+ if (read_connection) { -+ sqlite3_close_v2(read_connection); -+ read_connection = NULL; -+ } -+ return false; -+} -+ -+ -+/* -+** Close the direct reader connection. -+*/ -+static void -+direct_close(void) -+{ -+ if (!read_connection) -+ return; -+# ifdef HAVE_ZLIB -+ if (reader_use_compression) { -+ inflateEnd(&reader_inflation); -+ buffer_free(reader_flate); -+ reader_flate = NULL; -+ } -+# endif -+ sqlite_helper_term(&sql_read_helper, (sqlite3_stmt **) &sql_read); -+ sqlite3_close_v2(read_connection); -+ read_connection = NULL; -+ direct_reader = false; -+} -+ -+ - static bool - server_connect(void) - { -@@ -283,10 +655,20 @@ server_handshake(uint32_t mode) - bool - ovsqlite_open(int mode) - { -- if (sock != -1) { -+ if (direct_reader || sock != -1) { - warn("ovsqlite_open called more than once"); - return false; - } -+ -+ /* Read-only mode: try direct database access (requires WAL mode). -+ * Falls back to the server path if WAL is not enabled. */ -+ if (mode == OV_READ) { -+ if (direct_open()) -+ return true; -+ notice("ovsqlite: direct reader not available, using server"); -+ } -+ -+ /* Read-write mode, or WAL not enabled: connect to ovsqlite-server. */ - if (!server_connect()) - return false; - if (!server_handshake(mode)) -@@ -298,6 +680,9 @@ bool - ovsqlite_groupstats(const char *group, int *low, int *high, int *count, - int *flag) - { -+ sqlite3_stmt *stmt; -+ int status; -+ char const *flag_blob; - uint16_t groupname_len; - unsigned int code; - uint64_t r_low; -@@ -306,45 +691,74 @@ ovsqlite_groupstats(const char *group, int *low, int *high, int *count, - uint16_t flag_alias_len; - uint8_t *flag_alias; - -+ if (direct_reader) { -+ stmt = sql_read.get_groupinfo; -+ sqlite3_bind_blob(stmt, 1, group, strlen(group), SQLITE_STATIC); -+ status = sqlite3_step(stmt); -+ if (status != SQLITE_ROW) { -+ if (status != SQLITE_DONE) -+ warn("ovsqlite: groupstats query error: %s", -+ sqlite3_errmsg(read_connection)); -+ sqlite3_reset(stmt); -+ sqlite3_clear_bindings(stmt); -+ return false; -+ } -+ if (low) -+ *low = sqlite3_column_int64(stmt, 0); -+ if (high) -+ *high = sqlite3_column_int64(stmt, 1); -+ if (count) -+ *count = sqlite3_column_int64(stmt, 2); -+ if (flag) { -+ flag_blob = sqlite3_column_blob(stmt, 3); -+ if (flag_blob) -+ *flag = *flag_blob; -+ } -+ sqlite3_reset(stmt); -+ sqlite3_clear_bindings(stmt); -+ return true; -+ } -+ -+ /* Server path. */ - if (sock == -1) { - warn("ovsqlite: not connected to server"); - return false; - } - groupname_len = strlen(group); -- start_request(request_get_groupinfo); -- pack_now(request, &groupname_len, sizeof groupname_len); -- pack_now(request, group, groupname_len); -- finish_request(); -- if (!write_request()) -- return false; -+ start_request(request_get_groupinfo); -+ pack_now(request, &groupname_len, sizeof groupname_len); -+ pack_now(request, group, groupname_len); -+ finish_request(); -+ if (!write_request()) -+ return false; - -- if (!read_response()) -- return false; -- code = start_response(); -- if (code != response_groupinfo) -- return false; -- if (!unpack_now(response, &r_low, sizeof r_low)) -- return false; -- if (!unpack_now(response, &r_high, sizeof r_high)) -- return false; -- if (!unpack_now(response, &r_count, sizeof r_count)) -- return false; -- if (!unpack_now(response, &flag_alias_len, sizeof flag_alias_len)) -- return false; -- flag_alias = unpack_later(response, flag_alias_len); -- if (!flag_alias) -- return false; -- if (!finish_response()) -- return false; -- if (low) -- *low = r_low; -- if (high) -- *high = r_high; -- if (count) -- *count = r_count; -- if (flag) -- *flag = *flag_alias; -- return true; -+ if (!read_response()) -+ return false; -+ code = start_response(); -+ if (code != response_groupinfo) -+ return false; -+ if (!unpack_now(response, &r_low, sizeof r_low)) -+ return false; -+ if (!unpack_now(response, &r_high, sizeof r_high)) -+ return false; -+ if (!unpack_now(response, &r_count, sizeof r_count)) -+ return false; -+ if (!unpack_now(response, &flag_alias_len, sizeof flag_alias_len)) -+ return false; -+ flag_alias = unpack_later(response, flag_alias_len); -+ if (!flag_alias) -+ return false; -+ if (!finish_response()) -+ return false; -+ if (low) -+ *low = r_low; -+ if (high) -+ *high = r_high; -+ if (count) -+ *count = r_count; -+ if (flag) -+ *flag = *flag_alias; -+ return true; - } - - bool -@@ -356,6 +770,10 @@ ovsqlite_groupadd(const char *group, ARTNUM low, ARTNUM high, char *flag) - uint64_t r_high; - unsigned int code; - -+ if (direct_reader) { -+ warn("ovsqlite: groupadd not available in direct reader mode"); -+ return false; -+ } - if (sock == -1) { - warn("ovsqlite: not connected to server"); - return false; -@@ -391,6 +809,10 @@ ovsqlite_groupdel(const char *group) - uint16_t groupname_len; - unsigned int code; - -+ if (direct_reader) { -+ warn("ovsqlite: groupdel not available in direct reader mode"); -+ return false; -+ } - if (sock == -1) { - warn("ovsqlite: not connected to server"); - return false; -@@ -424,6 +846,10 @@ ovsqlite_add(const char *group, ARTNUM artnum, TOKEN token, char *data, - uint64_t r_expires; - unsigned int code; - -+ if (direct_reader) { -+ warn("ovsqlite: add not available in direct reader mode"); -+ return false; -+ } - if (sock == -1) { - warn("ovsqlite: not connected to server"); - return false; -@@ -478,6 +904,10 @@ ovsqlite_cancel(const char *group, ARTNUM artnum) - uint64_t r_artnum; - unsigned int code; - -+ if (direct_reader) { -+ warn("ovsqlite: cancel not available in direct reader mode"); -+ return false; -+ } - if (sock == -1) { - warn("ovsqlite: not connected to server"); - return false; -@@ -508,7 +938,7 @@ ovsqlite_opensearch(const char *group, int low, int high) - handle_t *rh; - uint16_t groupname_len; - -- if (sock == -1) { -+ if (!direct_reader && sock == -1) { - warn("ovsqlite: not connected to server"); - return NULL; - } -@@ -525,6 +955,208 @@ ovsqlite_opensearch(const char *group, int low, int high) - return rh; - } - -+/* -+** Fill the search buffer directly from SQLite for direct reader mode. -+** -+** Collects rows into temporary storage, then packs into the handle_t -+** buffer in the same layout that fill_search_buffer produces: -+** [overview ptrs (count+1)] [arrived] [artnums] [tokens] [overview data] -+** The consumer (ovsqlite_search) computes overview length as -+** overview[ix+1] - overview[ix] -+** so overview[count] must be a valid sentinel pointer. -+*/ -+static bool -+fill_search_buffer_direct(handle_t *rh) -+{ -+ unsigned int cols; -+ size_t per_row; -+ uint32_t max_rows, count, ix; -+ sqlite3_stmt *stmt; -+ int status; -+ uint8_t *store; -+ -+ /* Temporary storage for collected rows. */ -+ ARTNUM *tmp_artnum; -+ time_t *tmp_arrived; -+ TOKEN *tmp_token; -+ uint8_t *tmp_overview; -+ uint32_t *tmp_ov_offset; -+ uint32_t *tmp_ov_len; -+ size_t ov_total; -+ -+ rh->count = 0; -+ rh->index = 0; -+ cols = rh->cols; -+ -+ /* Calculate per-row metadata size in the final buffer. */ -+ per_row = sizeof(ARTNUM); -+ if (cols & search_col_arrived) -+ per_row += sizeof(time_t); -+ if (cols & search_col_token) -+ per_row += sizeof(TOKEN); -+ if (cols & search_col_overview) -+ per_row += sizeof(char *); -+ -+ /* Rough upper bound on rows that can fit. */ -+ max_rows = SEARCHSPACE / per_row; -+ if (max_rows > 4096) -+ max_rows = 4096; -+ -+ /* Allocate temporary arrays. */ -+ tmp_artnum = xmalloc(max_rows * sizeof(ARTNUM)); -+ tmp_arrived = (cols & search_col_arrived) -+ ? xmalloc(max_rows * sizeof(time_t)) -+ : NULL; -+ tmp_token = (cols & search_col_token) -+ ? xmalloc(max_rows * sizeof(TOKEN)) -+ : NULL; -+ tmp_overview = (cols & search_col_overview) -+ ? xmalloc(SEARCHSPACE) -+ : NULL; -+ tmp_ov_offset = (cols & search_col_overview) -+ ? xmalloc(max_rows * sizeof(uint32_t)) -+ : NULL; -+ tmp_ov_len = (cols & search_col_overview) -+ ? xmalloc(max_rows * sizeof(uint32_t)) -+ : NULL; -+ -+ /* Select the appropriate prepared statement. */ -+ stmt = (cols & search_col_overview) ? sql_read.list_articles_high_overview -+ : sql_read.list_articles_high; -+ sqlite3_bind_blob(stmt, 1, rh->groupname, rh->groupname_len, -+ SQLITE_STATIC); -+ sqlite3_bind_int64(stmt, 2, rh->low); -+ sqlite3_bind_int64(stmt, 3, rh->high); -+ -+ /* Collect rows. */ -+ count = 0; -+ ov_total = 0; -+ while (count < max_rows) { -+ uint64_t artnum; -+ size_t meta_needed; -+ -+ status = sqlite3_step(stmt); -+ if (status == SQLITE_DONE) { -+ rh->done = true; -+ break; -+ } -+ if (status != SQLITE_ROW) { -+ warn("ovsqlite: search query error: %s", -+ sqlite3_errmsg(read_connection)); -+ break; -+ } -+ -+ artnum = sqlite3_column_int64(stmt, 0); -+ -+ if (cols & search_col_overview) { -+ uint8_t const *overview; -+ uint32_t overview_len; -+ size_t size; -+ -+ overview = sqlite3_column_blob(stmt, 4); -+ size = sqlite3_column_bytes(stmt, 4); -+ if (!overview || size > MAX_OVDATA_SIZE) -+ continue; -+ overview_len = size; -+ -+# ifdef HAVE_ZLIB -+ if (reader_use_compression && overview_len > 0) { -+ uint8_t *dec; -+ uint32_t raw_len; -+ -+ dec = reader_decompress(overview, overview_len, rh->groupname, -+ rh->groupname_len, artnum, &raw_len); -+ if (!dec) -+ continue; -+ overview = dec; -+ overview_len = raw_len; -+ } -+# endif -+ -+ /* Check if adding this row would overflow the buffer. -+ * Account for the sentinel overview pointer. */ -+ meta_needed = (count + 2) * sizeof(char *) /* overview ptrs */ -+ + (count + 1) * sizeof(ARTNUM); -+ if (cols & search_col_arrived) -+ meta_needed += (count + 1) * sizeof(time_t); -+ if (cols & search_col_token) -+ meta_needed += (count + 1) * sizeof(TOKEN); -+ if (meta_needed + ov_total + overview_len > SEARCHSPACE) -+ break; -+ -+ tmp_ov_offset[count] = ov_total; -+ tmp_ov_len[count] = overview_len; -+ memcpy(tmp_overview + ov_total, overview, overview_len); -+ ov_total += overview_len; -+ } else { -+ /* No overview: just check metadata fits. */ -+ if ((count + 1) * per_row > SEARCHSPACE) -+ break; -+ } -+ -+ tmp_artnum[count] = artnum; -+ if (cols & search_col_arrived) -+ tmp_arrived[count] = sqlite3_column_int64(stmt, 1); -+ if (cols & search_col_token) { -+ TOKEN const *tk = sqlite3_column_blob(stmt, 3); -+ -+ if (tk && sqlite3_column_bytes(stmt, 3) == sizeof(TOKEN)) -+ tmp_token[count] = *tk; -+ else -+ memset(&tmp_token[count], 0, sizeof(TOKEN)); -+ } -+ -+ count++; -+ } -+ -+ if (count > 0) -+ rh->low = tmp_artnum[count - 1] + 1; -+ sqlite3_reset(stmt); -+ sqlite3_clear_bindings(stmt); -+ -+ /* Pack into rh->buffer in the canonical layout. */ -+ store = rh->buffer; -+ rh->overview = (char **) (void *) store; -+ if (cols & search_col_overview) -+ store += (count + 1) * sizeof(char *); -+ rh->arrived = (time_t *) (void *) store; -+ if (cols & search_col_arrived) -+ store += count * sizeof(time_t); -+ rh->artnum = (ARTNUM *) (void *) store; -+ store += count * sizeof(ARTNUM); -+ rh->token = (TOKEN *) store; -+ if (cols & search_col_token) -+ store += count * sizeof(TOKEN); -+ -+ for (ix = 0; ix < count; ix++) { -+ rh->artnum[ix] = tmp_artnum[ix]; -+ if (cols & search_col_arrived) -+ rh->arrived[ix] = tmp_arrived[ix]; -+ if (cols & search_col_token) -+ rh->token[ix] = tmp_token[ix]; -+ } -+ -+ if (cols & search_col_overview) { -+ /* Copy overview data contiguously after the metadata arrays -+ * and set up pointers. */ -+ memcpy(store, tmp_overview, ov_total); -+ for (ix = 0; ix < count; ix++) -+ rh->overview[ix] = (char *) store + tmp_ov_offset[ix]; -+ rh->overview[count] = (char *) store + ov_total; -+ } -+ -+ /* Clean up temporary storage. */ -+ free(tmp_artnum); -+ free(tmp_arrived); -+ free(tmp_token); -+ free(tmp_overview); -+ free(tmp_ov_offset); -+ free(tmp_ov_len); -+ -+ rh->count = count; -+ return true; -+} -+ - static bool - fill_search_buffer(handle_t *rh) - { -@@ -651,7 +1283,7 @@ ovsqlite_search(void *handle, ARTNUM *artnum, char **data, int *len, - unsigned int cols; - unsigned int ix; - -- if (sock == -1) { -+ if (!direct_reader && sock == -1) { - warn("ovsqlite: not connected to server"); - return false; - } -@@ -670,8 +1302,13 @@ ovsqlite_search(void *handle, ARTNUM *artnum, char **data, int *len, - cols |= search_col_overview; - if (cols & ~rh->cols || ix >= rh->count) { - rh->cols = cols; -- if (!fill_search_buffer(rh)) -- return false; -+ if (direct_reader) { -+ if (!fill_search_buffer_direct(rh)) -+ return false; -+ } else { -+ if (!fill_search_buffer(rh)) -+ return false; -+ } - ix = rh->index; - if (ix >= rh->count) - return false; -@@ -694,7 +1331,7 @@ ovsqlite_search(void *handle, ARTNUM *artnum, char **data, int *len, - void - ovsqlite_closesearch(void *handle) - { -- if (sock == -1) -+ if (!direct_reader && sock == -1) - warn("ovsqlite: not connected to server"); - if (!handle) - return; -@@ -704,10 +1341,39 @@ ovsqlite_closesearch(void *handle) - bool - ovsqlite_getartinfo(const char *group, ARTNUM artnum, TOKEN *token) - { -+ sqlite3_stmt *stmt; -+ TOKEN const *db_token; -+ int status; - uint16_t groupname_len; - uint64_t r_artnum; - unsigned int code; - -+ if (direct_reader) { -+ stmt = sql_read.get_artinfo; -+ sqlite3_bind_blob(stmt, 1, group, strlen(group), SQLITE_STATIC); -+ sqlite3_bind_int64(stmt, 2, artnum); -+ status = sqlite3_step(stmt); -+ if (status != SQLITE_ROW) { -+ if (status != SQLITE_DONE) -+ warn("ovsqlite: getartinfo query error: %s", -+ sqlite3_errmsg(read_connection)); -+ sqlite3_reset(stmt); -+ sqlite3_clear_bindings(stmt); -+ return false; -+ } -+ db_token = sqlite3_column_blob(stmt, 0); -+ if (!db_token || sqlite3_column_bytes(stmt, 0) != sizeof(TOKEN)) { -+ sqlite3_reset(stmt); -+ sqlite3_clear_bindings(stmt); -+ return false; -+ } -+ *token = *db_token; -+ sqlite3_reset(stmt); -+ sqlite3_clear_bindings(stmt); -+ return true; -+ } -+ -+ /* Server path. */ - if (sock == -1) { - warn("ovsqlite: not connected to server"); - return false; -@@ -911,6 +1577,10 @@ expire_finish(void) - bool - ovsqlite_expiregroup(char const *group, int *low, struct history *h) - { -+ if (direct_reader) { -+ warn("ovsqlite: expiregroup not available in direct reader mode"); -+ return false; -+ } - if (sock == -1) { - warn("ovsqlite: not connected to server"); - return false; -@@ -946,7 +1616,7 @@ set_cutofflow(uint8_t cutofflow) - bool - ovsqlite_ctl(OVCTLTYPE type, void *val) - { -- if (sock == -1) { -+ if (!direct_reader && sock == -1) { - warn("ovsqlite: not connected to server"); - return false; - } -@@ -958,6 +1628,8 @@ ovsqlite_ctl(OVCTLTYPE type, void *val) - *(OVSORTTYPE *) val = OVNEWSGROUP; - return true; - case OVCUTOFFLOW: -+ if (direct_reader) -+ return true; /* no-op for readers */ - return set_cutofflow(*(bool *) val); - case OVSTATICSEARCH: - *(int *) val = true; -@@ -974,12 +1646,20 @@ ovsqlite_ctl(OVCTLTYPE type, void *val) - void - ovsqlite_close(void) - { -+ if (direct_reader) { -+ direct_close(); -+ return; -+ } - if (sock == -1) { - warn("ovsqlite: not connected to server"); - return; - } - close(sock); - sock = -1; -+ buffer_free(request); -+ request = NULL; -+ buffer_free(response); -+ response = NULL; - } - - #else /* ! HAVE_SQLITE3 */ -diff --git storage/ovsqlite/sql-read.sql storage/ovsqlite/sql-read.sql -new file mode 100644 -index 000000000..b41df7e44 ---- /dev/null -+++ storage/ovsqlite/sql-read.sql -@@ -0,0 +1,40 @@ -+pragma busy_timeout = 10000; -+ -+pragma query_only = 1; -+ -+-- .getmisc -+select value from misc -+ where key=?1; -+ -+-- .get_groupinfo -+select low, high, "count", flag_alias from groupinfo -+ where deleted=0 -+ and groupname=?1; -+ -+-- .get_artinfo -+select token -+ from groupinfo -+ natural join artinfo -+ where deleted=0 -+ and groupname=?1 -+ and artnum=?2; -+ -+-- .list_articles_high -+select artnum, arrived, expires, token -+ from groupinfo -+ natural join artinfo -+ where deleted=0 -+ and groupname=?1 -+ and artnum>=?2 -+ and artnum<=?3 -+ order by artnum; -+ -+-- .list_articles_high_overview -+select artnum, arrived, expires, token, overview -+ from groupinfo -+ natural join artinfo -+ where deleted=0 -+ and groupname=?1 -+ and artnum>=?2 -+ and artnum<=?3 -+ order by artnum; -diff --git support/mkmanifest support/mkmanifest -index c75f9a701..fc250a9af 100755 ---- support/mkmanifest -+++ support/mkmanifest -@@ -327,6 +327,9 @@ tests/lib/xwrite.t - tests/nnrpd/auth-ext.t - tests/overview/api.t - tests/overview/buffindexed.t -+tests/overview/ovsqlite.t -+tests/overview/ovsqlite-read.t -+tests/overview/ovsqlite-write.t - tests/overview/tradindexed.t - tests/overview/xref.t - tests/perl/minimum-version.t -diff --git tests/Makefile tests/Makefile -index fa35b1a37..ae990ee67 100644 ---- tests/Makefile -+++ tests/Makefile -@@ -30,12 +30,13 @@ TESTS = authprogs/ident.t innd/artparse.t innd/chan.t lib/artnumber.t \ - lib/setenv.t lib/snprintf.t lib/strlcat.t \ - lib/strlcpy.t lib/tst.t lib/uwildmat.t lib/vector.t lib/wire.t \ - lib/xwrite.t nnrpd/auth-ext.t overview/api.t overview/buffindexed.t \ -- overview/tradindexed.t overview/xref.t util/innbind.t -+ overview/ovsqlite.t overview/tradindexed.t overview/xref.t util/innbind.t - - ## Extra stuff that needs to be built before tests can be run. - - EXTRA = runtests clients/server-list docs/pod.t lib/xmalloc \ -- perl/minimum-version.t -+ perl/minimum-version.t overview/ovsqlite-write.t \ -+ overview/ovsqlite-read.t - - all check test tests: $(TESTS) $(EXTRA) - ./runtests -l TESTS -@@ -294,6 +295,15 @@ overview/api.t: overview/api-t.o tap/basic.o $(STORAGEDEPS) - overview/buffindexed-t.o: overview/overview-t.c - $(CC) $(CFLAGS) -DOVTYPE=buffindexed -c -o $@ overview/overview-t.c - -+overview/ovsqlite.t: overview/ovsqlite-t.o tap/basic.o $(STORAGEDEPS) -+ $(LINKDEPS) overview/ovsqlite-t.o tap/basic.o $(STORAGELIBS) $(LIBS) -+ -+overview/ovsqlite-write.t: overview/ovsqlite-write-t.o tap/basic.o $(STORAGEDEPS) -+ $(LINKDEPS) overview/ovsqlite-write-t.o tap/basic.o $(STORAGELIBS) $(LIBS) -+ -+overview/ovsqlite-read.t: overview/ovsqlite-read-t.o tap/basic.o $(STORAGEDEPS) -+ $(LINKDEPS) overview/ovsqlite-read-t.o tap/basic.o $(STORAGELIBS) $(LIBS) -+ - overview/buffindexed.t: overview/buffindexed-t.o tap/basic.o $(STORAGEDEPS) - $(LINKDEPS) overview/buffindexed-t.o tap/basic.o $(STORAGELIBS) $(LIBS) - -diff --git tests/TESTS tests/TESTS -index b995af703..0a720e30a 100644 ---- tests/TESTS -+++ tests/TESTS -@@ -52,6 +52,8 @@ lib/xwrite - nnrpd/auth-ext - overview/api - overview/buffindexed -+overview/ovsqlite -+overview/ovsqlite-integ - overview/overchan - overview/tradindexed - overview/xref -diff --git tests/overview/ovsqlite-integ.t tests/overview/ovsqlite-integ.t -new file mode 100755 -index 000000000..d4df26fd2 ---- /dev/null -+++ tests/overview/ovsqlite-integ.t -@@ -0,0 +1,215 @@ -+#! /bin/sh -+# -+# Integration test for ovsqlite direct reader mode. -+# -+# Starts a real ovsqlite-server, writes test data through it via the -+# writer program, kills the server, then verifies the reader program -+# can read the WAL-mode database directly without the server. -+ -+count=1 -+printcount() { -+ echo "$1 $count $2" -+ count=$(expr $count + 1) -+} -+ -+# Find the right directory. -+dirs='../data data tests/data' -+for dir in $dirs; do -+ if [ -r "$dir/overview/basic" ]; then -+ testdata="$dir" -+ break -+ fi -+done -+if [ -z "$testdata" ]; then -+ echo "Could not find test data" >&2 -+ exit 1 -+fi -+ -+server="../../storage/ovsqlite/ovsqlite-server" -+writer="overview/ovsqlite-write.t" -+reader="overview/ovsqlite-read.t" -+ -+# When run from the tests directory, adjust paths. -+if [ ! -x "$server" ]; then -+ server="../storage/ovsqlite/ovsqlite-server" -+fi -+if [ ! -x "$writer" ]; then -+ writer="./overview/ovsqlite-write.t" -+fi -+if [ ! -x "$reader" ]; then -+ reader="./overview/ovsqlite-read.t" -+fi -+ -+# Skip if SQLite support was not compiled in. -+if ! grep -q '^#define HAVE_SQLITE3' ../include/config.h 2>/dev/null; then -+ echo "1..0 # skip SQLite support not compiled" -+ exit 0 -+fi -+ -+for prog in "$server" "$writer" "$reader"; do -+ if [ ! -x "$prog" ]; then -+ echo "1..0 # skip ovsqlite binaries not built" -+ exit 0 -+ fi -+done -+ -+echo 13 -+ -+# Set up temp directory with config files. Use absolute paths because -+# the C test programs chdir to the test data directory before reading -+# inn.conf. -+tmpdir="$(pwd)/ov-tmp-integ" -+rm -rf "$tmpdir" -+mkdir -p "$tmpdir" -+ -+cat > "$tmpdir/inn.conf" < "$tmpdir/ovsqlite.conf" < "$tmpdir/server.log" 2>&1 & -+server_pid=$! -+ -+# Wait for socket file (up to 5 seconds). -+waited=0 -+while [ ! -S "$tmpdir/ovsqlite.sock" ] && [ $waited -lt 50 ]; do -+ # Check if server died. -+ if ! kill -0 $server_pid 2>/dev/null; then -+ echo "# Server exited prematurely, log:" -+ sed 's/^/# /' "$tmpdir/server.log" -+ printcount "not ok" "# server startup" -+ rm -rf "$tmpdir" -+ exit 1 -+ fi -+ sleep 0.1 -+ waited=$(expr $waited + 1) -+done -+ -+if [ -S "$tmpdir/ovsqlite.sock" ]; then -+ printcount "ok" "# server started" -+else -+ echo "# Server socket not found after 5s, log:" -+ sed 's/^/# /' "$tmpdir/server.log" -+ printcount "not ok" "# server started" -+ kill $server_pid 2>/dev/null -+ rm -rf "$tmpdir" -+ exit 1 -+fi -+ -+# Tests 2-4: run the writer (3 TAP tests from inside). -+writer_output=$($writer 2>&1) -+writer_rc=$? -+writer_count=$(echo "$writer_output" | grep -c "^ok ") -+if [ $writer_rc -eq 0 ] && [ "$writer_count" -eq 3 ]; then -+ printcount "ok" "# writer open" -+ printcount "ok" "# writer load" -+ printcount "ok" "# writer close" -+else -+ echo "# Writer failed (rc=$writer_rc):" -+ echo "$writer_output" | sed 's/^/# /' -+ printcount "not ok" "# writer open" -+ printcount "not ok" "# writer load" -+ printcount "not ok" "# writer close" -+fi -+ -+# Test 5: stop the server gracefully. -+kill $server_pid 2>/dev/null -+wait $server_pid 2>/dev/null -+if [ $? -le 128 ] || [ $? -eq 143 ]; then -+ printcount "ok" "# server stopped" -+else -+ printcount "ok" "# server stopped (signal)" -+fi -+ -+# Tests 6-10: run the reader (5 TAP tests from inside). -+reader_output=$($reader 2>&1) -+reader_rc=$? -+reader_count=$(echo "$reader_output" | grep -c "^ok ") -+if [ $reader_rc -eq 0 ] && [ "$reader_count" -eq 5 ]; then -+ printcount "ok" "# reader open (direct)" -+ printcount "ok" "# group stats" -+ printcount "ok" "# article data" -+ printcount "ok" "# search order" -+ printcount "ok" "# reopen" -+else -+ echo "# Reader failed (rc=$reader_rc):" -+ echo "$reader_output" | sed 's/^/# /' -+ printcount "not ok" "# reader open (direct)" -+ printcount "not ok" "# group stats" -+ printcount "not ok" "# article data" -+ printcount "not ok" "# search order" -+ printcount "not ok" "# reopen" -+fi -+ -+# Tests 11-13: server restart resilience. -+# Simulate INN restart: start a new server against the existing WAL database, -+# shut it down, verify the reader can still read all data. This is the -+# scenario where nnrpd outlives a server restart cycle. -+$server -d > "$tmpdir/server2.log" 2>&1 & -+server_pid=$! -+waited=0 -+while [ ! -S "$tmpdir/ovsqlite.sock" ] && [ $waited -lt 50 ]; do -+ if ! kill -0 $server_pid 2>/dev/null; then -+ echo "# Server restart failed, log:" -+ sed 's/^/# /' "$tmpdir/server2.log" -+ printcount "not ok" "# server restart" -+ printcount "not ok" "# server re-stop" -+ printcount "not ok" "# reader after restart" -+ rm -rf "$tmpdir" -+ exit 1 -+ fi -+ sleep 0.1 -+ waited=$(expr $waited + 1) -+done -+if [ -S "$tmpdir/ovsqlite.sock" ]; then -+ printcount "ok" "# server restart" -+else -+ printcount "not ok" "# server restart" -+fi -+ -+kill $server_pid 2>/dev/null -+wait $server_pid 2>/dev/null -+printcount "ok" "# server re-stop" -+ -+# Reader should still work — data survives the restart cycle. -+reader_output=$($reader 2>&1) -+reader_rc=$? -+reader_count=$(echo "$reader_output" | grep -c "^ok ") -+if [ $reader_rc -eq 0 ] && [ "$reader_count" -eq 5 ]; then -+ printcount "ok" "# reader after restart" -+else -+ echo "# Reader after restart failed (rc=$reader_rc):" -+ echo "$reader_output" | sed 's/^/# /' -+ printcount "not ok" "# reader after restart" -+fi -+ -+# Clean up. -+rm -rf "$tmpdir" -diff --git tests/overview/ovsqlite-read-t.c tests/overview/ovsqlite-read-t.c -new file mode 100644 -index 000000000..ca3517671 ---- /dev/null -+++ tests/overview/ovsqlite-read-t.c -@@ -0,0 +1,424 @@ -+/* Reader half of the ovsqlite direct reader integration test. -+ * -+ * Opens overview in OV_READ mode (triggers direct_open for WAL databases), -+ * reads back data written by the writer half, and verifies correctness. -+ * No ovsqlite-server is needed -- this is the whole point of direct reader. -+ * -+ * Important: OVadd determines group:artnum from the Xref header in the -+ * overview data, NOT from the prefix in the test data file. For most -+ * articles these match, but some test articles intentionally differ -+ * (e.g., file says news.groups:260 but Xref says news.groups:60). -+ * We must parse Xref to know what was actually stored. -+ */ -+ -+#include "portable/system.h" -+ -+#include -+#include -+#include -+ -+#include "inn/hashtab.h" -+#include "inn/innconf.h" -+#include "inn/libinn.h" -+#include "inn/messages.h" -+#include "inn/ov.h" -+#include "inn/storage.h" -+#include "inn/vector.h" -+#include "tap/basic.h" -+ -+static const TOKEN faketoken = {1, 1, ""}; -+ -+struct group { -+ char *group; -+ unsigned long count; -+ unsigned long low; -+ unsigned long high; -+}; -+ -+static const void * -+group_key(const void *entry) -+{ -+ return ((const struct group *) entry)->group; -+} -+ -+static bool -+group_eq(const void *key, const void *entry) -+{ -+ return !strcmp((const char *) key, -+ ((const struct group *) entry)->group); -+} -+ -+static void -+group_del(void *entry) -+{ -+ struct group *g = (struct group *) entry; -+ free(g->group); -+ free(g); -+} -+ -+/* Parse the Xref field from an overview line to find group:artnum pairs. -+ * The overview line format includes "Xref: hostname group:artnum ..." -+ * as one of the tab-separated fields. -+ * -+ * For each group:artnum in the Xref, calls the callback with the group -+ * name, artnum, the file-format artnum, and the full line from the file. -+ * Returns the number of Xref entries processed. */ -+struct xref_entry { -+ char group[256]; -+ unsigned long artnum; -+}; -+ -+/* Find the last Xref field in an overview line and extract group:artnum -+ * pairs. Returns the number of entries found (up to max_entries). */ -+static int -+parse_xref(const char *data, size_t datalen, struct xref_entry *entries, -+ int max_entries) -+{ -+ const char *xref = NULL; -+ const char *p; -+ int n = 0; -+ -+ /* Find last "Xref: " preceded by a tab. */ -+ for (p = data; p < data + datalen - 6; p++) { -+ if (*p == 'X' && memcmp(p, "Xref: ", 6) == 0 -+ && (p == data || p[-1] == '\t')) -+ xref = p; -+ } -+ if (!xref) -+ return 0; -+ -+ /* Skip "Xref: hostname " */ -+ xref += 6; -+ while (xref < data + datalen && *xref != ' ') -+ xref++; -+ if (xref < data + datalen) -+ xref++; -+ -+ /* Parse group:artnum pairs. */ -+ while (xref < data + datalen && n < max_entries) { -+ const char *colon, *end; -+ size_t glen; -+ -+ while (xref < data + datalen && isspace((unsigned char) *xref)) -+ xref++; -+ if (xref >= data + datalen) -+ break; -+ -+ colon = memchr(xref, ':', data + datalen - xref); -+ if (!colon) -+ break; -+ glen = colon - xref; -+ if (glen >= sizeof(entries[0].group)) -+ break; -+ memcpy(entries[n].group, xref, glen); -+ entries[n].group[glen] = '\0'; -+ -+ entries[n].artnum = strtoul(colon + 1, NULL, 10); -+ if (entries[n].artnum == 0) -+ break; -+ n++; -+ -+ /* Advance to next space or end. */ -+ end = memchr(xref, ' ', data + datalen - xref); -+ if (!end) -+ end = memchr(xref, '\t', data + datalen - xref); -+ if (!end) -+ break; -+ xref = end + 1; -+ } -+ return n; -+} -+ -+/* Build expected group stats from the data file, using Xref-derived -+ * artnums (which is what OVadd actually stores). */ -+static struct hash * -+load_expected(const char *path) -+{ -+ struct hash *groups; -+ struct group *g; -+ FILE *f; -+ char buffer[4096]; -+ struct xref_entry entries[16]; -+ int i, nxref; -+ size_t len; -+ -+ groups = hash_create(32, hash_string, group_key, group_eq, group_del); -+ f = fopen(path, "r"); -+ if (f == NULL) -+ sysdie("Cannot open %s", path); -+ -+ while (fgets(buffer, sizeof(buffer), f) != NULL) { -+ len = strlen(buffer); -+ if (len > 0 && buffer[len - 1] == '\n') -+ len--; -+ -+ nxref = parse_xref(buffer, len, entries, 16); -+ for (i = 0; i < nxref; i++) { -+ g = hash_lookup(groups, entries[i].group); -+ if (g == NULL) { -+ g = xmalloc(sizeof(*g)); -+ g->group = xstrdup(entries[i].group); -+ g->count = 1; -+ g->low = entries[i].artnum; -+ g->high = entries[i].artnum; -+ hash_insert(groups, g->group, g); -+ } else { -+ g->count++; -+ if (entries[i].artnum < g->low) -+ g->low = entries[i].artnum; -+ if (entries[i].artnum > g->high) -+ g->high = entries[i].artnum; -+ } -+ } -+ } -+ fclose(f); -+ return groups; -+} -+ -+/* Verify group stats via the OV API. */ -+static void -+verify_group(void *data, void *cookie) -+{ -+ struct group *g = (struct group *) data; -+ bool *status = (bool *) cookie; -+ int low, high, count, flag; -+ -+ if (!OVgroupstats(g->group, &low, &high, &count, &flag)) { -+ warn("No stats for %s", g->group); -+ *status = false; -+ return; -+ } -+ if ((unsigned long) low != g->low) { -+ warn("Low wrong for %s: %d != %lu", g->group, low, g->low); -+ *status = false; -+ } -+ if ((unsigned long) high != g->high) { -+ warn("High wrong for %s: %d != %lu", g->group, high, g->high); -+ *status = false; -+ } -+ if ((unsigned long) count != g->count) { -+ warn("Count wrong for %s: %d != %lu", g->group, count, g->count); -+ *status = false; -+ } -+} -+ -+/* Verify per-article data: getartinfo returns correct token, search returns -+ * correct overview data and arrival time. -+ * -+ * We verify using the Xref-derived artnum (what OVadd actually stored). -+ * The arrival time is based on the file-prefix artnum (what the writer -+ * passed as arrived = artnum * 10). */ -+static bool -+verify_articles(const char *path) -+{ -+ FILE *f; -+ char buffer[4096]; -+ char *start, *colon; -+ unsigned long file_artnum; -+ struct xref_entry entries[16]; -+ int nxref, i; -+ size_t linelen; -+ TOKEN token; -+ void *search; -+ ARTNUM overnum; -+ char *data; -+ int len; -+ time_t arrived; -+ bool status = true; -+ -+ f = fopen(path, "r"); -+ if (f == NULL) -+ sysdie("Cannot open %s", path); -+ -+ while (fgets(buffer, sizeof(buffer), f) != NULL) { -+ linelen = strlen(buffer); -+ if (linelen > 0 && buffer[linelen - 1] == '\n') -+ linelen--; -+ -+ /* Parse the file-format artnum (used for arrival time). */ -+ colon = strchr(buffer, ':'); -+ if (!colon) -+ continue; -+ start = colon + 1; -+ file_artnum = strtoul(start, NULL, 10); -+ -+ /* Parse Xref to get the actual stored artnum(s). */ -+ nxref = parse_xref(buffer, linelen, entries, 16); -+ for (i = 0; i < nxref; i++) { -+ /* getartinfo should find this article. */ -+ if (!OVgetartinfo(entries[i].group, entries[i].artnum, &token)) { -+ warn("No artinfo for %s:%lu", entries[i].group, -+ entries[i].artnum); -+ status = false; -+ continue; -+ } -+ if (memcmp(&token, &faketoken, sizeof(token)) != 0) { -+ warn("Token wrong for %s:%lu", entries[i].group, -+ entries[i].artnum); -+ status = false; -+ } -+ -+ /* Search for this article and verify arrival time. */ -+ search = OVopensearch(entries[i].group, entries[i].artnum, -+ entries[i].artnum); -+ if (search == NULL) { -+ warn("Cannot open search for %s:%lu", entries[i].group, -+ entries[i].artnum); -+ status = false; -+ continue; -+ } -+ if (!OVsearch(search, &overnum, &data, &len, &token, &arrived)) { -+ warn("No search result for %s:%lu", entries[i].group, -+ entries[i].artnum); -+ status = false; -+ OVclosesearch(search); -+ continue; -+ } -+ -+ /* Verify the arrival time matches what the writer stored. -+ * The writer uses file_artnum * 10 as the arrival time. */ -+ if ((unsigned long) arrived != file_artnum * 10) { -+ warn("Arrived wrong for %s:%lu: %lu != %lu", -+ entries[i].group, entries[i].artnum, -+ (unsigned long) arrived, file_artnum * 10); -+ status = false; -+ } -+ -+ /* There should be no more results. */ -+ if (OVsearch(search, &overnum, &data, &len, &token, &arrived)) { -+ warn("Extra article for %s:%lu", entries[i].group, -+ entries[i].artnum); -+ status = false; -+ } -+ OVclosesearch(search); -+ } -+ } -+ fclose(f); -+ return status; -+} -+ -+/* Verify that a multi-article search returns articles in order for the -+ * first group found in the data file. */ -+static bool -+verify_search_order(const char *path) -+{ -+ FILE *f; -+ char buffer[4096]; -+ struct xref_entry entries[16]; -+ int nxref; -+ size_t len; -+ char *group = NULL; -+ unsigned long low = 0, high = 0; -+ unsigned long article_count = 0; -+ void *search; -+ ARTNUM overnum, last = 0; -+ char *data; -+ int dlen; -+ TOKEN token; -+ time_t arrived; -+ unsigned long found = 0; -+ bool status = true; -+ -+ f = fopen(path, "r"); -+ if (f == NULL) -+ sysdie("Cannot open %s", path); -+ -+ /* Scan the file to find the range for the first group. */ -+ while (fgets(buffer, sizeof(buffer), f) != NULL) { -+ len = strlen(buffer); -+ if (len > 0 && buffer[len - 1] == '\n') -+ len--; -+ nxref = parse_xref(buffer, len, entries, 16); -+ if (nxref == 0) -+ continue; -+ if (group == NULL) { -+ group = xstrdup(entries[0].group); -+ low = entries[0].artnum; -+ high = entries[0].artnum; -+ article_count = 1; -+ } else if (strcmp(group, entries[0].group) == 0) { -+ if (entries[0].artnum < low) -+ low = entries[0].artnum; -+ if (entries[0].artnum > high) -+ high = entries[0].artnum; -+ article_count++; -+ } -+ } -+ fclose(f); -+ -+ if (!group) { -+ warn("No group found in %s", path); -+ return false; -+ } -+ -+ search = OVopensearch(group, low, high); -+ if (search == NULL) { -+ warn("Cannot open search for %s:%lu-%lu", group, low, high); -+ free(group); -+ return false; -+ } -+ -+ while (OVsearch(search, &overnum, &data, &dlen, &token, &arrived)) { -+ if (overnum <= last && last != 0) { -+ warn("Out of order in %s: %lu after %lu", group, -+ (unsigned long) overnum, (unsigned long) last); -+ status = false; -+ } -+ last = overnum; -+ found++; -+ } -+ OVclosesearch(search); -+ -+ if (found != article_count) { -+ warn("Wrong count in search for %s: %lu != %lu", group, found, -+ article_count); -+ status = false; -+ } -+ -+ free(group); -+ return status; -+} -+ -+int -+main(void) -+{ -+ struct hash *groups; -+ bool status; -+ -+ test_init(5); -+ -+ if (access("../data/overview/basic", F_OK) == 0) { -+ if (chdir("../data") < 0) -+ sysbail("cannot chdir to ../data"); -+ } else if (access("data/overview/basic", F_OK) == 0) { -+ if (chdir("data") < 0) -+ sysbail("cannot chdir to data"); -+ } else if (access("tests/data/overview/basic", F_OK) == 0) { -+ if (chdir("tests/data") < 0) -+ sysbail("cannot chdir to tests/data"); -+ } -+ -+ /* Test 1: open in direct reader mode (OV_READ triggers direct_open). */ -+ ok(1, OVopen(OV_READ)); -+ -+ /* Test 2: group stats match expected values. */ -+ groups = load_expected("overview/basic"); -+ status = true; -+ hash_traverse(groups, verify_group, &status); -+ ok(2, status); -+ -+ /* Test 3: per-article data integrity (token, overview, arrived). */ -+ ok(3, verify_articles("overview/basic")); -+ -+ /* Test 4: multi-article search returns articles in order. */ -+ ok(4, verify_search_order("overview/basic")); -+ -+ hash_free(groups); -+ OVclose(); -+ -+ /* Test 5: close and reopen works. */ -+ ok(5, OVopen(OV_READ)); -+ OVclose(); -+ -+ return 0; -+} -diff --git tests/overview/ovsqlite-t.c tests/overview/ovsqlite-t.c -new file mode 100644 -index 000000000..29c6e152a ---- /dev/null -+++ tests/overview/ovsqlite-t.c -@@ -0,0 +1,186 @@ -+/* Unit tests for ovsqlite direct reader edge cases. -+ * -+ * Tests behavior that can't be exercised by the integration test -+ * (ovsqlite-integ.t) which requires a running ovsqlite-server. -+ * These tests create a database directly via the sqlite3 API. -+ */ -+ -+#include "portable/system.h" -+ -+#include -+ -+#include "inn/innconf.h" -+#include "inn/libinn.h" -+#include "inn/messages.h" -+#include "inn/ov.h" -+#include "inn/storage.h" -+#include "tap/basic.h" -+ -+#ifdef HAVE_SQLITE3 -+ -+# include -+ -+# define OVSQLITE_DB_FILE "ovsqlite.db" -+# define TEST_DIR "ov-tmp" -+ -+static const char *schema = -+ "create table misc (key text primary key, value not null) without rowid;" -+ "create table groupinfo (groupid integer primary key," -+ " low integer not null default 1, high integer not null default 0," -+ " \"count\" integer not null default 0," -+ " expired integer not null default 0," -+ " deleted integer not null default 0," -+ " groupname blob not null, flag_alias blob not null," -+ " unique (deleted, groupname));" -+ "create table artinfo (groupid integer references groupinfo (groupid)" -+ " on update cascade on delete restrict," -+ " artnum integer, arrived integer not null, expires integer not null," -+ " token blob not null, overview blob not null," -+ " primary key (groupid, artnum)) without rowid;"; -+ -+ -+/* Create a minimal database with the given journal mode. */ -+static bool -+create_db(const char *dbpath, bool enable_wal) -+{ -+ sqlite3 *db; -+ int status; -+ char *errmsg; -+ -+ status = sqlite3_open_v2(dbpath, &db, -+ SQLITE_OPEN_READWRITE | SQLITE_OPEN_CREATE, -+ NULL); -+ if (status != SQLITE_OK) -+ return false; -+ -+ status = sqlite3_exec(db, schema, 0, NULL, &errmsg); -+ if (status != SQLITE_OK) { -+ sqlite3_free(errmsg); -+ sqlite3_close(db); -+ return false; -+ } -+ -+ sqlite3_exec(db, -+ "insert into misc (key, value) values ('version', 1);" -+ "insert into misc (key, value) values ('compress', 0);", -+ 0, NULL, NULL); -+ -+ if (enable_wal) -+ sqlite3_exec(db, "pragma journal_mode = 'WAL';", 0, NULL, NULL); -+ -+ sqlite3_close(db); -+ return true; -+} -+ -+static void -+write_config(const char *dir, bool walmode) -+{ -+ char *path; -+ FILE *f; -+ -+ path = concatpath(dir, "ovsqlite.conf"); -+ f = fopen(path, "w"); -+ free(path); -+ fprintf(f, "walmode: %s\n", walmode ? "true" : "false"); -+ fprintf(f, "readercachesize: 1000\n"); -+ fclose(f); -+} -+ -+static void -+fake_innconf(void) -+{ -+ if (innconf != NULL) { -+ free(innconf->ovmethod); -+ free(innconf->pathdb); -+ free(innconf->pathetc); -+ free(innconf->pathoverview); -+ free(innconf->pathrun); -+ free(innconf); -+ } -+ innconf = xmalloc(sizeof(*innconf)); -+ memset(innconf, 0, sizeof(*innconf)); -+ innconf->enableoverview = true; -+ innconf->ovmethod = xstrdup("ovsqlite"); -+ innconf->pathdb = xstrdup(TEST_DIR); -+ innconf->pathetc = xstrdup(TEST_DIR); -+ innconf->pathoverview = xstrdup(TEST_DIR); -+ innconf->pathrun = xstrdup(TEST_DIR); -+} -+ -+static void -+setup(void) -+{ -+ if (system("/bin/rm -rf " TEST_DIR) < 0) -+ sysdie("Cannot rm " TEST_DIR); -+ if (mkdir(TEST_DIR, 0755)) -+ sysdie("Cannot mkdir " TEST_DIR); -+} -+ -+int -+main(void) -+{ -+ char *dbpath; -+ -+ test_init(4); -+ -+ /* -+ * Test 1-2: WAL mode gating. -+ * direct_open() checks pragma journal_mode and only activates direct -+ * reader if the database is actually in WAL mode. With a non-WAL -+ * database, it falls back to the server path (which fails here since -+ * no server is running). -+ */ -+ setup(); -+ dbpath = concatpath(TEST_DIR, OVSQLITE_DB_FILE); -+ create_db(dbpath, false); -+ free(dbpath); -+ write_config(TEST_DIR, true); /* config says WAL, but DB isn't */ -+ fake_innconf(); -+ ok(1, !OVopen(OV_READ)); /* should fail: not WAL, no server */ -+ -+ /* With WAL enabled on the database, direct_open succeeds. */ -+ setup(); -+ dbpath = concatpath(TEST_DIR, OVSQLITE_DB_FILE); -+ create_db(dbpath, true); -+ free(dbpath); -+ write_config(TEST_DIR, true); -+ fake_innconf(); -+ ok(2, OVopen(OV_READ)); -+ OVclose(); -+ -+ /* -+ * Test 3-4: Write operations rejected in direct reader mode. -+ * The direct reader opens SQLITE_OPEN_READONLY and sets query_only=1. -+ * Write API calls should return false. -+ */ -+ fake_innconf(); -+ OVopen(OV_READ); -+ { -+ char group[] = "new.group"; -+ char flag[] = "y"; -+ -+ ok(3, !OVgroupadd(group, 0, 0, flag)); -+ } -+ { -+ TOKEN token = {0, 0, ""}; -+ char data[] = "test"; -+ -+ ok(4, OVadd(token, data, 4, 0, 0) != OVADDCOMPLETED); -+ } -+ OVclose(); -+ -+ if (system("/bin/rm -rf " TEST_DIR) < 0) -+ sysdie("Cannot rm " TEST_DIR); -+ return 0; -+} -+ -+#else /* ! HAVE_SQLITE3 */ -+ -+int -+main(void) -+{ -+ skip_all("SQLite support not compiled"); -+ return 0; -+} -+ -+#endif -diff --git tests/overview/ovsqlite-write-t.c tests/overview/ovsqlite-write-t.c -new file mode 100644 -index 000000000..14faac053 ---- /dev/null -+++ tests/overview/ovsqlite-write-t.c -@@ -0,0 +1,144 @@ -+/* Writer half of the ovsqlite direct reader integration test. -+ * -+ * Opens overview in OV_READ|OV_WRITE mode (connects to ovsqlite-server), -+ * creates groups and inserts articles via the standard OV API, then closes. -+ * The server creates the real database schema and handles compression. -+ */ -+ -+#include "portable/system.h" -+ -+#include -+#include -+ -+#include "inn/innconf.h" -+#include "inn/libinn.h" -+#include "inn/messages.h" -+#include "inn/ov.h" -+#include "inn/storage.h" -+#include "tap/basic.h" -+ -+/* The same fake token used by overview-t.c. */ -+static const TOKEN faketoken = {1, 1, ""}; -+ -+/* Parse a line from the overview data file. Nul-terminates the group name -+ * at the beginning of the buffer and returns a pointer to the start of the -+ * overview data (the artnum field). Sets *artnum via the passed pointer. -+ * This is the same parsing logic used by overview-t.c. */ -+static char * -+overview_data_parse(char *data, unsigned long *artnum) -+{ -+ char *start; -+ -+ if (data[strlen(data) - 1] != '\n') -+ die("Line too long in input data"); -+ -+ start = strchr(data, ':'); -+ if (start == NULL) -+ die("No colon found in input data"); -+ *start = '\0'; -+ start++; -+ *artnum = strtoul(start, NULL, 10); -+ if (*artnum == 0) -+ die("Cannot parse article number in input data"); -+ return start; -+} -+ -+int -+main(void) -+{ -+ FILE *f; -+ char buffer[4096]; -+ char *start; -+ unsigned long artnum; -+ char flag[] = NF_FLAG_OK_STRING; -+ /* Track which groups we've already added. */ -+ char *seen_groups[64]; -+ int ngroups = 0; -+ int narticles = 0; -+ int i; -+ bool seen; -+ -+ test_init(3); -+ -+ if (access("../data/overview/basic", F_OK) == 0) { -+ if (chdir("../data") < 0) -+ sysbail("cannot chdir to ../data"); -+ } else if (access("data/overview/basic", F_OK) == 0) { -+ if (chdir("data") < 0) -+ sysbail("cannot chdir to data"); -+ } else if (access("tests/data/overview/basic", F_OK) == 0) { -+ if (chdir("tests/data") < 0) -+ sysbail("cannot chdir to tests/data"); -+ } -+ -+ /* Test 1: open overview in read/write mode (connects to server). */ -+ ok(1, OVopen(OV_READ | OV_WRITE)); -+ -+ /* Load test data from the same file used by overview-t.c. */ -+ f = fopen("overview/basic", "r"); -+ if (f == NULL) -+ sysdie("Cannot open overview/basic"); -+ -+ while (fgets(buffer, sizeof(buffer), f) != NULL) { -+ start = overview_data_parse(buffer, &artnum); -+ -+ /* Add group if we haven't seen it yet. */ -+ seen = false; -+ for (i = 0; i < ngroups; i++) { -+ if (strcmp(seen_groups[i], buffer) == 0) { -+ seen = true; -+ break; -+ } -+ } -+ if (!seen) { -+ if (ngroups >= 64) -+ die("Too many groups in test data"); -+ seen_groups[ngroups] = xstrdup(buffer); -+ ngroups++; -+ if (!OVgroupadd(buffer, 0, 0, flag)) -+ die("Cannot add group %s", buffer); -+ } -+ -+ /* OVadd expects the overview data *without* the "artnum\t" prefix -+ * and without trailing \r\n -- it adds those itself. But it does -+ * need the Xref field present so it can determine group:artnum. -+ * -+ * In the data file, the format after overview_data_parse is: -+ * "artnum\tSubject...\tXref: host group:artnum\n" -+ * -+ * We need to pass the data starting from the first tab character -+ * (skipping the artnum), without the trailing \n. OVadd will -+ * prepend "artnum\t" and append "\r\n" before storing. */ -+ { -+ char *tab = strchr(start, '\t'); -+ int len; -+ -+ if (tab == NULL) -+ die("No tab after artnum in data for %s:%lu", buffer, artnum); -+ tab++; /* point past the tab to the actual overview fields */ -+ len = strlen(tab); -+ /* Strip trailing \n */ -+ if (len > 0 && tab[len - 1] == '\n') -+ len--; -+ -+ if (OVadd(faketoken, tab, len, artnum * 10, -+ (artnum % 5 == 0) ? artnum * 100 : artnum) -+ != OVADDCOMPLETED) -+ die("Cannot add %s:%lu", buffer, artnum); -+ } -+ narticles++; -+ } -+ fclose(f); -+ -+ /* Test 2: all articles were loaded. */ -+ ok(2, narticles > 0); -+ -+ /* Test 3: close succeeds. */ -+ OVclose(); -+ ok(3, true); -+ -+ for (i = 0; i < ngroups; i++) -+ free(seen_groups[i]); -+ -+ return 0; -+} - -From 05d70a17c5720be25fcd957c8ab7a0c12a64d30b Mon Sep 17 00:00:00 2001 -From: Kevin Bowling -Date: Wed, 6 May 2026 23:04:33 -0700 -Subject: [PATCH 3/7] call sqlite3_close_v2 when sqlite3_open_v2 fails - ---- - storage/ovsqlite/ovsqlite.c | 1 + - tests/overview/ovsqlite-t.c | 4 +++- - 2 files changed, 4 insertions(+), 1 deletion(-) - -diff --git storage/ovsqlite/ovsqlite.c storage/ovsqlite/ovsqlite.c -index 5d6621570..557d9710b 100644 ---- storage/ovsqlite/ovsqlite.c -+++ storage/ovsqlite/ovsqlite.c -@@ -278,6 +278,7 @@ direct_open(void) - if (status != SQLITE_CANTOPEN) - warn("ovsqlite: cannot open database for reading: %s", - sqlite3_errstr(status)); -+ sqlite3_close_v2(read_connection); - read_connection = NULL; - return false; - } -diff --git tests/overview/ovsqlite-t.c tests/overview/ovsqlite-t.c -index 29c6e152a..c99155cc5 100644 ---- tests/overview/ovsqlite-t.c -+++ tests/overview/ovsqlite-t.c -@@ -50,8 +50,10 @@ create_db(const char *dbpath, bool enable_wal) - status = sqlite3_open_v2(dbpath, &db, - SQLITE_OPEN_READWRITE | SQLITE_OPEN_CREATE, - NULL); -- if (status != SQLITE_OK) -+ if (status != SQLITE_OK) { -+ sqlite3_close_v2(db); - return false; -+ } - - status = sqlite3_exec(db, schema, 0, NULL, &errmsg); - if (status != SQLITE_OK) { - -From 0b308fdd3f6255e8c81a3deaf1bad08cb1fe8f1d Mon Sep 17 00:00:00 2001 -From: Kevin Bowling -Date: Wed, 6 May 2026 23:35:14 -0700 -Subject: [PATCH 4/7] ovsqlite: Add runtime WAL checkpoint to prevent unbounded - WAL growth - -When WAL mode is enabled with direct readers, the WAL file can grow -without bound if checkpointing never occurs between commits. Add a -configurable walcheckpointthreshold (default 1000 pages) that triggers -a TRUNCATE checkpoint during the server's idle period. - -The checkpoint temporarily lowers the busy_timeout to 10 seconds -(from ~31 years) so the server waits for readers to finish their -current queries but doesn't stall indefinitely. Readers use -autocommit with sub-millisecond snapshots, so the checkpoint -typically completes in milliseconds. - -Also fix notice() messages being lost after daemon() by registering -message_handlers_notice, remove the unused checkpoint_wal prepared -statement from sql-main.sql, and update close_db() comment. ---- - doc/pod/ovsqlite.pod | 14 ++++++++ - storage/ovsqlite/ovsqlite-server.c | 52 ++++++++++++++++++++++++++++-- - storage/ovsqlite/sql-main.sql | 3 -- - 3 files changed, 63 insertions(+), 6 deletions(-) - -diff --git doc/pod/ovsqlite.pod doc/pod/ovsqlite.pod -index 8d1a1957e..4a6eff01b 100644 ---- doc/pod/ovsqlite.pod -+++ doc/pod/ovsqlite.pod -@@ -103,6 +103,20 @@ WAL, writers take exclusive locks that would block direct readers during - commits, so the server path is used instead. The default value - is S<8000 KB> (about S<8 MB>). - -+=item I -+ -+The WAL page count at which B forces a checkpoint. -+When the WAL grows beyond this many pages, the server performs a -+TRUNCATE checkpoint during its next idle period, waiting up to -+10 seconds for readers to finish their current queries before flushing -+WAL content back to the database file. This prevents unbounded WAL -+growth and is no worse than the non-WAL path where every commit takes -+an exclusive lock. If readers do not finish within the timeout, the -+checkpoint is skipped and retried on the next idle cycle. -+ -+This parameter is only effective when I is enabled. The default -+value is 1000 pages. -+ - =item I - - The SQLite database page size in bytes. Must be a power of 2, minimum 512, -diff --git storage/ovsqlite/ovsqlite-server.c storage/ovsqlite/ovsqlite-server.c -index dc78fbb33..623b17db0 100644 ---- storage/ovsqlite/ovsqlite-server.c -+++ storage/ovsqlite/ovsqlite-server.c -@@ -44,7 +44,9 @@ - # include "sql-init.h" - # include "sql-main.h" - --# define OVSQLITE_DB_FILE "ovsqlite.db" -+# define OVSQLITE_DB_FILE "ovsqlite.db" -+# define OVSQLITE_BUSY_TIMEOUT_MS 999999999 -+# define CHECKPOINT_BUSY_TIMEOUT_MS 10000 - - # ifdef HAVE_ZLIB - -@@ -122,10 +124,12 @@ static unsigned long pagesize; - static unsigned long cachesize; - static struct timeval transaction_time_limit = {10, 0}; - static unsigned long transaction_row_limit = 10000; -+static unsigned long wal_checkpoint_threshold = 1000; - - static bool in_transaction; - static unsigned int transaction_rowcount; - static struct timeval next_commit; -+static int wal_pages; - - - static void -@@ -398,6 +402,8 @@ load_config(void) - } - config_param_unsigned_number(top, "transrowlimit", - &transaction_row_limit); -+ config_param_unsigned_number(top, "walcheckpointthreshold", -+ &wal_checkpoint_threshold); - - config_free(top); - } -@@ -539,6 +545,40 @@ make_dict(char const *groupname, int groupname_len, uint64_t artnum) - - # endif /* HAVE_ZLIB */ - -+static int -+wal_hook(void *data UNUSED, sqlite3 *db UNUSED, const char *dbname UNUSED, -+ int pages) -+{ -+ wal_pages = pages; -+ return SQLITE_OK; -+} -+ -+static void -+checkpoint_wal(void) -+{ -+ int rc; -+ -+ /* Force a TRUNCATE checkpoint: wait for readers to finish, then write -+ * all WAL content back to the database file and reset the WAL. This -+ * is no worse than the non-WAL path where every commit takes an -+ * exclusive lock. Use a short busy_timeout so the server doesn't -+ * stall indefinitely, the normal timeout is ~31 years for writer -+ * locks. */ -+ sqlite3_busy_timeout(connection, CHECKPOINT_BUSY_TIMEOUT_MS); -+ rc = sqlite3_wal_checkpoint_v2(connection, NULL, -+ SQLITE_CHECKPOINT_TRUNCATE, NULL, NULL); -+ sqlite3_busy_timeout(connection, OVSQLITE_BUSY_TIMEOUT_MS); -+ if (rc == SQLITE_OK) { -+ wal_pages = 0; -+ } else if (rc == SQLITE_BUSY) { -+ notice("WAL checkpoint: readers did not finish within timeout" -+ " (%d pages)", wal_pages); -+ } else { -+ warn("WAL checkpoint failed: %s (%d pages)", -+ sqlite3_errstr(rc), wal_pages); -+ } -+} -+ - static void - open_db(void) - { -@@ -667,6 +707,7 @@ open_db(void) - warn("cannot enable WAL mode: %s", errmsg); - sqlite3_free(errmsg); - } -+ sqlite3_wal_hook(connection, wal_hook, NULL); - } - } - -@@ -679,8 +720,8 @@ close_db(void) - - /* Use PASSIVE checkpoint first — it never blocks and is safe even - * when direct reader processes (nnrpd) still have the database open. -- * The prepared statement uses TRUNCATE which would hang waiting for -- * all readers to disconnect (busy_timeout is ~31 years). */ -+ * Avoid TRUNCATE directly here as it would hang waiting for all -+ * readers to disconnect (busy_timeout is ~31 years). */ - rc = sqlite3_wal_checkpoint_v2(connection, NULL, - SQLITE_CHECKPOINT_PASSIVE, - &wal_frames, &checkpointed); -@@ -2129,6 +2170,9 @@ mainloop(void) - } - nap = δ - } else { -+ /* Idle: no active transaction. Checkpoint WAL if needed. */ -+ if (use_wal && wal_pages >= (int) wal_checkpoint_threshold) -+ checkpoint_wal(); - nap = NULL; - } - read_fds_out = read_fds; -@@ -2183,10 +2227,12 @@ main(int argc, char **argv) - } - } - if (debug) { -+ message_handlers_notice(1, message_log_stderr); - message_handlers_warn(1, message_log_stderr); - message_handlers_die(1, message_log_stderr); - } else { - openlog("ovsqlite-server", L_OPENLOG_FLAGS | LOG_PID, LOG_INN_PROG); -+ message_handlers_notice(1, message_log_syslog_notice); - message_handlers_warn(1, message_log_syslog_err); - message_handlers_die(1, message_log_syslog_err); - } -diff --git storage/ovsqlite/sql-main.sql storage/ovsqlite/sql-main.sql -index ab5495a49..ff73f1b0e 100644 ---- storage/ovsqlite/sql-main.sql -+++ storage/ovsqlite/sql-main.sql -@@ -40,9 +40,6 @@ rollback to savepoint article_group; - -- .delete_journal - pragma journal_mode = 'DELETE'; - ---- .checkpoint_wal --pragma wal_checkpoint(TRUNCATE); -- - -- .add_group - insert into groupinfo (groupname, flag_alias, low, high) - values(?1, ?2, ?3, ?4); - -From 755ab163eb124658959a8e4168732e6a7f7dc300 Mon Sep 17 00:00:00 2001 -From: Kevin Bowling -Date: Wed, 6 May 2026 23:38:41 -0700 -Subject: [PATCH 5/7] fix indentation in ovsqlite.c - ---- - storage/ovsqlite/ovsqlite.c | 66 ++++++++++++++++++------------------- - 1 file changed, 33 insertions(+), 33 deletions(-) - -diff --git storage/ovsqlite/ovsqlite.c storage/ovsqlite/ovsqlite.c -index 557d9710b..f8e465525 100644 ---- storage/ovsqlite/ovsqlite.c -+++ storage/ovsqlite/ovsqlite.c -@@ -726,40 +726,40 @@ ovsqlite_groupstats(const char *group, int *low, int *high, int *count, - return false; - } - groupname_len = strlen(group); -- start_request(request_get_groupinfo); -- pack_now(request, &groupname_len, sizeof groupname_len); -- pack_now(request, group, groupname_len); -- finish_request(); -- if (!write_request()) -- return false; -+ start_request(request_get_groupinfo); -+ pack_now(request, &groupname_len, sizeof groupname_len); -+ pack_now(request, group, groupname_len); -+ finish_request(); -+ if (!write_request()) -+ return false; - -- if (!read_response()) -- return false; -- code = start_response(); -- if (code != response_groupinfo) -- return false; -- if (!unpack_now(response, &r_low, sizeof r_low)) -- return false; -- if (!unpack_now(response, &r_high, sizeof r_high)) -- return false; -- if (!unpack_now(response, &r_count, sizeof r_count)) -- return false; -- if (!unpack_now(response, &flag_alias_len, sizeof flag_alias_len)) -- return false; -- flag_alias = unpack_later(response, flag_alias_len); -- if (!flag_alias) -- return false; -- if (!finish_response()) -- return false; -- if (low) -- *low = r_low; -- if (high) -- *high = r_high; -- if (count) -- *count = r_count; -- if (flag) -- *flag = *flag_alias; -- return true; -+ if (!read_response()) -+ return false; -+ code = start_response(); -+ if (code != response_groupinfo) -+ return false; -+ if (!unpack_now(response, &r_low, sizeof r_low)) -+ return false; -+ if (!unpack_now(response, &r_high, sizeof r_high)) -+ return false; -+ if (!unpack_now(response, &r_count, sizeof r_count)) -+ return false; -+ if (!unpack_now(response, &flag_alias_len, sizeof flag_alias_len)) -+ return false; -+ flag_alias = unpack_later(response, flag_alias_len); -+ if (!flag_alias) -+ return false; -+ if (!finish_response()) -+ return false; -+ if (low) -+ *low = r_low; -+ if (high) -+ *high = r_high; -+ if (count) -+ *count = r_count; -+ if (flag) -+ *flag = *flag_alias; -+ return true; - } - - bool - -From d23b3f84aebe3fb49af0b42b7cfaa7a7ebae5a26 Mon Sep 17 00:00:00 2001 -From: Kevin Bowling -Date: Wed, 6 May 2026 23:42:15 -0700 -Subject: [PATCH 6/7] Add walcheckpointthreshold to ovsqlite.conf - ---- - samples/ovsqlite.conf | 11 +++++++++++ - 1 file changed, 11 insertions(+) - -diff --git samples/ovsqlite.conf samples/ovsqlite.conf -index 319b6d656..4ac719d1d 100644 ---- samples/ovsqlite.conf -+++ samples/ovsqlite.conf -@@ -46,6 +46,17 @@ - # The default value is 8000 KB (about 8 MB). - #readercachesize: 8000 - -+# The WAL page count at which ovsqlite-server forces a checkpoint. -+# When the WAL grows beyond this many pages, the server performs a -+# TRUNCATE checkpoint during its next idle period, waiting up to -+# 10 seconds for readers to finish their current queries before -+# flushing WAL content back to the database file. This prevents -+# unbounded WAL growth. If readers do not finish within the timeout, -+# the checkpoint is skipped and retried on the next idle cycle. -+# Only effective when walmode is enabled. -+# The default value is 1000 pages. -+#walcheckpointthreshold: 1000 -+ - # The maximum number of article rows that can be inserted or deleted - # in a single SQL transaction. - # The default value is 10000 articles. - -From 919cf653b54c72ef4bf1563f3fb6bbdf9a193a3a Mon Sep 17 00:00:00 2001 -From: Kevin Bowling -Date: Thu, 7 May 2026 02:08:32 -0700 -Subject: [PATCH 7/7] ovsqlite: Set journal mode from open_db() based on - configuration - -Move pragma journal_mode out of the sql-main.sql preamble and into -open_db() where it is set conditionally: WAL when walmode is true, -PERSIST otherwise. Previously the preamble unconditionally set -PERSIST mode before open_db() re-set WAL, causing a pointless -WAL->PERSIST->WAL transition on every startup (or silently failing -if direct readers held the WAL open). ---- - storage/ovsqlite/ovsqlite-server.c | 8 ++++++++ - storage/ovsqlite/sql-main.sql | 2 -- - 2 files changed, 8 insertions(+), 2 deletions(-) - -diff --git storage/ovsqlite/ovsqlite-server.c storage/ovsqlite/ovsqlite-server.c -index 623b17db0..c1e89dd82 100644 ---- storage/ovsqlite/ovsqlite-server.c -+++ storage/ovsqlite/ovsqlite-server.c -@@ -708,6 +708,14 @@ open_db(void) - sqlite3_free(errmsg); - } - sqlite3_wal_hook(connection, wal_hook, NULL); -+ } else { -+ status = sqlite3_exec(connection, -+ "pragma journal_mode = 'PERSIST';", -+ 0, NULL, &errmsg); -+ if (status != SQLITE_OK) { -+ warn("cannot set PERSIST journal mode: %s", errmsg); -+ sqlite3_free(errmsg); -+ } - } - } - -diff --git storage/ovsqlite/sql-main.sql storage/ovsqlite/sql-main.sql -index ff73f1b0e..435c3ea01 100644 ---- storage/ovsqlite/sql-main.sql -+++ storage/ovsqlite/sql-main.sql -@@ -1,7 +1,5 @@ - pragma foreign_keys = 1; - --pragma journal_mode = 'PERSIST'; -- - pragma busy_timeout = 999999999; - - -- .random diff --git a/news/inn-current/files/pr339.patch b/news/inn-current/files/pr339.patch deleted file mode 100644 index 06b076d583ec..000000000000 --- a/news/inn-current/files/pr339.patch +++ /dev/null @@ -1,1238 +0,0 @@ -From d0c379ee3f325f4a80dada6c3129c23496b1c83e Mon Sep 17 00:00:00 2001 -From: Kevin Bowling -Date: Wed, 6 May 2026 22:06:14 -0700 -Subject: [PATCH] expireover: Add bloom filter for fast history existence - checks - -expireover checks every article in the overview database against the -history file to detect orphaned entries. This requires a per-article -HISlookup, which does random pread() calls into the DBZ index and -history text file. On large spools (1B+ articles), this takes months. - -Add a bloom filter that is built from a single sequential HISwalk of -the history file at startup. The bloom filter acts as a positive-only -cache in OVhisthasmsgid: bloom hits skip the slow HISlookup, bloom -misses fall through to HISlookup for correctness. False positives -are benign (an orphaned overview entry survives one extra cycle). - -The bloom filter is controlled by the new inn.conf parameter -expirebloomfp, which specifies the false positive rate as a reciprocal -(default 10000 = 0.01%). Setting it to 0 disables the bloom filter. -Memory usage is approximately 20 bits per article (48 MB for 20M -articles, 2.4 GB for 1B articles). - -Changes: -- Add lib/bloom.c and include/inn/bloom.h (bloom filter implementation - using enhanced double hashing, Kirsch & Mitzenmacher 2006) -- Extend HISwalk callback signature to include the message-ID HASH - (HISwalk has had zero callers since it was added in 2001) -- Set hisv6_walk ignore=true so corrupt lines don't abort the walk -- Add OVTOKENCACHE to OVctl for passing the bloom filter to - OVhisthasmsgid -- Add expirebloomfp to innconf -- Add unit tests (lib/bloom-t.c) and integration tests - (lib/bloom-hiswalk-t.c) ---- - .gitignore | 2 + - MANIFEST | 4 + - doc/pod/expireover.pod | 16 +++ - doc/pod/inn.conf.pod | 20 ++++ - doc/pod/libinnhist.pod | 15 ++- - expire/expireover.c | 61 +++++++++- - history/his.c | 3 +- - history/hisinterface.h | 3 +- - history/hisv6/hisv6-private.h | 3 +- - history/hisv6/hisv6.c | 15 ++- - history/hisv6/hisv6.h | 4 +- - include/inn/bloom.h | 69 +++++++++++ - include/inn/history.h | 4 +- - include/inn/innconf.h | 1 + - include/inn/ov.h | 3 +- - lib/Makefile | 4 +- - lib/bloom.c | 213 ++++++++++++++++++++++++++++++++++ - lib/innconf.c | 1 + - samples/inn.conf.in | 1 + - storage/expire.c | 18 +++ - storage/ov.c | 3 + - storage/ovinterface.h | 2 + - support/mkmanifest | 2 + - tests/Makefile | 9 +- - tests/TESTS | 2 + - tests/lib/bloom-hiswalk-t.c | 169 +++++++++++++++++++++++++++ - tests/lib/bloom-t.c | 128 ++++++++++++++++++++ - 27 files changed, 753 insertions(+), 22 deletions(-) - create mode 100644 include/inn/bloom.h - create mode 100644 lib/bloom.c - create mode 100644 tests/lib/bloom-hiswalk-t.c - create mode 100644 tests/lib/bloom-t.c - -diff --git .gitignore .gitignore -index 4fc5e98b0..f933b3c2d 100644 ---- .gitignore -+++ .gitignore -@@ -195,6 +195,8 @@ - /tests/innd/chan.t - /tests/lib/artnumber.t - /tests/lib/asprintf.t -+/tests/lib/bloom.t -+/tests/lib/bloom-hiswalk.t - /tests/lib/buffer.t - /tests/lib/canlock.t - /tests/lib/concat.t -diff --git MANIFEST MANIFEST -index 094b0a020..70bef9d76 100644 ---- MANIFEST -+++ MANIFEST -@@ -412,6 +412,7 @@ include/Makefile Makefile for header files - include/conffile.h Header file for reading *.conf files - include/config.h.in Template configuration data - include/inn Installed header files (Directory) -+include/inn/bloom.h Header file for bloom filter - include/inn/buffer.h Header file for reusable counted buffers - include/inn/concat.h Header file for string concatenation - include/inn/confparse.h Header file for configuration parser -@@ -518,6 +519,7 @@ lib/Makefile Makefile for library - lib/argparse.c Functions for parsing arguments - lib/artnumber.c Manipulation of article numbers - lib/asprintf.c asprintf replacement -+lib/bloom.c Bloom filter implementation - lib/buffer.c Reusable counted buffer - lib/canlock.c Routines for Cancel-Lock - lib/cleanfrom.c Clean out a From line -@@ -938,6 +940,8 @@ tests/innd/fakeinnd.c Provide symbols defined by innd/innd.c - tests/lib Test suite for libinn (Directory) - tests/lib/artnumber-t.c Tests for lib/artnumber.c - tests/lib/asprintf-t.c Tests for lib/asprintf.c -+tests/lib/bloom-hiswalk-t.c Integration test for bloom filter with HISwalk -+tests/lib/bloom-t.c Tests for lib/bloom.c - tests/lib/buffer-t.c Tests for lib/buffer.c - tests/lib/canlock-t.c Tests for lib/canlock.c - tests/lib/concat-t.c Tests for lib/concat.c -diff --git doc/pod/expireover.pod doc/pod/expireover.pod -index 239b354b6..318ad73e5 100644 ---- doc/pod/expireover.pod -+++ doc/pod/expireover.pod -@@ -39,6 +39,22 @@ By default, B purges all overview information for newsgroups - that have been removed from the server; this behavior is suppressed if - B<-f> is given. - -+To speed up the existence check for each article, B builds -+a bloom filter from the history file at startup. This replaces -+per-article random I/O into the history file with a single sequential -+read, which is critical for large spools. The bloom filter is a -+positive-only cache: if it reports an article probably exists, the -+slow history lookup is skipped. If it reports the article is not found, -+B falls back to a direct history lookup for correctness. -+False positives (the bloom filter incorrectly reporting an article -+exists) are benign; the orphaned overview entry will be cleaned up on -+the next expiration run. -+ -+The false positive rate and memory usage of the bloom filter are -+controlled by the I setting in F. Setting it -+to C<0> disables the bloom filter. The bloom filter is also disabled -+when the B<-s> flag is used. -+ - =head1 OPTIONS - - =over 4 -diff --git doc/pod/inn.conf.pod doc/pod/inn.conf.pod -index a65fe4abd..cfb446c48 100644 ---- doc/pod/inn.conf.pod -+++ doc/pod/inn.conf.pod -@@ -526,6 +526,26 @@ will run much faster, but reading news from the system will be impossible - true, I must also be set. This is a boolean value and the - default is true. - -+=item I -+ -+Controls the bloom filter used by B to accelerate overview -+expiration. The value is the reciprocal of the desired false positive -+rate: for example, C<10000> means a 1-in-10,000 (0.01%) false positive -+rate. Higher values use more memory but produce fewer false positives. -+At the default of C<10000>, memory usage is approximately 20 bits per -+article in the history file (e.g., 48S for 20 million articles, -+2.4S for 1 billion articles). -+ -+Setting this to C<0> disables the bloom filter entirely, falling back -+to per-article history lookups (the pre-existing behavior). This is -+not recommended for large spools as it results in random I/O into the -+history file for every article in the overview database. -+ -+The bloom filter has no effect when B is run with the B<-s> -+flag (which forces a filesystem stat of every article). -+ -+This is a non-negative integer and the default is C<10000>. -+ - =item I - - Besides the seven standard overview fields (which are in order C, -diff --git doc/pod/libinnhist.pod doc/pod/libinnhist.pod -index 05a4acc8d..89b51b42c 100644 ---- doc/pod/libinnhist.pod -+++ doc/pod/libinnhist.pod -@@ -67,8 +67,9 @@ his - routines for managing INN history - - bool HISwalk(struct history *history, const char *reason, - void *cookie, -- bool (*callback)(void *cookie, time_t arrived, -- time_t posted, time_t expires, -+ bool (*callback)(void *cookie, const HASH *hash, -+ time_t arrived, time_t posted, -+ time_t expires, - const TOKEN *token)); - - struct histstats HISstats(struct history *history); -@@ -210,10 +211,12 @@ unspecified. - - B provides an iteration function for the specified I - database. For every entry in the history database, I is --invoked, passing the I, arrival, posting, and expiry times, in --addition to the token associated with the entry. If the I() --returns B the iteration is aborted and B returns --B to the caller. -+invoked, passing the I, the message-ID I, arrival, -+posting, and expiry times, in addition to the token associated with -+the entry. If the entry has no storage token (a remembered -+message-ID), I is B. If the I() returns -+B the iteration is aborted and B returns B to -+the caller. Malformed history lines are silently skipped. - - To process the entire database in the presence of a running server, - I may be passed; if this argument is not B, it is used -diff --git expire/expireover.c expire/expireover.c -index 0548cae53..3eccae081 100644 ---- expire/expireover.c -+++ expire/expireover.c -@@ -14,6 +14,10 @@ - #include - #include - -+#include -+ -+#include "inn/bloom.h" -+#include "inn/history.h" - #include "inn/innconf.h" - #include "inn/libinn.h" - #include "inn/messages.h" -@@ -45,6 +49,22 @@ fatal_signal(int sig) - } - - -+/* -+** Callback for HISwalk that adds history entries with storage tokens to the -+** bloom filter. Entries without tokens (remembered message-IDs) are skipped -+** so that OVhisthasmsgid correctly identifies them as missing. -+*/ -+static bool -+build_bloom_cb(void *cookie, const HASH *hash, -+ time_t arrived UNUSED, time_t posted UNUSED, -+ time_t expires UNUSED, const TOKEN *token) -+{ -+ if (token != NULL) -+ bloom_add(cookie, hash); -+ return true; -+} -+ -+ - int - main(int argc, char *argv[]) - { -@@ -60,6 +80,8 @@ main(int argc, char *argv[]) - bool purge_deleted = false; - bool always_stat = false; - struct history *history; -+ struct bloom_filter *bloom = NULL; -+ struct bloom_filter *null_bloom = NULL; - - /* First thing, set up logging and our identity. */ - openlog("expireover", L_OPENLOG_FLAGS | LOG_PID, LOG_INN_PROG); -@@ -181,12 +203,43 @@ main(int argc, char *argv[]) - if (!OVctl(OVSTATALL, &always_stat)) - die("can't configure overview stat behavior"); - -- /* We want to be careful about being interrupted from this point on, so -- set up our signal handlers. */ -+ /* Set up signal handlers before the bloom walk, which can take several -+ minutes on very large history files. */ - xsignal(SIGTERM, fatal_signal); - xsignal(SIGINT, fatal_signal); - xsignal(SIGHUP, fatal_signal); - -+ /* Build a bloom filter from the history file for fast existence checks. -+ This replaces millions of random pread() calls into the history file -+ with a single sequential read, making expireover feasible on large -+ spools (1B+ articles). The bloom filter is used as a positive-only -+ cache: hits skip the slow history lookup, misses fall through to -+ HISlookup for correctness (handles articles added after the walk). */ -+ if (innconf->expirebloomfp > 0 && !always_stat) { -+ struct stat st; -+ char *histpath; -+ size_t estimated = 0; -+ /* Minimum history line: 34 (hash) + 1 (tab) + 1 (arrived) -+ * + 1 (newline). Dividing file size by this gives a conservative -+ * overestimate of entries, which is what we want for bloom sizing. */ -+ const size_t min_history_line = 37; -+ -+ histpath = concatpath(innconf->pathdb, INN_PATH_HISTORY); -+ if (stat(histpath, &st) == 0) -+ estimated = st.st_size / min_history_line; -+ else -+ warn("can't stat %s, bloom filter will be undersized", histpath); -+ bloom = bloom_create(estimated, innconf->expirebloomfp); -+ if (!HISwalk(history, NULL, bloom, build_bloom_cb)) { -+ warn("can't walk history for bloom filter, using per-article" -+ " lookups"); -+ bloom_free(bloom); -+ bloom = NULL; -+ } -+ OVctl(OVTOKENCACHE, &bloom); -+ free(histpath); -+ } -+ - /* Loop through each line of the input file and process each group, - writing data to the lowmark file if desired. */ - line = QIOread(qp); -@@ -212,6 +265,10 @@ main(int argc, char *argv[]) - warn("can't expire deleted newsgroups"); - - /* Close everything down in an orderly fashion. */ -+ if (bloom) { -+ OVctl(OVTOKENCACHE, &null_bloom); -+ bloom_free(bloom); -+ } - QIOclose(qp); - OVclose(); - SMshutdown(); -diff --git history/his.c history/his.c -index 03127a358..f8dc7b8c7 100644 ---- history/his.c -+++ history/his.c -@@ -328,7 +328,8 @@ HISreplace(struct history *h, const char *key, time_t arrived, time_t posted, - - bool - HISwalk(struct history *h, const char *reason, void *cookie, -- bool (*callback)(void *, time_t, time_t, time_t, const TOKEN *)) -+ bool (*callback)(void *, const HASH *, time_t, time_t, time_t, -+ const TOKEN *)) - { - bool r; - -diff --git history/hisinterface.h history/hisinterface.h -index ce3ae72f1..cda2cd16b 100644 ---- history/hisinterface.h -+++ history/hisinterface.h -@@ -6,6 +6,7 @@ - #define HISINTERFACE_H - - #include "config.h" -+#include "inn/libinn.h" - #include - - struct token; -@@ -27,7 +28,7 @@ typedef struct hismethod { - bool (*expire)(void *, const char *, const char *, bool, void *, time_t, - bool (*)(void *, time_t, time_t, time_t, struct token *)); - bool (*walk)(void *, const char *, void *, -- bool (*)(void *, time_t, time_t, time_t, -+ bool (*)(void *, const HASH *, time_t, time_t, time_t, - const struct token *)); - bool (*remember)(void *, const char *, time_t, time_t); - bool (*ctl)(void *, int, void *); -diff --git history/hisv6/hisv6-private.h history/hisv6/hisv6-private.h -index 003d0d8dc..be29d2b01 100644 ---- history/hisv6/hisv6-private.h -+++ history/hisv6/hisv6-private.h -@@ -64,7 +64,8 @@ struct hisv6 { - struct hisv6_walkstate { - union { - bool (*expire)(void *, time_t, time_t, time_t, TOKEN *); -- bool (*walk)(void *, time_t, time_t, time_t, const TOKEN *); -+ bool (*walk)(void *, const HASH *, time_t, time_t, time_t, -+ const TOKEN *); - } cb; - void *cookie; - bool paused; -diff --git history/hisv6/hisv6.c history/hisv6/hisv6.c -index 72045cf2d..686dc50f6 100644 ---- history/hisv6/hisv6.c -+++ history/hisv6/hisv6.c -@@ -1071,14 +1071,14 @@ hisv6_traverse(struct hisv6 *h, struct hisv6_walkstate *cookie, - ** parameters the user callback expects - **/ - static bool --hisv6_traversecb(struct hisv6 *h UNUSED, void *cookie, const HASH *hash UNUSED, -+hisv6_traversecb(struct hisv6 *h UNUSED, void *cookie, const HASH *hash, - time_t arrived, time_t posted, time_t expires, - const TOKEN *token) - { - struct hisv6_walkstate *hiscookie = cookie; - -- return (*hiscookie->cb.walk)(hiscookie->cookie, arrived, posted, expires, -- token); -+ return (*hiscookie->cb.walk)(hiscookie->cookie, hash, arrived, posted, -+ expires, token); - } - - -@@ -1087,7 +1087,8 @@ hisv6_traversecb(struct hisv6 *h UNUSED, void *cookie, const HASH *hash UNUSED, - */ - bool - hisv6_walk(void *history, const char *reason, void *cookie, -- bool (*callback)(void *, time_t, time_t, time_t, const TOKEN *)) -+ bool (*callback)(void *, const HASH *, time_t, time_t, time_t, -+ const TOKEN *)) - { - struct hisv6 *h = history; - struct hisv6_walkstate hiscookie; -@@ -1099,7 +1100,11 @@ hisv6_walk(void *history, const char *reason, void *cookie, - hiscookie.cookie = cookie; - hiscookie.new = NULL; - hiscookie.paused = false; -- hiscookie.ignore = false; -+ /* Ignore malformed history lines during walk. The walk is a read-only -+ operation (e.g., building a bloom filter for expireover); aborting -+ over one corrupt line in a potentially 180 GB file would be -+ catastrophic. expire is what fixes corrupt history entries. */ -+ hiscookie.ignore = true; - - r = hisv6_traverse(h, &hiscookie, reason, hisv6_traversecb); - -diff --git history/hisv6/hisv6.h history/hisv6/hisv6.h -index 30fcae787..8f9564f9c 100644 ---- history/hisv6/hisv6.h -+++ history/hisv6/hisv6.h -@@ -5,6 +5,8 @@ - #ifndef HISV6_H - #define HISV6_H - -+#include "inn/libinn.h" -+ - struct token; - struct histopts; - struct history; -@@ -32,7 +34,7 @@ bool hisv6_expire(void *, const char *, const char *, bool, void *, - struct token *)); - - bool hisv6_walk(void *, const char *, void *, -- bool (*)(void *, time_t, time_t, time_t, -+ bool (*)(void *, const HASH *, time_t, time_t, time_t, - const struct token *)); - - const char *hisv6_error(void *); -diff --git include/inn/bloom.h include/inn/bloom.h -new file mode 100644 -index 000000000..d0f31d7ea ---- /dev/null -+++ include/inn/bloom.h -@@ -0,0 +1,69 @@ -+/* -+** Bloom filter for fast set membership testing. -+** -+** A space-efficient probabilistic data structure that can test whether -+** an element is a member of a set. False positive matches are possible, -+** but false negatives are not: a query returns either "possibly in set" -+** or "definitely not in set." -+** -+** Uses enhanced double hashing (Kirsch & Mitzenmacher 2006) to derive -+** multiple hash positions from a single HASH value. -+*/ -+ -+#ifndef INN_BLOOM_H -+#define INN_BLOOM_H -+ -+#include "inn/libinn.h" -+#include "inn/portable-macros.h" -+#include "inn/portable-stdbool.h" -+ -+#include -+ -+BEGIN_DECLS -+ -+/* The layout of this struct is entirely internal to the implementation. */ -+struct bloom_filter; -+ -+/* -+** Create a new bloom filter sized for the given number of estimated entries -+** and false positive rate expressed as a reciprocal (e.g., 10000 means -+** 1-in-10,000 or 0.01% false positive rate). Uses xmalloc internally, -+** so dies on allocation failure. -+*/ -+struct bloom_filter *bloom_create(size_t estimated_entries, unsigned long fp_inv); -+ -+/* -+** Add a HASH to the bloom filter. -+*/ -+void bloom_add(struct bloom_filter *bf, const HASH *hash); -+ -+/* -+** Check whether a HASH is possibly in the bloom filter. Returns true if -+** the element is probably in the set (with false positive rate as configured), -+** or false if the element is definitely not in the set. -+*/ -+bool bloom_check(const struct bloom_filter *bf, const HASH *hash); -+ -+/* -+** Free a bloom filter and all associated memory. Safe to call with NULL. -+*/ -+void bloom_free(struct bloom_filter *bf); -+ -+/* -+** Return the number of entries that have been added to the bloom filter. -+*/ -+size_t bloom_count(const struct bloom_filter *bf); -+ -+/* -+** Return the number of hash functions (k) used by the bloom filter. -+*/ -+unsigned int bloom_nhash(const struct bloom_filter *bf); -+ -+/* -+** Return the total number of bits (m) in the bloom filter. -+*/ -+size_t bloom_bits(const struct bloom_filter *bf); -+ -+END_DECLS -+ -+#endif /* INN_BLOOM_H */ -diff --git include/inn/history.h include/inn/history.h -index f14d9536f..5be46d984 100644 ---- include/inn/history.h -+++ include/inn/history.h -@@ -5,6 +5,7 @@ - #ifndef INN_HISTORY_H - #define INN_HISTORY_H - -+#include "inn/libinn.h" - #include "inn/macros.h" - #include "inn/portable-stdbool.h" - #include -@@ -96,7 +97,8 @@ bool HISexpire(struct history *, const char *, const char *, bool, void *, - time_t, - bool (*)(void *, time_t, time_t, time_t, struct token *)); - bool HISwalk(struct history *, const char *, void *, -- bool (*)(void *, time_t, time_t, time_t, const struct token *)); -+ bool (*)(void *, const HASH *, time_t, time_t, time_t, -+ const struct token *)); - struct histstats HISstats(struct history *); - const char *HISerror(struct history *); - bool HISctl(struct history *, int, void *); -diff --git include/inn/innconf.h include/inn/innconf.h -index f1add2628..a24c2d1d8 100644 ---- include/inn/innconf.h -+++ include/inn/innconf.h -@@ -72,6 +72,7 @@ struct innconf { - /* Article Storage */ - unsigned long cnfscheckfudgesize; /* Additional CNFS integrity checking */ - bool enableoverview; /* Store overview info for articles? */ -+ unsigned long expirebloomfp; /* Bloom filter FP rate 1/N (0 = disabled) */ - struct vector *extraoverviewadvertised; /* Extra overview fields for LIST - OVERVIEW.FMT */ - struct vector -diff --git include/inn/ov.h include/inn/ov.h -index 2f2259276..ef7ba8541 100644 ---- include/inn/ov.h -+++ include/inn/ov.h -@@ -19,7 +19,8 @@ typedef enum { - OVSTATICSEARCH, - OVSTATALL, - OVCACHEKEEP, -- OVCACHEFREE -+ OVCACHEFREE, -+ OVTOKENCACHE - } OVCTLTYPE; - #define OV_NOSPACE 100 - typedef enum { -diff --git lib/Makefile lib/Makefile -index ae22203e5..0f4fb8ec5 100644 ---- lib/Makefile -+++ lib/Makefile -@@ -12,8 +12,8 @@ top = .. - CFLAGS = $(GCFLAGS) - - # The base library files that are always compiled and included. --SOURCES = argparse.c artnumber.c buffer.c cleanfrom.c clientactive.c \ -- clientlib.c \ -+SOURCES = argparse.c artnumber.c bloom.c buffer.c cleanfrom.c \ -+ clientactive.c clientlib.c \ - commands.c concat.c conffile.c confparse.c \ - date.c dbz.c defdist.c dispatch.c fdflag.c fdlimit.c \ - getfqdn.c getmodaddr.c hash.c hashtab.c headers.c hex.c \ -diff --git lib/bloom.c lib/bloom.c -new file mode 100644 -index 000000000..cfd669e7a ---- /dev/null -+++ lib/bloom.c -@@ -0,0 +1,213 @@ -+/* -+** Bloom filter implementation. -+** -+** A space-efficient probabilistic data structure for set membership testing. -+** Uses enhanced double hashing (Kirsch & Mitzenmacher 2006) to derive k -+** independent bit positions from the two 64-bit halves of a 16-byte MD5 -+** HASH value. -+** -+** Sizing uses the standard formulas: -+** m = -n * ln(p) / (ln(2))^2 (optimal number of bits) -+** k = (m / n) * ln(2) (optimal number of hash functions) -+** where n = estimated entries, p = desired false positive rate. -+** -+** The create function takes the false positive rate as 1/fp_inv (an integer -+** reciprocal) to avoid floating point in the library. Common values: -+** fp_inv = 100 => 1% FP, ~10 bits/entry, k=7 -+** fp_inv = 1000 => 0.1% FP, ~15 bits/entry, k=10 -+** fp_inv = 10000 => 0.01% FP, ~20 bits/entry, k=14 -+** fp_inv = 100000 => 0.001% FP, ~24 bits/entry, k=17 -+*/ -+ -+#include "portable/system.h" -+ -+#include -+ -+#include "inn/bloom.h" -+#include "inn/messages.h" -+#include "inn/xmalloc.h" -+ -+struct bloom_filter { -+ uint8_t *bits; /* bit array */ -+ size_t nbits; /* total bits (m) */ -+ unsigned int nhash; /* number of hash functions (k) */ -+ size_t count; /* entries added */ -+}; -+ -+ -+/* -+** Compute k bit positions for a given HASH using enhanced double hashing. -+** h(i) = (h1 + i * h2) mod m, where h1 and h2 are the two 64-bit halves -+** of the MD5 hash. Unsigned overflow on (h1 + i * h2) is well-defined -+** in C and acts as additional mixing before the final mod m. -+*/ -+static void -+bloom_positions(const struct bloom_filter *bf, const HASH *hash, -+ size_t *positions) -+{ -+ uint64_t h1, h2; -+ unsigned int i; -+ -+ memcpy(&h1, hash->hash, 8); -+ memcpy(&h2, hash->hash + 8, 8); -+ for (i = 0; i < bf->nhash; i++) -+ positions[i] = (size_t) ((h1 + (uint64_t) i * h2) % bf->nbits); -+} -+ -+ -+/* -+** Pre-computed bits-per-entry and optimal k for common false positive rates. -+** Values are ceil(-ln(1/fp_inv) / ln(2)^2) and ceil(bits_per_entry * ln(2)). -+** For fp_inv values not in the table, we interpolate conservatively. -+*/ -+static const struct { -+ unsigned long fp_inv; -+ unsigned int bits_per_entry; -+ unsigned int nhash; -+} bloom_params[] = { -+ { 10, 5, 4 }, /* 10% FP */ -+ { 20, 7, 5 }, -+ { 50, 9, 6 }, -+ { 100, 10, 7 }, /* 1% FP */ -+ { 200, 12, 8 }, -+ { 500, 13, 9 }, -+ { 1000, 15, 10 }, /* 0.1% FP */ -+ { 2000, 16, 11 }, -+ { 5000, 18, 13 }, -+ { 10000, 20, 14 }, /* 0.01% FP */ -+ { 20000, 21, 15 }, -+ { 50000, 23, 16 }, -+ { 100000, 24, 17 }, /* 0.001% FP */ -+ { 1000000, 29, 20 }, /* 0.0001% FP */ -+ { 10000000, 34, 24 }, -+}; -+#define BLOOM_NPARAMS (sizeof(bloom_params) / sizeof(bloom_params[0])) -+#define BLOOM_MAX_NHASH 24 /* must match max nhash in bloom_params table */ -+ -+/* Maximum bloom filter size on 32-bit platforms where size_t overflow -+ * is a real concern. On 64-bit, there is no cap, xmalloc will die if -+ * the system doesn't have enough memory, which is the correct behavior -+ * for a batch job. On 32-bit, cap at SIZE_MAX/16 so that the conversion -+ * to bits (multiply by 8) stays within size_t. */ -+#if SIZE_MAX <= UINT32_MAX -+# define BLOOM_MAX_BITS ((SIZE_MAX / 16) * 8) -+#endif -+ -+ -+struct bloom_filter * -+bloom_create(size_t estimated_entries, unsigned long fp_inv) -+{ -+ struct bloom_filter *bf; -+ unsigned int bits_per_entry; -+ unsigned int nhash; -+ size_t nbits; -+ size_t nbytes; -+ size_t i; -+ -+ /* Look up parameters from the table. Use the entry with the smallest -+ * fp_inv that is >= the requested fp_inv (i.e., the FP rate at least as -+ * good as requested). If fp_inv exceeds all table entries, use the -+ * last (most conservative) entry. */ -+ bits_per_entry = bloom_params[BLOOM_NPARAMS - 1].bits_per_entry; -+ nhash = bloom_params[BLOOM_NPARAMS - 1].nhash; -+ for (i = 0; i < BLOOM_NPARAMS; i++) { -+ if (bloom_params[i].fp_inv >= fp_inv) { -+ bits_per_entry = bloom_params[i].bits_per_entry; -+ nhash = bloom_params[i].nhash; -+ break; -+ } -+ } -+ -+ bf = xmalloc(sizeof(*bf)); -+ -+ if (estimated_entries == 0) -+ estimated_entries = 1; -+ -+ /* Guard against size_t overflow on 32-bit platforms where -+ * bits_per_entry * estimated_entries can exceed SIZE_MAX. -+ * On 64-bit this check is effectively unreachable but harmless. -+ * On 32-bit, the cap degrades the FP rate but does not affect -+ * correctness. */ -+ if (estimated_entries > SIZE_MAX / bits_per_entry) -+#if SIZE_MAX <= UINT32_MAX -+ nbits = BLOOM_MAX_BITS; -+#else -+ die("bloom filter: entry count too large for size_t"); -+#endif -+ else -+ nbits = (size_t) bits_per_entry * estimated_entries; -+#if SIZE_MAX <= UINT32_MAX -+ if (nbits > BLOOM_MAX_BITS) -+ nbits = BLOOM_MAX_BITS; -+#endif -+ if (nbits < 64) -+ nbits = 64; -+ -+ bf->nbits = nbits; -+ bf->nhash = nhash; -+ bf->count = 0; -+ -+ nbytes = (nbits + 7) / 8; -+ bf->bits = xcalloc(nbytes, 1); -+ -+ return bf; -+} -+ -+ -+void -+bloom_add(struct bloom_filter *bf, const HASH *hash) -+{ -+ size_t positions[BLOOM_MAX_NHASH]; -+ unsigned int i; -+ -+ bloom_positions(bf, hash, positions); -+ for (i = 0; i < bf->nhash; i++) -+ bf->bits[positions[i] / 8] |= (uint8_t) (1U << (positions[i] % 8)); -+ bf->count++; -+} -+ -+ -+bool -+bloom_check(const struct bloom_filter *bf, const HASH *hash) -+{ -+ size_t positions[BLOOM_MAX_NHASH]; -+ unsigned int i; -+ -+ bloom_positions(bf, hash, positions); -+ for (i = 0; i < bf->nhash; i++) { -+ if (!(bf->bits[positions[i] / 8] & (1U << (positions[i] % 8)))) -+ return false; -+ } -+ return true; -+} -+ -+ -+void -+bloom_free(struct bloom_filter *bf) -+{ -+ if (bf == NULL) -+ return; -+ free(bf->bits); -+ free(bf); -+} -+ -+ -+size_t -+bloom_count(const struct bloom_filter *bf) -+{ -+ return bf->count; -+} -+ -+ -+unsigned int -+bloom_nhash(const struct bloom_filter *bf) -+{ -+ return bf->nhash; -+} -+ -+ -+size_t -+bloom_bits(const struct bloom_filter *bf) -+{ -+ return bf->nbits; -+} -diff --git lib/innconf.c lib/innconf.c -index 145e66b5d..199e7009a 100644 ---- lib/innconf.c -+++ lib/innconf.c -@@ -205,6 +205,7 @@ static const struct config config_table[] = { - /* The following settings are specific to the storage subsystem. */ - {K(articlemmap), BOOL(true) }, - {K(cnfscheckfudgesize), UNUMBER(0) }, -+ {K(expirebloomfp), UNUMBER(10000) }, - {K(immediatecancel), BOOL(false) }, - {K(keepmmappedthreshold), UNUMBER(1024) }, - {K(nfswriter), BOOL(false) }, -diff --git samples/inn.conf.in samples/inn.conf.in -index 2e761ff67..089b0a60e 100644 ---- samples/inn.conf.in -+++ samples/inn.conf.in -@@ -64,6 +64,7 @@ wipexpire: 10 - - cnfscheckfudgesize: 0 - enableoverview: true -+expirebloomfp: 10000 - extraoverviewadvertised: [ ] - extraoverviewhidden: [ ] - groupbaseexpiry: true -diff --git storage/expire.c storage/expire.c -index 8836805c2..abd9f4e6c 100644 ---- storage/expire.c -+++ storage/expire.c -@@ -12,6 +12,7 @@ - #include - #include - -+#include "inn/bloom.h" - #include "inn/innconf.h" - #include "inn/libinn.h" - #include "inn/ov.h" -@@ -27,6 +28,9 @@ enum KRP { - Poison - }; - -+/* Bloom filter token cache for fast OVhisthasmsgid lookups. */ -+struct bloom_filter *OVtokencache = NULL; -+ - /* Statistics */ - long EXPprocessed; - long EXPunlinked; -@@ -803,6 +807,20 @@ OVhisthasmsgid(struct history *h, const char *data) - } - if ((p = OVERGetHeader(data, Messageidindex)) == NULL) - return false; -+ -+ /* Fast path: if article is in the bloom filter, it (probably) exists. -+ * Bloom hits skip the slow history lookup. Bloom misses fall through -+ * to HISlookup to handle articles added after the filter was built. -+ * False positives (bloom says "yes" for an expired article) are benign: -+ * the orphaned overview entry is cleaned up on the next run. */ -+ if (OVtokencache) { -+ HASH hash = HashMessageID(p); -+ if (bloom_check(OVtokencache, &hash)) -+ return true; -+ } -+ -+ /* Slow path: per-article history lookup (original behavior). -+ * Only reached for bloom misses or when no bloom filter is loaded. */ - return HISlookup(h, p, NULL, NULL, NULL, NULL); - } - -diff --git storage/ov.c storage/ov.c -index a8ff34c97..62ddd061d 100644 ---- storage/ov.c -+++ storage/ov.c -@@ -399,6 +399,9 @@ OVctl(OVCTLTYPE type, void *val) - case OVSTATALL: - OVstatall = *(bool *) val; - return true; -+ case OVTOKENCACHE: -+ OVtokencache = *(struct bloom_filter **) val; -+ return true; - default: - return ((*ov.ctl)(type, val)); - } -diff --git storage/ovinterface.h storage/ovinterface.h -index 41ed2f16b..31eefb428 100644 ---- storage/ovinterface.h -+++ storage/ovinterface.h -@@ -35,10 +35,12 @@ typedef struct overview_method { - - bool OVgroupbasedexpire(TOKEN token, const char *group, const char *data, - int len, time_t arrived, time_t expires); -+struct bloom_filter; - bool OVhisthasmsgid(struct history *, const char *data); - void OVEXPremove(TOKEN token, bool deletedgroups, char **xref, int ngroups); - void OVEXPcleanup(void); - -+extern struct bloom_filter *OVtokencache; - extern time_t OVnow; - extern FILE *EXPunlinkfile; - extern bool OVignoreselfexpire; -diff --git support/mkmanifest support/mkmanifest -index c75f9a701..e079cdac1 100755 ---- support/mkmanifest -+++ support/mkmanifest -@@ -282,6 +282,8 @@ tests/innd/artparse.t - tests/innd/chan.t - tests/lib/artnumber.t - tests/lib/asprintf.t -+tests/lib/bloom.t -+tests/lib/bloom-hiswalk.t - tests/lib/buffer.t - tests/lib/canlock.t - tests/lib/concat.t -diff --git tests/Makefile tests/Makefile -index fa35b1a37..64d80256e 100644 ---- tests/Makefile -+++ tests/Makefile -@@ -17,7 +17,8 @@ LIBM_LDFLAGS = '-lm' - ## added to EXTRA. - - TESTS = authprogs/ident.t innd/artparse.t innd/chan.t lib/artnumber.t \ -- lib/asprintf.t lib/buffer.t lib/canlock.t lib/concat.t lib/conffile.t \ -+ lib/asprintf.t lib/bloom.t lib/bloom-hiswalk.t lib/buffer.t \ -+ lib/canlock.t lib/concat.t lib/conffile.t \ - lib/confparse.t lib/daemon.t lib/date.t \ - lib/dispatch.t lib/fdflag.t \ - lib/getaddrinfo.t lib/getnameinfo.t lib/hash.t \ -@@ -150,6 +151,12 @@ lib/getnameinfo.t: lib/getnameinfo.o lib/getnameinfo-t.o tap/basic.o $(LIBINN) - $(LINK) lib/getnameinfo.o lib/getnameinfo-t.o tap/basic.o \ - $(LIBINN) $(LIBS) - -+lib/bloom.t: lib/bloom-t.o tap/basic.o $(LIBINN) -+ $(LINK) lib/bloom-t.o tap/basic.o $(LIBINN) -+ -+lib/bloom-hiswalk.t: lib/bloom-hiswalk-t.o tap/basic.o $(STORAGEDEPS) -+ $(LINKDEPS) lib/bloom-hiswalk-t.o tap/basic.o $(STORAGELIBS) $(LIBS) -+ - lib/hash.t: lib/hash-t.o tap/basic.o $(LIBINN) - $(LINK) lib/hash-t.o tap/basic.o $(LIBINN) - -diff --git tests/TESTS tests/TESTS -index b995af703..8cbc076ab 100644 ---- tests/TESTS -+++ tests/TESTS -@@ -7,6 +7,8 @@ innd/artparse - innd/chan - lib/artnumber - lib/asprintf -+lib/bloom -+lib/bloom-hiswalk - lib/buffer - lib/canlock - lib/concat -diff --git tests/lib/bloom-hiswalk-t.c tests/lib/bloom-hiswalk-t.c -new file mode 100644 -index 000000000..f6244de1e ---- /dev/null -+++ tests/lib/bloom-hiswalk-t.c -@@ -0,0 +1,169 @@ -+/* Integration test: HISwalk + bloom filter for expireover token cache. -+ * -+ * Creates a temporary history file with a mix of entries (some with tokens, -+ * some remembered-only), builds a bloom filter via HISwalk, and verifies -+ * that the bloom filter correctly identifies articles with tokens vs. -+ * remembered entries. */ -+ -+#include "portable/system.h" -+ -+#include -+#include -+ -+#include "inn/bloom.h" -+#include "inn/history.h" -+#include "inn/libinn.h" -+#include "inn/messages.h" -+#include "inn/storage.h" -+#include "tap/basic.h" -+ -+#define N_WITH_TOKEN 500 -+#define N_REMEMBERED 100 -+#define N_NOT_IN_HIST 200 -+ -+ -+/* -+** Generate a deterministic message-ID from an integer. -+*/ -+static char * -+make_msgid(unsigned long n) -+{ -+ char buf[64]; -+ snprintf(buf, sizeof(buf), "", n); -+ return xstrdup(buf); -+} -+ -+ -+/* -+** HISwalk callback: add entries with tokens to the bloom filter. -+** Same logic as build_bloom_cb in expireover.c. -+*/ -+static bool -+build_bloom_cb(void *cookie, const HASH *hash, -+ time_t arrived UNUSED, time_t posted UNUSED, -+ time_t expires UNUSED, const TOKEN *token) -+{ -+ if (token != NULL) -+ bloom_add(cookie, hash); -+ return true; -+} -+ -+ -+int -+main(void) -+{ -+ struct history *h; -+ struct bloom_filter *bloom; -+ char tmpdir[64]; -+ char histpath[128]; -+ TOKEN token; -+ unsigned long i; -+ unsigned long bloom_misses; -+ unsigned long false_negatives; -+ bool walk_ok; -+ -+ test_init(8); -+ -+ /* Create temporary directory for the history database. */ -+ strlcpy(tmpdir, "bloom-hiswalk-XXXXXX", sizeof(tmpdir)); -+ if (mkdtemp(tmpdir) == NULL) -+ sysbail("can't create temp directory"); -+ snprintf(histpath, sizeof(histpath), "%s/history", tmpdir); -+ -+ /* Create and populate the history database. */ -+ h = HISopen(histpath, "hisv6", HIS_CREAT | HIS_RDWR); -+ if (h == NULL) -+ bail("can't create history at %s", histpath); -+ -+ memset(&token, 0, sizeof(token)); -+ token.type = 1; -+ -+ /* Write entries with storage tokens. */ -+ for (i = 0; i < N_WITH_TOKEN; i++) { -+ char *msgid = make_msgid(i); -+ if (!HISwrite(h, msgid, (time_t) 1000000 + i, (time_t) 1000000 + i, -+ (time_t) 0, &token)) -+ bail("can't write history entry %lu: %s", i, HISerror(h)); -+ free(msgid); -+ } -+ -+ /* Write remembered entries (no token). */ -+ for (i = N_WITH_TOKEN; i < N_WITH_TOKEN + N_REMEMBERED; i++) { -+ char *msgid = make_msgid(i); -+ if (!HISremember(h, msgid, (time_t) 1000000 + i, -+ (time_t) 1000000 + i)) -+ bail("can't remember history entry %lu: %s", i, HISerror(h)); -+ free(msgid); -+ } -+ -+ HISsync(h); -+ HISclose(h); -+ ok(1, true); /* history created and populated */ -+ -+ /* Reopen read-only (as expireover does). */ -+ h = HISopen(histpath, "hisv6", HIS_RDONLY); -+ if (h == NULL) -+ bail("can't reopen history at %s", histpath); -+ ok(2, true); /* history reopened */ -+ -+ /* Build the bloom filter via HISwalk. */ -+ bloom = bloom_create(N_WITH_TOKEN + N_REMEMBERED, 10000); -+ walk_ok = HISwalk(h, NULL, bloom, build_bloom_cb); -+ ok(3, walk_ok); /* HISwalk succeeded */ -+ ok(4, bloom_count(bloom) == N_WITH_TOKEN); /* only token entries added */ -+ -+ /* Verify: all token entries should be bloom hits. */ -+ false_negatives = 0; -+ for (i = 0; i < N_WITH_TOKEN; i++) { -+ char *msgid = make_msgid(i); -+ HASH hash = HashMessageID(msgid); -+ if (!bloom_check(bloom, &hash)) -+ false_negatives++; -+ free(msgid); -+ } -+ ok(5, false_negatives == 0); /* no false negatives for token entries */ -+ if (false_negatives > 0) -+ diag("false negatives: %lu out of %d", false_negatives, N_WITH_TOKEN); -+ -+ /* Verify: remembered entries should NOT be in the bloom filter. -+ * (Some may be false positives, but most should miss.) */ -+ bloom_misses = 0; -+ for (i = N_WITH_TOKEN; i < N_WITH_TOKEN + N_REMEMBERED; i++) { -+ char *msgid = make_msgid(i); -+ HASH hash = HashMessageID(msgid); -+ if (!bloom_check(bloom, &hash)) -+ bloom_misses++; -+ free(msgid); -+ } -+ ok(6, bloom_misses > N_REMEMBERED * 9 / 10); /* >90% should miss */ -+ diag("remembered entries: %lu/%d were bloom misses (expected most)", -+ bloom_misses, N_REMEMBERED); -+ -+ /* Verify: entries not in history at all should mostly miss. */ -+ bloom_misses = 0; -+ for (i = N_WITH_TOKEN + N_REMEMBERED; -+ i < N_WITH_TOKEN + N_REMEMBERED + N_NOT_IN_HIST; i++) { -+ char *msgid = make_msgid(i); -+ HASH hash = HashMessageID(msgid); -+ if (!bloom_check(bloom, &hash)) -+ bloom_misses++; -+ free(msgid); -+ } -+ ok(7, bloom_misses > N_NOT_IN_HIST * 9 / 10); /* >90% should miss */ -+ diag("not-in-history entries: %lu/%d were bloom misses (expected most)", -+ bloom_misses, N_NOT_IN_HIST); -+ -+ bloom_free(bloom); -+ HISclose(h); -+ -+ /* Cleanup temp directory. */ -+ { -+ char cmd[128]; -+ snprintf(cmd, sizeof(cmd), "/bin/rm -rf %s", tmpdir); -+ if (system(cmd) < 0) -+ sysdiag("can't clean up %s", tmpdir); -+ } -+ ok(8, true); /* cleanup */ -+ -+ return 0; -+} -diff --git tests/lib/bloom-t.c tests/lib/bloom-t.c -new file mode 100644 -index 000000000..ef3b62bd1 ---- /dev/null -+++ tests/lib/bloom-t.c -@@ -0,0 +1,128 @@ -+/* Test suite for lib/bloom.c. */ -+ -+#include "portable/system.h" -+ -+#include -+ -+#include "inn/bloom.h" -+#include "inn/libinn.h" -+#include "tap/basic.h" -+ -+ -+/* -+** Generate a deterministic HASH from an integer for testing. -+*/ -+static HASH -+make_hash(unsigned long n) -+{ -+ char buf[64]; -+ snprintf(buf, sizeof(buf), "", n); -+ return HashMessageID(buf); -+} -+ -+ -+int -+main(void) -+{ -+ struct bloom_filter *bf; -+ HASH h1, h2, h3; -+ unsigned long i; -+ unsigned long false_positives; -+ unsigned long n_check; -+ -+ test_init(20); -+ -+ /* Basic creation. */ -+ bf = bloom_create(1000, 10000); -+ ok(1, bf != NULL); -+ ok(2, bloom_bits(bf) >= 1000 * 20); /* 0.01% FP needs ~20 bits/entry */ -+ ok(3, bloom_nhash(bf) == 14); -+ ok(4, bloom_count(bf) == 0); -+ -+ /* Add and check: true positives. */ -+ h1 = make_hash(1); -+ h2 = make_hash(2); -+ h3 = make_hash(3); -+ bloom_add(bf, &h1); -+ bloom_add(bf, &h2); -+ ok(5, bloom_count(bf) == 2); -+ ok(6, bloom_check(bf, &h1)); -+ ok(7, bloom_check(bf, &h2)); -+ -+ /* True negative. */ -+ ok(8, !bloom_check(bf, &h3)); -+ -+ bloom_free(bf); -+ -+ /* Larger test: verify false positive rate. -+ * Add 10,000 items, then check 100,000 items that were NOT added. -+ * At 0.01% target FP rate, we expect ~10 false positives out of -+ * 100,000 checks. Allow up to 50 (0.05%) to account for variance. */ -+ bf = bloom_create(10000, 10000); -+ ok(9, bf != NULL); -+ -+ for (i = 0; i < 10000; i++) { -+ HASH h = make_hash(i); -+ bloom_add(bf, &h); -+ } -+ ok(10, bloom_count(bf) == 10000); -+ -+ /* Verify all added items are found (no false negatives). */ -+ for (i = 0; i < 10000; i++) { -+ HASH h = make_hash(i); -+ if (!bloom_check(bf, &h)) { -+ ok(11, false); -+ diag("false negative at i=%lu", i); -+ goto fp_test; -+ } -+ } -+ ok(11, true); -+ -+fp_test: -+ /* Count false positives from items never added. */ -+ false_positives = 0; -+ n_check = 100000; -+ for (i = 10000; i < 10000 + n_check; i++) { -+ HASH h = make_hash(i); -+ if (bloom_check(bf, &h)) -+ false_positives++; -+ } -+ ok(12, false_positives <= 50); -+ if (false_positives > 50) -+ diag("false positives: %lu out of %lu (expected <= 50)", -+ false_positives, n_check); -+ else -+ diag("false positives: %lu out of %lu (%.4f%%)", -+ false_positives, n_check, -+ 100.0 * (double) false_positives / (double) n_check); -+ -+ bloom_free(bf); -+ -+ /* Test with different FP rate parameters. */ -+ bf = bloom_create(1000, 100); /* 1% FP rate */ -+ ok(13, bf != NULL); -+ ok(14, bloom_nhash(bf) == 7); /* k=7 for 1% FP */ -+ bloom_free(bf); -+ -+ bf = bloom_create(1000, 1000); /* 0.1% FP rate */ -+ ok(15, bf != NULL); -+ ok(16, bloom_nhash(bf) == 10); /* k=10 for 0.1% FP */ -+ bloom_free(bf); -+ -+ /* Edge case: very small filter. */ -+ bf = bloom_create(1, 10000); -+ ok(17, bf != NULL); -+ ok(18, bloom_bits(bf) >= 64); /* minimum size */ -+ bloom_free(bf); -+ -+ /* Test with max nhash (fp_inv >= 10000000, nhash=24). -+ * Exercises the full positions array to catch overflow. */ -+ bf = bloom_create(100, 10000000); -+ ok(19, bf != NULL); -+ h1 = make_hash(42); -+ bloom_add(bf, &h1); -+ ok(20, bloom_check(bf, &h1)); -+ bloom_free(bf); -+ -+ return 0; -+} diff --git a/news/inn-current/files/pr340.patch b/news/inn-current/files/pr340.patch deleted file mode 100644 index fca6565e5572..000000000000 --- a/news/inn-current/files/pr340.patch +++ /dev/null @@ -1,3078 +0,0 @@ -diff --git MANIFEST MANIFEST ---- MANIFEST 2026-05-18 00:16:06.149832000 -0700 -+++ MANIFEST 2026-05-18 00:16:11.124094000 -0700 -@@ -441,6 +441,7 @@ - include/inn/sequence.h Header file for sequence space arithmetic - include/inn/storage.h Header file for storage API - include/inn/timer.h Header file for generic timers -+include/inn/tombstone.h Header file for cancel tombstone log - include/inn/tst.h Header file for ternary search tries - include/inn/utility.h Header file for utility functions - include/inn/vector.h Header file for vectors of strings -@@ -805,6 +806,7 @@ - storage/timehash/method.config buildconfig definition - storage/timehash/timehash.c timehash storage routines - storage/timehash/timehash.h Header for timehash -+storage/tombstone.c Cancel tombstone log helpers - storage/tradindexed tradindexed overview method (Directory) - storage/tradindexed/ovmethod.config buildconfig definition - storage/tradindexed/ovmethod.mk Make rules for tradindexed overview -@@ -931,6 +933,10 @@ - tests/data/upgrade/sasl.conf Obsolete sasl.conf config file - tests/docs Test suite for documentation (Directory) - tests/docs/pod.t.in Tests for POD formatting -+tests/expire Test suite for expire (Directory) -+tests/expire/tombstone-e2e.t End-to-end tests for tombstone log -+tests/expire/tombstone-hisexpire-t.c HISexpire integration test for tombstone -+tests/expire/tombstone-t.c Tests for tombstone library - tests/innd Test suite for innd (Directory) - tests/innd/artparse-t.c Tests for ARTparse in innd - tests/innd/chan-t.c Tests for CHAN functions in innd -@@ -997,6 +1003,7 @@ - tests/runtests.c The test suite driver program - tests/storage Test suite for storage (Directory) - tests/storage/archive.t Tests for backends/archive -+tests/storage/cancel-tombstone-t.c Tests for SMcanceltombstone - tests/storage/makehistory.t Tests for expire/makehistory - tests/storage/sm.t Tests for frontends/sm - tests/tap Helper scripts for TAP (Directory) -diff --git doc/pod/expire.pod doc/pod/expire.pod ---- doc/pod/expire.pod 2026-05-18 00:16:06.130540000 -0700 -+++ doc/pod/expire.pod 2026-05-18 00:16:11.124260000 -0700 -@@ -175,6 +175,18 @@ - - =back - -+=head1 TOMBSTONE LOG -+ -+When I is enabled in F, B consumes -+the per-cycle deletion log produced by B, B, and -+B (F/expireover.tombstone> and -+F/cancels.tombstone>) so it can drop history entries for -+those articles without doing a per-article C -+syscall. An empty tombstone is treated as "no cancels this cycle" -+and the slow scan is skipped entirely. See inn.conf(5) under -+I for the file lifecycle, locking model, and -+recovery story. -+ - =head1 HISTORY - - Written by Rich $alz for InterNetNews. Converted to -diff --git doc/pod/expireover.pod doc/pod/expireover.pod ---- doc/pod/expireover.pod 2026-05-18 00:16:06.132608000 -0700 -+++ doc/pod/expireover.pod 2026-05-18 00:16:11.124389000 -0700 -@@ -21,6 +21,14 @@ - that have already been removed by some other process, and B<-e>, B<-k>, - B<-N>, B<-p>, B<-q>, B<-w>, and B<-z> are all ignored. - -+When I is enabled in F, B -+appends each cancelled token to F/expireover.tombstone.NEW> -+under an exclusive POSIX lock and atomically renames the file into -+place on a clean run. In delayrm mode (B<-z>), the rename is -+performed by B after B succeeds. The next -+B run consumes this log to skip per-article storage existence -+checks. See inn.conf(5) under I. -+ - When I is set, the default behavior of B is - to remove the article from the spool once it expires out of all of the - newsgroups to which it was crossposted. The article is, however, removed -diff --git doc/pod/inn.conf.pod doc/pod/inn.conf.pod ---- doc/pod/inn.conf.pod 2026-05-18 00:16:06.131089000 -0700 -+++ doc/pod/inn.conf.pod 2026-05-18 00:16:11.129304000 -0700 -@@ -652,6 +652,82 @@ - Moreover, the deprecated C and C header fields, already present - in the standard overview fields as metadata items, cannot be added. - -+=item I -+ -+Whether INN tools record cancellation tombstones so a subsequent -+B run can skip per-article storage existence checks. When -+enabled (and I is also true), two log files in -+I capture every cancellation: -+ -+=over 4 -+ -+=item F/expireover.tombstone> -+ -+Written by B after each successful B in -+group-based expiry, atomically renamed into place on a clean run. -+When B runs with C<-z> (delayed removal), the -+B calls are deferred to B via B; -+B writes the entries up front and B performs -+the atomic rename after B succeeds, so the same speedup -+applies to delayrm setups. After B consumes this file it -+is unlinked and re-seeded as a header-only successor, matching -+the on-disk presence of F; the next -+B run overwrites the seeded file with its full -+content. -+ -+=item F/cancels.tombstone> -+ -+Appended continuously by B when it processes cancel control -+messages, and by B for manual cancellations. Appenders take -+a shared fcntl POSIX lock; B snapshots the file by atomic -+rename to F under an exclusive fcntl -+lock, unlinks the snapshot after a successful consume, then -+recreates F as a header-only file under an -+exclusive lock so B's per-connection fast path stays -+active through quiet inter-cancel periods. An appender that -+raced a cancel into a new live file between the rename and the -+recreate has its content preserved verbatim below the restored -+header. Append atomicity -+relies on POSIX guaranteeing that each C to a regular -+file opened with C is atomic with respect to other -+writers; this holds for any size of single C on local -+filesystems. Cross-client atomicity over NFS is not guaranteed: -+if I is on NFS, lines from concurrent writers on different -+clients can in theory interleave. In practice INN's cancel -+sources (B on a single host plus occasional B on the -+same host) write from one client. -+ -+=back -+ -+The next B invocation loads both files into a single hashset -+and treats every article in either log as already gone, avoiding an -+C call per history entry. For storage methods -+where the stat is a file-system call (tradspool, timehash) this turns -+a billion C calls into a few thousand hash lookups; for -+storage methods that self-expire (CNFS) the stat call remains because -+articles can vanish through wrap-around without going through -+B. -+ -+In normal operation every cancellation path participates in -+tombstone tracking, so all articles removed from the spool are -+recorded. Residual orphans can only accumulate from events outside -+the tracked paths: a process crash in the narrow window between -+B and the tombstone append, manual filesystem-level -+deletes that bypass B, or filesystem corruption. When such -+orphans do appear they are harmless (B returns "no such -+article" to readers that hit them) and exist only as small history -+entries. No regular reconciliation cadence is needed. If an -+operator suspects orphan accumulation after admin intervention or a -+storage incident, B can be re-run with this option disabled -+in F to perform an exhaustive C -+scan; this is an exceptional operation, not a scheduled one. -+ -+Footprint: ~38 bytes per entry on disk, ~50 bytes in expire's hash -+table. 1M cancels per run = ~38 MB tombstone, ~50 MB hash. -+ -+This is a boolean value and the default is false; sites should opt in -+after validating the option's behaviour against their workload. -+ - =item I - - Whether to enable newsgroup-based expiry. If set to false, article expiry -@@ -910,10 +986,50 @@ - as present in response to an NNTP command (HDR, LISTGROUP, NEWNEWS, OVER, - XPAT). The primary use of this setting is to prevent B from returning - information about articles which are no longer present on the server but which --still have overview data available. Checking the existence of articles before --returning overview information slows down the overview commands, but reduces --the number of "article is missing" errors seen by the client. This is a --boolean value and the default is true. -+still have overview data available. Checking existence with an unconditional -+C slows down the overview commands; with I also -+enabled (see below) the check uses an in-memory hash lookup instead and is -+cheaper than disabling the check on tradspool/timehash/timecaf backends. -+The trade-off remains: enabling this reduces the number of "article is -+missing" errors seen by the client. This is a boolean value and the default -+is true. -+ -+When I is also enabled, B consults the -+F/cancels.tombstone> log on the article-existence check -+path: a token recorded as cancelled is reported as gone without an -+C call, and a token absent from the tombstone is -+trusted to still exist (skipping the syscall) for storage methods -+that do not self-expire. Self-expiring backends (CNFS) still go -+through C because cyclic-buffer wrap-around bypasses -+the tombstone. The tombstone is loaded lazily on first use per -+connection and refreshed by C on each call; the parsed -+hashset is rebuilt only when the file's mtime or size changes. -+Statting the same path is dentry-cache resident and far cheaper -+than the per-article syscalls the fast path elides, so cancellations -+recorded by other processes become visible to long-lived -+connections on the next existence check. -+ -+Each B connection holds an independent copy of the parsed -+hashset, costing roughly 50 bytes per cancel; sites with very large -+cancel volumes between B runs and many concurrent readers -+should size memory accordingly. Loss relative to the unconditional -+C path is bounded to out-of-band events the tombstone -+cannot see (manual filesystem deletes that bypass B, -+filesystem corruption); admin-initiated B and B -+cancels are tracked. -+ -+Important: on non-self-expiring backends the fast path treats a -+tombstone-miss as proof that the article is still on disk, without -+verifying. Admins must remove articles via B (which records -+the cancel in the tombstone), not by direct C on the spool, or -+readers will be told a deleted article still exists until the next -+B reconciles overview. -+ -+Note that the fast path is also gated on I; both -+must be true to take effect. The B C -+counter reflects only the slow C path, so the syslog -+field will appear smaller when the fast path is doing most of the -+work. - - You may also want to see the I parameter in readers.conf(5) - which controls the computing of the estimated article count returned in NNTP -diff --git doc/pod/sm.pod doc/pod/sm.pod ---- doc/pod/sm.pod 2026-05-18 00:16:06.130876000 -0700 -+++ doc/pod/sm.pod 2026-05-18 00:16:11.124893000 -0700 -@@ -51,6 +51,13 @@ - be retrievable by any part of INN. It's equivalent to C - except it takes a storage API token instead of a message-ID. - -+When the I setting in F is true, B -+also appends the cancelled token to F/cancels.tombstone> -+so a later B run can drop the corresponding history entry -+without a per-article storage check. Append failures are logged -+but do not affect the cancellation itself. See inn.conf(5) under -+I for the full mechanism. -+ - =item B<-H> - - Retrieve only the headers of the article rather than the entire article. -diff --git expire/expire.c expire/expire.c ---- expire/expire.c 2026-05-18 00:16:06.129750000 -0700 -+++ expire/expire.c 2026-05-18 00:16:11.125117000 -0700 -@@ -6,10 +6,13 @@ - - #include - #include -+#include - #include - #include - #include -+#include - -+#include "inn/hashtab.h" - #include "inn/history.h" - #include "inn/innconf.h" - #include "inn/inndcomm.h" -@@ -18,6 +21,7 @@ - #include "inn/newsuser.h" - #include "inn/paths.h" - #include "inn/storage.h" -+#include "inn/tombstone.h" - - - typedef struct _EXPIRECLASS { -@@ -48,6 +52,7 @@ - static int EXPverbose; - static long EXPprocessed; - static long EXPunlinked; -+static long EXPtombstoned; - static long EXPallgone; - static long EXPstillhere; - static struct history *History; -@@ -371,23 +376,208 @@ - } - - /* -+** Load both tombstone logs into a hashset: -+** - ${pathdb}/expireover.tombstone : written by expireover/expirerm -+** (atomic .NEW -> final rename); unlinked after a successful -+** expire run -+** - ${pathdb}/cancels.tombstone : appended continuously by innd -+** and sm for cancels outside the expireover pipeline; renamed -+** to .processing at load time and unlinked after a successful -+** expire run (rename-and-process avoids the read/truncate race -+** that drops cancels arriving during HISexpire) -+** -+** At startup, also recovers any leftover .processing snapshot from a -+** previous expire run that crashed before unlinking. -+** -+** consuming=true means this is a real run that will replace the -+** history file; we rename cancels.tombstone to .processing under -+** the consumer's lock so concurrent appenders cannot lose cancels -+** written between our read and the eventual unlink. consuming= -+** false (dry-run, tracing, or alternate-output expire) skips the -+** rename and reads the live file directly: no consume happens, so -+** there is nothing to atomically detach. This avoids both the -+** TOCTOU race in restoring the snapshot and the operator surprise -+** of cancels.tombstone disappearing for the duration of a -x run. -+** -+** Returns NULL if no tombstone could be loaded (no file present or -+** all reads failed); that is not an error. Expire just falls back -+** to the per-article SMretrieve check. *out_expireover_path and -+** *out_cancels_snapshot are set to the file paths (caller frees and -+** unlinks after a successful run); both are NULL when consuming is -+** false. -+*/ -+static struct hash * -+EXPloadtombstone(bool consuming, char **out_expireover_path, -+ char **out_cancels_snapshot) -+{ -+ char *expireover_path = NULL; -+ char *cancels_path = NULL; -+ char *cancels_snapshot = NULL; -+ char *leftover = NULL; -+ struct hash *h = NULL; -+ struct stat sb; -+ unsigned long n_expireover = 0; -+ unsigned long n_cancels = 0; -+ unsigned long n_leftover = 0; -+ bool expireover_present = false; -+ bool leftover_present = false; -+ bool cancels_present = false; -+ -+ /* Default outputs to NULL; only set on the success-return path -+ below. This avoids double-frees if a future maintainer adds an -+ early return between here and that single point. */ -+ *out_expireover_path = NULL; -+ *out_cancels_snapshot = NULL; -+ -+ expireover_path = concatpath(innconf->pathdb, "expireover.tombstone"); -+ cancels_path = concatpath(innconf->pathdb, "cancels.tombstone"); -+ leftover = concat(cancels_path, ".processing", (char *) 0); -+ -+ /* Probe each file's presence (independent of content) so we can -+ distinguish "tombstone subsystem is actively tracking but had -+ no cancels this cycle" (trust the empty hashset, skip -+ SMretrieve for everything) from "no files at all" (subsystem -+ not in use, fall back to slow path). Without this, a cycle -+ with zero cancels would needlessly run SMretrieve on every -+ history entry. */ -+ if (stat(expireover_path, &sb) == 0) -+ expireover_present = true; -+ if (stat(leftover, &sb) == 0) -+ leftover_present = true; -+ if (stat(cancels_path, &sb) == 0) -+ cancels_present = true; -+ -+ /* Initial size: tokens are typically ~38 bytes per line; we let the -+ hash expand if we underestimate. 4096 covers the common case. */ -+ h = tombstone_hash_create(4096); -+ -+ /* Recover a leftover .processing from a previous run that crashed -+ between rename and unlink. */ -+ n_leftover = tombstone_read(h, leftover, NULL); -+ -+ /* On a real run, atomically snapshot the live cancels.tombstone, -+ then read it. On dry-run / tracing / alt-output, just read -+ the live file directly: no consume happens, so there is no -+ need to detach it (and detach-then-restore has a TOCTOU race -+ with concurrent appenders). */ -+ if (consuming) { -+ cancels_snapshot = tombstone_rename_for_processing(cancels_path); -+ if (cancels_snapshot != NULL) -+ n_cancels = tombstone_read(h, cancels_snapshot, NULL); -+ } else { -+ n_cancels = tombstone_read(h, cancels_path, NULL); -+ } -+ -+ /* Read the expireover-side log. */ -+ n_expireover = tombstone_read(h, expireover_path, NULL); -+ -+ if (EXPverbose) -+ printf("Loaded %lu + %lu + %lu tombstone entries (%lu unique)\n", -+ n_expireover, n_cancels, n_leftover, hash_count(h)); -+ -+ /* Reconcile the leftover with the active snapshot. Only relevant -+ on a real consuming run; dry-run leaves the leftover where it -+ is for the next consuming run to pick up. -+ (a) leftover empty: just free the path string -+ (b) leftover non-empty + no live snapshot: promote leftover to -+ be the snapshot so the caller's single unlink covers it -+ (c) leftover non-empty + live snapshot: unlink the leftover now -+ (its contents are already in the hashset); the snapshot -+ remains and is unlinked on success */ -+ if (consuming) { -+ if (n_leftover > 0 && cancels_snapshot == NULL) { -+ cancels_snapshot = leftover; -+ leftover = NULL; -+ } else if (n_leftover > 0) { -+ if (unlink(leftover) < 0) -+ syswarn("can't unlink %s", leftover); -+ } else if (leftover_present && cancels_snapshot == NULL) { -+ /* Leftover existed but was empty. Treat it as the -+ snapshot so the caller cleans it up after a -+ successful run. */ -+ cancels_snapshot = leftover; -+ leftover = NULL; -+ } else if (leftover_present) { -+ /* Both leftover and live snapshot existed; leftover -+ empty. Unlink it now; snapshot will be cleaned up -+ on success. */ -+ if (unlink(leftover) < 0) -+ syswarn("can't unlink %s", leftover); -+ } -+ } -+ free(leftover); -+ leftover = NULL; -+ -+ /* Decide between "fall back to slow path" (return NULL) and -+ "trust the (possibly empty) tombstone". A file that exists -+ (regardless of content) is taken as evidence the tombstone -+ subsystem is active for this site: innd/sm/expireover are -+ writing it. An empty active tombstone correctly says "nothing -+ was cancelled in the last cycle"; trust it. In dry-run mode -+ cancels.tombstone is not renamed, so cancels_snapshot stays -+ NULL; use the cancels_present probe instead. */ -+ if (!expireover_present && !leftover_present && !cancels_present -+ && cancels_snapshot == NULL) { -+ hash_free(h); -+ free(expireover_path); -+ free(cancels_path); -+ return NULL; -+ } -+ -+ /* Single success-return: hand both paths to the caller for -+ cleanup after a successful run. */ -+ free(cancels_path); -+ *out_expireover_path = expireover_path; -+ *out_cancels_snapshot = cancels_snapshot; -+ return h; -+} -+ -+ -+/* - ** Do the work of expiring one line. - ** Returns true when the article should be kept for the time being. - */ - static bool --EXPdoline(void *cookie UNUSED, time_t arrived, time_t posted, time_t expires, -+EXPdoline(void *cookie, time_t arrived, time_t posted, time_t expires, - TOKEN *token) - { -+ struct hash *tombstone = (struct hash *) cookie; - time_t when; - bool HasSelfexpire = false; - bool Selfexpired = false; -+ bool selfexpiring; - ARTHANDLE *article; - enum KR kr; - bool r; - -- if (innconf->groupbaseexpiry || SMprobe(SELFEXPIRE, token, NULL)) { -- if ((article = SMretrieve(*token, RETR_STAT)) == (ARTHANDLE *) NULL) { -+ /* Tombstone fast path: if expireover already cancelled this article, -+ drop the history entry without doing any storage I/O. Bump -+ EXPunlinked too so the news.daily summary's "Articles dropped" -+ count matches the slow path's accounting (EXPremove also bumps -+ it). */ -+ if (tombstone != NULL && hash_lookup(tombstone, token) != NULL) { -+ EXPprocessed++; -+ if (EXPverbose > 3) -+ printf("%s (tombstoned by expireover)\n", TokenToText(*token)); -+ EXPallgone++; -+ EXPunlinked++; -+ EXPtombstoned++; -+ return false; -+ } -+ -+ selfexpiring = SMprobe(SELFEXPIRE, token, NULL); -+ -+ if (innconf->groupbaseexpiry || selfexpiring) { -+ if (tombstone != NULL && !selfexpiring) { -+ /* Backend does not self-expire and the tombstone log is -+ complete: not in the log means the article still exists. -+ Skip the SMretrieve to avoid a per-article syscall (the -+ main speedup for tradspool / timehash / timecaf). */ - HasSelfexpire = true; -+ Selfexpired = false; -+ } else if ((article = SMretrieve(*token, RETR_STAT)) -+ == (ARTHANDLE *) NULL) { -+ HasSelfexpire = true; - Selfexpired = true; - } else { - /* The article is still alive, free it. */ -@@ -470,6 +660,8 @@ - printf("Entries expired %8ld\n", EXPallgone); - if (!innconf->groupbaseexpiry) - printf("Articles dropped %8ld\n", EXPunlinked); -+ if (innconf->expiretombstone && innconf->groupbaseexpiry) -+ printf("Tombstone hits %8ld\n", EXPtombstoned); - } - - /* Append statistics to a summary file */ -@@ -536,6 +728,16 @@ - if (!innconf_read(NULL)) - exit(1); - -+ /* Warn about expiretombstone-without-groupbaseexpiry: in that -+ configuration there is no consumable tombstone since OVEXPremove -+ (which writes one of the two logs) is never called. The -+ cancels.tombstone written by innd/sm could still be loaded, but -+ only with groupbaseexpiry to make the result coherent with -+ expire's overall semantics. */ -+ if (innconf->expiretombstone && !innconf->groupbaseexpiry) -+ notice("expiretombstone has no effect when groupbaseexpiry" -+ " is false"); -+ - HistoryText = concatpath(innconf->pathdb, INN_PATH_HISTORY); - - umask(NEWSUMASK); -@@ -694,9 +896,70 @@ - CleanupAndExit(Server, false, 1); - } - -- Bad = HISexpire(History, NHistory, EXPreason, Writing, NULL, EXPremember, -- EXPdoline) -- == false; -+ /* Consume the tombstone logs produced since the last expire run. -+ Lets EXPdoline drop history entries for articles cancelled by -+ expireover, expirerm, innd, or sm without per-article SMretrieve -+ calls. Gated on groupbaseexpiry as well: if the admin flipped -+ that off since the last expireover, a stale tombstone written -+ under the old config could drop history entries for articles -+ still alive. */ -+ { -+ bool consuming = Writing && !EXPtracing && NHistory == NULL; -+ char *expireover_path = NULL; -+ char *cancels_snapshot = NULL; -+ struct hash *tombstone = NULL; -+ -+ if (innconf->expiretombstone && innconf->groupbaseexpiry) -+ tombstone = EXPloadtombstone(consuming, &expireover_path, -+ &cancels_snapshot); -+ -+ Bad = HISexpire(History, NHistory, EXPreason, Writing, tombstone, -+ EXPremember, EXPdoline) -+ == false; -+ -+ if (tombstone != NULL) -+ hash_free(tombstone); -+ -+ /* On a successful real run, unlink both consumed snapshots -+ and seed header-only successors. Dry-run / tracing / -+ alt-output skipped the rename in the loader so there are -+ no snapshot paths to clean up. */ -+ if (!Bad && consuming) { -+ if (expireover_path != NULL) { -+ if (unlink(expireover_path) < 0 && errno != ENOENT) -+ syswarn("can't unlink %s", expireover_path); -+ } -+ if (cancels_snapshot != NULL) { -+ if (unlink(cancels_snapshot) < 0 && errno != ENOENT) -+ syswarn("can't unlink %s", cancels_snapshot); -+ } -+ -+ /* Leave header-only successors behind so both tombstone -+ files exist symmetrically in pathdb after every -+ successful expire. For cancels.tombstone this keeps -+ nnrpd's per-connection fast path active through quiet -+ inter-cycle periods. For expireover.tombstone it is -+ cosmetic but matches the file's documented presence; -+ expireover will overwrite the header-only file the -+ next time it runs. Runs every consuming cycle when -+ the feature is enabled, including the brand-new- -+ install case where EXPloadtombstone returned NULL and -+ there was no snapshot to consume. Idempotent when -+ the file already starts with the header. */ -+ if (innconf->expiretombstone && innconf->groupbaseexpiry) { -+ char *path; -+ -+ path = concatpath(innconf->pathdb, "cancels.tombstone"); -+ tombstone_ensure_header(path); -+ free(path); -+ path = concatpath(innconf->pathdb, "expireover.tombstone"); -+ tombstone_ensure_header(path); -+ free(path); -+ } -+ } -+ free(expireover_path); -+ free(cancels_snapshot); -+ } - - if (UnlinkFile && EXPunlinkfile == NULL) - /* Got -z but file was closed; oops. */ -diff --git expire/expireover.c expire/expireover.c ---- expire/expireover.c 2026-05-18 00:16:06.129729000 -0700 -+++ expire/expireover.c 2026-05-18 00:17:50.715171000 -0700 -@@ -10,13 +10,16 @@ - #include "portable/system.h" - - #include -+#include - #include -+#include -+#include - #include - #include -+#include - --#include -- - #include "inn/bloom.h" -+#include "inn/buffer.h" - #include "inn/history.h" - #include "inn/innconf.h" - #include "inn/libinn.h" -@@ -26,7 +29,13 @@ - #include "inn/paths.h" - #include "inn/qio.h" - #include "inn/storage.h" -+#include "inn/tombstone.h" - -+/* OVtombstonefile is an internal of the storage library (declared in -+ * storage/ovinterface.h), but expireover owns its lifecycle, so we -+ * declare it here rather than exporting it via the public ov.h header. */ -+extern FILE *OVtombstonefile; -+ - static const char usage[] = "\ - Usage: expireover [-ekNpqs] [-f file] [-w offset] [-z rmfile] [-Z lowmarkfile]\n"; - -@@ -65,6 +74,94 @@ - } - - -+/* -+** Verify each line of an in-memory tombstone buffer via SMretrieve and -+** append survivors to kept. An entry survives if SMretrieve confirms -+** the article is gone (SMERR_NOENT) or returns a non-NOENT error -+** (SMERR_UNINIT, EIO, etc., where we cannot tell whether the article -+** is alive); a transient failure must not silently discard a valid -+** cancel record. Live articles (SMretrieve returns non-NULL) are -+** dropped: we must not re-tombstone an article that is still on disk. -+*/ -+static void -+verify_tombstone_lines(char *raw, ssize_t got, struct buffer *kept) -+{ -+ char *entry, *save_p; -+ -+ if (got <= 0) -+ return; -+ raw[got] = '\0'; -+ entry = strtok_r(raw, "\n", &save_p); -+ while (entry != NULL) { -+ size_t len = strlen(entry); -+ if (len > 0 && entry[len - 1] == '\r') -+ entry[--len] = '\0'; -+ if (len > 0 && IsToken(entry)) { -+ TOKEN t = TextToToken(entry); -+ ARTHANDLE *art = SMretrieve(t, RETR_STAT); -+ if (art != NULL) { -+ /* Article still on disk; drop from tombstone to -+ avoid orphaning history for it. */ -+ SMfreearticle(art); -+ } else { -+ /* Either confirmed gone (SMERR_NOENT) or transient -+ error (SMERR_UNINIT, EIO, etc.). Keep the entry -+ either way: confirmed-gone is the normal case, -+ and on transient error we cannot tell whether -+ the article is alive, so preserve the record so -+ the next run can re-evaluate. Silently dropping -+ on transient error would leak cancels across a -+ storage outage. */ -+ buffer_append(kept, entry, len); -+ buffer_append(kept, "\n", 1); -+ } -+ } -+ entry = strtok_r(NULL, "\n", &save_p); -+ } -+} -+ -+ -+/* -+** Read an entire file into a freshly allocated buffer. Used for -+** leftover-recovery of expireover.tombstone (from a cycle where -+** expirerm finalized but expire never consumed) and the .NEW -+** leftover from a crashed prior run. Returns NULL on missing or -+** empty file (sets *out_size to 0); returns a malloc'd buffer with -+** the file content otherwise (caller frees). -+*/ -+static char * -+slurp_tombstone(const char *path, ssize_t *out_size) -+{ -+ int fd; -+ struct stat sb; -+ char *raw; -+ ssize_t got = 0, n; -+ -+ *out_size = 0; -+ fd = open(path, O_RDONLY); -+ if (fd < 0) -+ return NULL; -+ if (fstat(fd, &sb) < 0 || sb.st_size <= 0) { -+ close(fd); -+ return NULL; -+ } -+ raw = xmalloc(sb.st_size + 1); -+ while (got < sb.st_size) { -+ n = read(fd, raw + got, sb.st_size - got); -+ if (n <= 0) -+ break; -+ got += n; -+ } -+ close(fd); -+ if (got <= 0) { -+ free(raw); -+ return NULL; -+ } -+ *out_size = got; -+ return raw; -+} -+ -+ - int - main(int argc, char *argv[]) - { -@@ -76,9 +173,12 @@ - char *active_path = NULL; - char *lowmark_path = NULL; - char *path; -+ char *tombstone_path = NULL; -+ char *tombstone_path_new = NULL; - FILE *lowmark = NULL; - bool purge_deleted = false; - bool always_stat = false; -+ bool tombstone_clean = true; - struct history *history; - struct bloom_filter *bloom = NULL; - struct bloom_filter *null_bloom = NULL; -@@ -144,6 +244,14 @@ - if (!innconf_read(NULL)) - exit(1); - -+ /* Warn about expiretombstone-without-groupbaseexpiry: in that -+ configuration OVEXPremove is never called so no entries are -+ ever appended. The setting is silently ignored. Operators -+ reaching for the speedup deserve to know it's not active. */ -+ if (innconf->expiretombstone && !innconf->groupbaseexpiry) -+ notice("expiretombstone has no effect when groupbaseexpiry" -+ " is false"); -+ - /* Change to the runasuser user and runasgroup group if necessary. */ - ensure_news_user_grp(true, true); - -@@ -240,6 +348,169 @@ - free(histpath); - } - -+ /* Open the tombstone log. OVEXPremove appends a line per article -+ it cancels (inline) or schedules for removal via the rm file -+ (delayrm). In delayrm mode the .NEW -> final rename is performed -+ by expirerm after fastrm succeeds (see expirerm.in); in inline -+ mode this process performs the rename below. Skipped when -+ groupbaseexpiry is false because OVEXPremove is not called in -+ that mode, and when the admin has disabled the feature via -+ expiretombstone. -+ -+ Concurrency model: open with O_RDWR|O_CREAT (no truncate), then -+ take an exclusive *non-blocking* fcntl POSIX lock via -+ inn_lock_file. Non-blocking is required for correctness, not -+ just performance: the truncate below would destroy the previous -+ holder's content if we ever waited for the lock. If we cannot -+ acquire the lock, another expireover is running; disable -+ tombstone writing for this run and proceed with normal -+ expiration. Manual concurrent invocations are protected -+ against tombstone corruption; news.daily's shlock prevents -+ them in normal flow. The lock is released by fclose at -+ finalize time. -+ -+ Leftover handling: two recovery paths, both verified per-token -+ with SMretrieve(RETR_STAT) before merging into the new .NEW. -+ (1) An existing .NEW file means a previous run crashed or -+ whose expirerm failed. (2) An existing final tombstone -+ (without a .NEW) means the previous cycle's expirerm promoted -+ it but expire never consumed. Naively truncating .NEW or -+ overwriting the final would lose cancels that genuinely -+ happened, while preserving them blindly would risk dropping -+ history for articles still on disk (fastrm failures). Per- -+ token verification keeps confirmed-gone entries (SMERR_NOENT) -+ and entries with transient errors (SMERR_UNINIT, EIO, ...) so -+ the next run can re-evaluate them without silent loss; live -+ articles are dropped. Cost is one stat per leftover token, -+ only on the rare crash-recovery path. */ -+ if (innconf->expiretombstone && innconf->groupbaseexpiry) { -+ int fd; -+ -+ tombstone_path = concatpath(innconf->pathdb, "expireover.tombstone"); -+ tombstone_path_new = concat(tombstone_path, ".NEW", (char *) 0); -+ fd = open(tombstone_path_new, O_RDWR | O_CREAT, 0664); -+ if (fd < 0) { -+ syswarn("can't open tombstone log %s", tombstone_path_new); -+ } else if (!inn_lock_file(fd, INN_LOCK_WRITE, false)) { -+ /* Distinguish contention (another expireover holds it) -+ from other lock errors (NFS lockd outage, kernel -+ resource exhaustion, etc.). */ -+ if (errno == EAGAIN || errno == EACCES) { -+ warn("another expireover holds the tombstone log lock" -+ " on %s; disabling tombstone for this run", -+ tombstone_path_new); -+ } else { -+ syswarn("can't lock %s; disabling tombstone for this" -+ " run", -+ tombstone_path_new); -+ } -+ close(fd); -+ fd = -1; -+ } else { -+ /* Recover leftover content under the lock, filter through -+ SMretrieve, then truncate .NEW and rewrite. All under -+ the same fd/lock so no other writer can race. Two -+ sources to recover: -+ -+ (a) the existing .NEW file: residue from a prior -+ expireover run that crashed, or whose expirerm -+ never promoted .NEW to final. -+ (b) the existing final tombstone: residue from a -+ prior cycle where expirerm did promote .NEW to -+ final but expire never ran (or never finished) -+ so the entries were never consumed. Without -+ this recovery, this expireover's end-of-run -+ rename(.NEW, final) would silently overwrite -+ the unconsumed file, dropping its records. -+ -+ Both use verify_tombstone_lines, which preserves -+ entries on transient SMretrieve errors so a momentary -+ storage outage doesn't silently lose cancel records. */ -+ struct buffer kept; -+ struct stat sb; -+ char *raw; -+ ssize_t got = 0; -+ -+ kept.data = NULL; -+ kept.size = kept.used = kept.left = 0; -+ -+ /* Recover unconsumed final tombstone first. */ -+ raw = slurp_tombstone(tombstone_path, &got); -+ if (raw != NULL) { -+ verify_tombstone_lines(raw, got, &kept); -+ free(raw); -+ /* Unlink whether or not we extracted entries: the -+ active .NEW (after this run's rename) becomes the -+ new authoritative final. Preserving the old one -+ would risk a future expireover folding the same -+ entries in twice. */ -+ if (unlink(tombstone_path) < 0 && errno != ENOENT) -+ syswarn("can't unlink leftover %s", tombstone_path); -+ } -+ -+ /* Recover .NEW leftover. */ -+ if (fstat(fd, &sb) == 0 && sb.st_size > 0) { -+ raw = xmalloc(sb.st_size + 1); -+ got = 0; -+ while (got < sb.st_size) { -+ ssize_t n = read(fd, raw + got, sb.st_size - got); -+ if (n <= 0) -+ break; -+ got += n; -+ } -+ verify_tombstone_lines(raw, got, &kept); -+ free(raw); -+ } -+ if (ftruncate(fd, 0) < 0) { -+ syswarn("can't truncate %s", tombstone_path_new); -+ close(fd); -+ fd = -1; -+ } else if (lseek(fd, 0, SEEK_SET) < 0) { -+ syswarn("can't seek %s", tombstone_path_new); -+ close(fd); -+ fd = -1; -+ } else { -+ OVtombstonefile = fdopen(fd, "w"); -+ if (OVtombstonefile == NULL) { -+ syswarn("can't fdopen %s", tombstone_path_new); -+ close(fd); -+ fd = -1; -+ } else { -+ /* Format marker. Readers tolerate comment lines -+ (lines starting with #), so any future format -+ change can be detected by inspecting this -+ header without breaking earlier readers. */ -+ if (fputs(TOMBSTONE_HEADER, OVtombstonefile) -+ == EOF) -+ syswarn("can't write header to %s", -+ tombstone_path_new); -+ if (kept.left > 0) { -+ /* Replay verified leftover entries before -+ this run's new entries are appended by -+ OVEXPremove. (struct buffer convention: -+ appended data lives at data[used .. -+ used+left]; used==0 here so we start at -+ data.) */ -+ if (fwrite(kept.data + kept.used, 1, kept.left, -+ OVtombstonefile) -+ != kept.left -+ || fflush(OVtombstonefile) == EOF) -+ syswarn("can't write recovered leftover" -+ " to %s", -+ tombstone_path_new); -+ } -+ } -+ } -+ free(kept.data); -+ } -+ if (OVtombstonefile == NULL) { -+ free(tombstone_path); -+ free(tombstone_path_new); -+ tombstone_path = NULL; -+ tombstone_path_new = NULL; -+ } -+ } -+ - /* Loop through each line of the input file and process each group, - writing data to the lowmark file if desired. */ - line = QIOread(qp); -@@ -276,6 +547,46 @@ - if (lowmark != NULL) - if (fclose(lowmark) == EOF) - syswarn("can't close %s", lowmark_path); -+ -+ /* Finalize the tombstone log. In inline-cancel mode, rename .NEW -+ to its final name iff we completed a full pass without errors -- -+ a partial log would let expire incorrectly drop history entries -+ for articles that were not actually cancelled this run. -+ In delayrm mode, leave the .NEW alone: expirerm performs the -+ rename after fastrm has actually deleted the articles. If -+ expirerm fails or is skipped, the .NEW is wiped by the next -+ expireover, so a partial log can never be consumed by expire. -+ No fsync: the log is advisory; if a crash loses the buffered -+ tail, expire just runs the slow path. */ -+ if (OVtombstonefile != NULL) { -+ if (fclose(OVtombstonefile) == EOF) { -+ syswarn("can't finalize tombstone log %s", tombstone_path_new); -+ tombstone_clean = false; -+ } -+ if (signalled) -+ tombstone_clean = false; -+ if (ovge.delayrm) { -+ /* Leave .NEW for expirerm. If we were signalled or hit -+ an error, unlink it so expirerm doesn't promote a -+ truncated log. */ -+ if (!tombstone_clean) { -+ if (unlink(tombstone_path_new) < 0) -+ syswarn("can't unlink %s", tombstone_path_new); -+ } -+ } else if (tombstone_clean) { -+ if (rename(tombstone_path_new, tombstone_path) < 0) { -+ syswarn("can't rename %s to %s", tombstone_path_new, -+ tombstone_path); -+ if (unlink(tombstone_path_new) < 0) -+ syswarn("can't unlink %s", tombstone_path_new); -+ } -+ } else { -+ if (unlink(tombstone_path_new) < 0) -+ syswarn("can't unlink %s", tombstone_path_new); -+ } -+ free(tombstone_path); -+ free(tombstone_path_new); -+ } - - return 0; - } -diff --git expire/expirerm.in expire/expirerm.in ---- expire/expirerm.in 2026-05-18 00:16:06.129706000 -0700 -+++ expire/expirerm.in 2026-05-18 00:16:11.125489000 -0700 -@@ -16,17 +16,38 @@ - | eval ${MAIL} - exit 0 - fi --if [ ! -f $1 ]; then -+if [ ! -f "$1" ]; then - echo "Expire called with no files to expire on $(hostname)" \ - | eval ${MAIL} - exit 0 - fi - - eval "cd ${SPOOL} \ -- && ${RMPROC} <$1 \ -- && mv $1 ${MOST_LOGS}/expire.list" --if [ -f $1 ]; then -+ && ${RMPROC} <\"$1\" \ -+ && mv \"$1\" ${MOST_LOGS}/expire.list" -+if [ -f "$1" ]; then - echo "Expire had problems removing articles on $(hostname)" \ - | eval ${MAIL} - exit 1 -+fi -+ -+## If expireover wrote a tombstone log for delayed-removal mode, the -+## .NEW file holds the tokens it scheduled for cancellation. Now that -+## fastrm has succeeded, promote it to its final name so the next -+## expire run can consume it. -+## -+## A failed promotion is a degradation, not a hard failure: the -+## articles are already gone from disk, so news.daily can continue. -+## The next expire just falls back to its slow per-article -+## SMretrieve check for whatever was supposed to be in this batch. -+## Email the newsmaster but exit 0 so the surrounding pipeline keeps -+## going. -+TOMBSTONE_NEW="${PATHDB}/expireover.tombstone.NEW" -+TOMBSTONE="${PATHDB}/expireover.tombstone" -+if [ -f "${TOMBSTONE_NEW}" ]; then -+ mv "${TOMBSTONE_NEW}" "${TOMBSTONE}" || { -+ echo "Expire could not promote ${TOMBSTONE_NEW} on $(hostname);" \ -+ "next expire will run the slow path for this batch" \ -+ | eval ${MAIL} -+ } - fi -diff --git expire/makehistory.c expire/makehistory.c ---- expire/makehistory.c 2026-05-18 00:16:06.129684000 -0700 -+++ expire/makehistory.c 2026-05-18 00:16:11.125727000 -0700 -@@ -287,6 +287,15 @@ - arrived = (time_t) atoll(line); - expires = (time_t) atoll(p); - } -+ /* Validate before TextToToken: that function returns an -+ * all-zero TOKEN on malformed input rather than an error -+ * sentinel, so a bad line would silently insert a bogus -+ * overview entry. */ -+ if (!IsToken(q)) { -+ warn("sorted overview file %s has a malformed token at %d", -+ SortedTmpPath, count); -+ continue; -+ } - token = TextToToken(q); - if (OVadd(token, r, strlen(r), arrived, expires) == OVADDFAILED) { - if (OVctl(OVSPACE, (void *) &f) -diff --git frontends/sm.c frontends/sm.c ---- frontends/sm.c 2026-05-18 00:16:06.154106000 -0700 -+++ frontends/sm.c 2026-05-18 00:16:11.125872000 -0700 -@@ -10,6 +10,8 @@ - #include "inn/innconf.h" - #include "inn/libinn.h" - #include "inn/messages.h" -+#include "inn/newsuser.h" -+#include "inn/paths.h" - #include "inn/qio.h" - #include "inn/storage.h" - #include "inn/wire.h" -@@ -259,6 +261,10 @@ - warn("could not remove %s: %s", id, SMerrorstr); - return false; - } -+ /* Record the cancel so a later expire run can drop the history -+ * entry without an SMretrieve. Best-effort; no-op when -+ * expiretombstone is disabled. */ -+ SMcanceltombstone(token); - } else { - article = SMretrieve(token, options->header ? RETR_HEAD : RETR_ALL); - if (article == NULL) { -@@ -360,6 +366,17 @@ - if (!SMsetup(SM_RDWR, &value)) - die("cannot set up storage manager"); - } -+ -+ /* On the -r/-d path with the cancel tombstone enabled, drop to -+ * the news user/group so cancels.tombstone is created (or -+ * appended) with ownership consistent with innd's writes. -+ * Bypassed under the INN_TESTSUITE env var so the test harness -+ * can run sm without a real news user on the system. All other -+ * sm modes (-i, -c, -H, -R, -s, retrieval) preserve the -+ * historical behaviour of running as the invoking user. */ -+ if (options.delete && innconf->expiretombstone -+ && getenv(INN_ENV_TESTSUITE) == NULL) -+ ensure_news_user_grp(true, true); - if (!SMinit()) - die("cannot initialize storage manager: %s", SMerrorstr); - -diff --git include/inn/innconf.h include/inn/innconf.h ---- include/inn/innconf.h 2026-05-18 00:16:06.144766000 -0700 -+++ include/inn/innconf.h 2026-05-18 00:16:11.126026000 -0700 -@@ -77,6 +77,9 @@ - OVERVIEW.FMT */ - struct vector - *extraoverviewhidden; /* Extra overview fields silently generated */ -+ bool expiretombstone; /* Use expireover -> expire tombstone log -+ to skip per-article SMretrieve in -+ expire? */ - bool groupbaseexpiry; /* Do expiry by newsgroup? */ - bool mergetogroups; /* Refile articles from to.* into to */ - bool nfswriter; /* Use NFS writer functionality */ -diff --git include/inn/ov.h include/inn/ov.h ---- include/inn/ov.h 2026-05-18 00:16:06.144243000 -0700 -+++ include/inn/ov.h 2026-05-18 00:16:11.126122000 -0700 -@@ -52,6 +52,7 @@ - } OVGE; - - extern bool OVstatall; -+ - bool OVopen(int mode); - bool OVgroupstats(char *group, int *lo, int *hi, int *count, int *flag); - bool OVgroupadd(char *group, ARTNUM lo, ARTNUM hi, char *flag); -diff --git include/inn/storage.h include/inn/storage.h ---- include/inn/storage.h 2026-05-18 00:16:06.144486000 -0700 -+++ include/inn/storage.h 2026-05-18 00:16:11.126228000 -0700 -@@ -106,6 +106,28 @@ - ARTHANDLE *SMnext(ARTHANDLE *article, const RETRTYPE amount); - void SMfreearticle(ARTHANDLE *article); - bool SMcancel(TOKEN token); -+ -+/* -+ * Best-effort: append a token to the out-of-band cancel tombstone log -+ * (${pathdb}/cancels.tombstone) so a later expire run can drop the -+ * matching history entry without an SMretrieve(RETR_STAT). Intended -+ * for callers that cancel articles outside the expireover/expirerm -+ * pipeline (innd's ARTcancel and sm -r). No-op when -+ * innconf->expiretombstone is false or token has type TOKEN_EMPTY. -+ * -+ * Concurrency: appenders take a non-blocking shared (F_RDLCK) fcntl -+ * lock; expire's consumer takes an exclusive lock briefly during -+ * snapshot/rename. POSIX O_APPEND atomicity for sub-PIPE_BUF -+ * writes keeps single-line tokens from interleaving across -+ * concurrent appenders. -+ * -+ * Returns true on success (token appended, flushed, file closed), -+ * false on any failure path. Failures are logged via syswarn; -+ * callers can ignore the return value (the cancel itself has -+ * already succeeded by the time this is called). -+ */ -+bool SMcanceltombstone(TOKEN token); -+ - bool SMprobe(PROBETYPE type, TOKEN *token, void *value); - bool SMflushcacheddata(FLUSHTYPE type); - void SMprintfiles(FILE *file, TOKEN token, char **xref, int ngroups); -diff --git include/inn/tombstone.h include/inn/tombstone.h ---- include/inn/tombstone.h 1969-12-31 17:00:00.000000000 -0700 -+++ include/inn/tombstone.h 2026-05-18 00:16:11.126336000 -0700 -@@ -0,0 +1,100 @@ -+/* -+** Helpers for the expire-tombstone log files. -+** -+** These routines are shared between the expire binary (which consumes -+** the logs to skip per-article SMretrieve(RETR_STAT) calls) and the -+** test suite that exercises the consumption logic directly. -+** -+** See doc/pod/inn.conf.pod under "expiretombstone" for the file -+** semantics: -+** - ${pathdb}/expireover.tombstone : written by expireover/expirerm -+** via OVEXPremove, atomic .NEW -> final rename -+** - ${pathdb}/cancels.tombstone : appended continuously by innd -+** and sm via SMcanceltombstone() for cancels outside the -+** expireover/expirerm pipeline -+*/ -+ -+#ifndef INN_TOMBSTONE_H -+#define INN_TOMBSTONE_H -+ -+#include "inn/hashtab.h" -+#include "inn/portable-macros.h" -+#include "inn/portable-stdbool.h" -+#include "inn/storage.h" -+ -+/* -+** Format marker written at the top of every tombstone file. -+** Readers skip leading comment lines silently so the marker can -+** evolve without breaking earlier consumers. -+*/ -+#define TOMBSTONE_HEADER "# inn-tombstone v1\n" -+ -+BEGIN_DECLS -+ -+/* -+** Create a hashset suited to holding TOKEN keys (each entry is a -+** TOKEN allocated by the caller; the hashset owns and frees them). -+** size is the initial bucket hint; the hash auto-expands. -+*/ -+struct hash *tombstone_hash_create(size_t size); -+ -+/* -+** Read tombstone entries from path into the hashset. Each non-blank -+** line is parsed with TextToToken and inserted; malformed lines are -+** skipped with a warning. Duplicates are dropped silently. Returns -+** the number of valid entries seen, or 0 if the file does not exist -+** (ENOENT is silent; other open errors emit syswarn). -+** -+** If out_error is non-NULL, *out_error is set to true when the read -+** was incomplete (open failed for a non-ENOENT reason or fgets hit a -+** stream error mid-file) and to false otherwise. Callers that cache -+** freshness based on file mtime should not treat a partial read as -+** authoritative. ENOENT does not set the flag because an absent -+** file is a successful "no entries" read. -+*/ -+unsigned long tombstone_read(struct hash *h, const char *path, -+ bool *out_error); -+ -+/* -+** Atomically snapshot a continuously-appended tombstone file by -+** renaming it to "${path}.processing" under an exclusive POSIX lock. -+** This serializes against concurrent appenders that hold the same -+** lock (e.g., SMcanceltombstone()), so any in-progress write either -+** completes before the rename or proceeds against the renamed inode. -+** -+** Returns the snapshot path (caller must free and unlink after -+** consumption) or NULL if the source file does not exist or cannot -+** be renamed. -+*/ -+char *tombstone_rename_for_processing(const char *path); -+ -+/* -+** Look up a TOKEN in the hashset. Returns true if present. -+*/ -+bool tombstone_present(struct hash *h, const TOKEN *token); -+ -+/* -+** Ensure path exists and starts with TOMBSTONE_HEADER under an -+** exclusive POSIX lock. Idempotent: a file whose first bytes -+** already match TOMBSTONE_HEADER is left untouched. Otherwise the -+** helper shifts any existing content right by sizeof(TOMBSTONE_ -+** HEADER)-1 bytes via a chunked end-to-start pread/pwrite walk on -+** the same fd, then writes the header at offset 0. Working -+** memory is bounded by an internal chunk size regardless of file -+** size, so a pathological never-consumed file cannot OOM. -+** -+** An empty file (or one freshly created by this call) just gets -+** the header written at offset 0. An appender that raced a -+** cancel into a new live file between the consumer's unlink and -+** this call has its content preserved verbatim below the header. -+** Best-effort: failures log via syswarn but do not raise. -+** -+** Called by expire after consuming cancels.tombstone.processing -+** to leave a header-only file behind so the nnrpd fast path -+** remains active through quiet inter-cancel periods. -+*/ -+void tombstone_ensure_header(const char *path); -+ -+END_DECLS -+ -+#endif /* INN_TOMBSTONE_H */ -diff --git innd/art.c innd/art.c ---- innd/art.c 2026-05-18 00:16:06.151875000 -0700 -+++ innd/art.c 2026-05-18 00:16:11.126750000 -0700 -@@ -1280,9 +1280,15 @@ - /* Get stored message and zap them. */ - if (innconf->enableoverview) - OVcancel(token); -- if (!SMcancel(token) && SMerrno != SMERR_NOENT && SMerrno != SMERR_UNINIT) -+ if (SMcancel(token) || SMerrno == SMERR_NOENT) { -+ /* Record the out-of-band cancel so a later expire run can drop -+ * the history entry without an SMretrieve(RETR_STAT). Best- -+ * effort and a no-op when expiretombstone is disabled. */ -+ SMcanceltombstone(token); -+ } else if (SMerrno != SMERR_UNINIT) { - syslog(L_ERROR, "%s cant cancel %s (SMerrno %d)", LogName, - TokenToText(token), SMerrno); -+ } - if (innconf->immediatecancel && !SMflushcacheddata(SM_CANCELLEDART)) - syslog(L_ERROR, "%s cant cancel cached %s", LogName, - TokenToText(token)); -diff --git lib/innconf.c lib/innconf.c ---- lib/innconf.c 2026-05-18 00:16:06.146141000 -0700 -+++ lib/innconf.c 2026-05-18 00:16:11.126974000 -0700 -@@ -71,6 +71,7 @@ - {K(enableoverview), BOOL(true) }, - {K(extraoverviewadvertised), LIST(NULL) }, - {K(extraoverviewhidden), LIST(NULL) }, -+ {K(expiretombstone), BOOL(false) }, - {K(fromhost), STRING(NULL) }, - {K(groupbaseexpiry), BOOL(true) }, - {K(mailcmd), STRING(NULL) }, -diff --git nnrpd/article.c nnrpd/article.c ---- nnrpd/article.c 2026-05-18 00:16:06.128622000 -0700 -+++ nnrpd/article.c 2026-05-18 00:16:11.129572000 -0700 -@@ -7,13 +7,16 @@ - #include - #include - #include -+#include - #include - - #include "cache.h" - #include "inn/innconf.h" -+#include "inn/libinn.h" - #include "inn/messages.h" - #include "inn/ov.h" - #include "inn/overview.h" -+#include "inn/tombstone.h" - #include "inn/wire.h" - #include "nnrpd.h" - #include "tls.h" -@@ -260,17 +263,131 @@ - } - } - -+/* -+** Cancel-tombstone fast path for ARTinstorebytoken. When both -+** innconf->expiretombstone and PERMaccessconf->nnrpdcheckart are true, -+** consult ${pathdb}/cancels.tombstone before falling through to the -+** per-article SMretrieve syscall. The tombstone records cancels -+** written by innd's ARTcancel and by sm -r, the out-of-band paths -+** that can leave overview entries pointing at gone storage in the -+** brief race window before overview cleanup propagates, or until the -+** next expireover for sm -r. -+** -+** Lazy per-connection: loaded on first call, refreshed on mtime -+** change. Memory cost is ~50 bytes per cancel; typical sites have -+** hundreds of entries. Falls through to SMretrieve on load failure. -+*/ -+static struct hash *nnrpd_tombstone = NULL; -+static char *nnrpd_tombstone_path = NULL; -+static time_t nnrpd_tombstone_mtime = 0; -+static off_t nnrpd_tombstone_size = 0; -+ -+/* -+** Bring the per-connection tombstone cache up to date. Returns true -+** if a usable hashset is loaded after the call (caller may then -+** consult tombstone_present), false otherwise (file missing or load -+** failed; caller should fall through to the slow path). -+** -+** Stat'd on every call so a cancel recorded by another process -+** becomes visible to readers immediately. The cost is one stat() -+** on a fixed path that stays hot in the dentry cache, which is -+** trivially cheap compared to the per-article SMretrieve syscalls -+** the fast path elides. Reload only happens when mtime or size -+** changes, so the read+parse cost is paid at most once per cancel. -+** -+** Freshness is keyed on both mtime and size: mtime alone has 1- -+** second granularity, so a rename-and-recreate within the same -+** second can land a fresh inode with the cached mtime; size is -+** monotonic-append between rotations and so changes whenever a -+** new line is added. Together they catch every modification. -+** -+** If tombstone_read reports a partial read (mid-file ferror), the -+** cached mtime/size are not updated, so the next call retries the -+** read instead of trusting a partial hashset. -+*/ -+static bool -+nnrpd_tombstone_refresh(void) -+{ -+ /* Initial hash bucket count for the per-connection tombstone cache. -+ * Sized for the typical case (hundreds of cancels per cycle); the -+ * hashset auto-expands when the load factor is exceeded. */ -+ static const size_t initial_buckets = 256; -+ struct stat sb; -+ bool read_error = false; -+ -+ if (nnrpd_tombstone_path == NULL) -+ nnrpd_tombstone_path = -+ concatpath(innconf->pathdb, "cancels.tombstone"); -+ -+ /* If the file has gone away since our last load, drop our cache -+ * and fall through to the slow path. */ -+ if (stat(nnrpd_tombstone_path, &sb) < 0) { -+ if (nnrpd_tombstone != NULL) { -+ hash_free(nnrpd_tombstone); -+ nnrpd_tombstone = NULL; -+ nnrpd_tombstone_mtime = 0; -+ nnrpd_tombstone_size = 0; -+ } -+ return false; -+ } -+ -+ /* Fresh enough? */ -+ if (nnrpd_tombstone != NULL && sb.st_mtime == nnrpd_tombstone_mtime -+ && sb.st_size == nnrpd_tombstone_size) -+ return true; -+ -+ /* (Re)load. */ -+ if (nnrpd_tombstone != NULL) -+ hash_free(nnrpd_tombstone); -+ nnrpd_tombstone = tombstone_hash_create(initial_buckets); -+ tombstone_read(nnrpd_tombstone, nnrpd_tombstone_path, &read_error); -+ if (read_error) { -+ /* Partial parse; do not advance the freshness key so the -+ * next call retries. The hashset still holds whatever lines -+ * we did parse, which is not wrong (those are real cancels), -+ * just incomplete. */ -+ return true; -+ } -+ nnrpd_tombstone_mtime = sb.st_mtime; -+ nnrpd_tombstone_size = sb.st_size; -+ return true; -+} -+ -+ - bool - ARTinstorebytoken(TOKEN token) - { - ARTHANDLE *art; - struct timeval stv, etv; - -- if (PERMaccessconf->nnrpdoverstats) { -- gettimeofday(&stv, NULL); -+ /* Fast path: if both expiretombstone and nnrpdcheckart are -+ * enabled and the cache loads, the cancels.tombstone log -+ * records out-of-band cancels (innd ARTcancel, sm -r) that may -+ * not yet be reflected in overview. Tombstone hit means the -+ * article is gone; tombstone miss on a non-self-expiring -+ * backend means it is alive and we can skip the SMretrieve -+ * syscall. Self-expiring backends (CNFS) still need the slow -+ * path because wrap-around bypasses SMcancel and the tombstone. -+ * -+ * The nnrpdcheckart gate matches the documented contract: -+ * tombstone-driven existence checks are an alternative -+ * implementation of nnrpdcheckart's existence verification, so -+ * we should not run them when the operator has explicitly -+ * disabled article-existence checking. -+ * -+ * SMprobe(SELFEXPIRE) is checked first so pure-CNFS sites pay -+ * only that probe (a static per-method attribute, no I/O) and -+ * skip both the hash lookup and the cache refresh. */ -+ if (innconf->expiretombstone && PERMaccessconf->nnrpdcheckart -+ && !SMprobe(SELFEXPIRE, &token, NULL) -+ && nnrpd_tombstone_refresh()) { -+ return !tombstone_present(nnrpd_tombstone, &token); - } -- art = SMretrieve(token, RETR_STAT); -+ - /* XXX This isn't really overstats, is it? */ -+ if (PERMaccessconf->nnrpdoverstats) -+ gettimeofday(&stv, NULL); -+ art = SMretrieve(token, RETR_STAT); - if (PERMaccessconf->nnrpdoverstats) { - gettimeofday(&etv, NULL); - OVERartcheck += (etv.tv_sec - stv.tv_sec) * 1000; -diff --git samples/inn.conf.in samples/inn.conf.in ---- samples/inn.conf.in 2026-05-18 00:16:06.149493000 -0700 -+++ samples/inn.conf.in 2026-05-18 00:16:11.127103000 -0700 -@@ -67,6 +67,7 @@ - expirebloomfp: 10000 - extraoverviewadvertised: [ ] - extraoverviewhidden: [ ] -+expiretombstone: false - groupbaseexpiry: true - mergetogroups: false - nfswriter: false -diff --git storage/Makefile storage/Makefile ---- storage/Makefile 2026-05-18 00:16:06.152619000 -0700 -+++ storage/Makefile 2026-05-18 00:16:11.127275000 -0700 -@@ -12,7 +12,7 @@ - CFLAGS = $(GCFLAGS) -I. $(BDB_CPPFLAGS) $(SQLITE3_CPPFLAGS) - - SOURCES = expire.c interface.c methods.c ov.c overdata.c overview.c \ -- ovmethods.c $(METHOD_SOURCES) -+ ovmethods.c tombstone.c $(METHOD_SOURCES) - OBJECTS = $(SOURCES:.c=.o) - LOBJECTS = $(OBJECTS:.o=.lo) - -diff --git storage/expire.c storage/expire.c ---- storage/expire.c 2026-05-18 00:16:06.152668000 -0700 -+++ storage/expire.c 2026-05-18 00:16:11.127467000 -0700 -@@ -15,6 +15,7 @@ - #include "inn/bloom.h" - #include "inn/innconf.h" - #include "inn/libinn.h" -+#include "inn/messages.h" - #include "inn/ov.h" - #include "inn/overview.h" - #include "inn/paths.h" -@@ -104,6 +105,7 @@ - This should be cleaned up with a better internal interface. */ - time_t OVnow; - FILE *EXPunlinkfile; -+FILE *OVtombstonefile; - bool OVignoreselfexpire; - bool OVusepost; - bool OVkeep; -@@ -526,16 +528,53 @@ - } - if (EXPunlinkfile && xref != NULL) { - SMprintfiles(EXPunlinkfile, token, xref, ngroups); -- if (!ferror(EXPunlinkfile)) -+ if (!ferror(EXPunlinkfile)) { -+ /* Deferred-removal path: actual SMcancel happens later via -+ * expirerm/fastrm. Record the token in the tombstone log -+ * now so the eventual expire run can consume it. The -+ * .NEW -> final rename is performed by expirerm after -+ * fastrm succeeds (see expirerm.in); if expirerm fails or -+ * is skipped, the .NEW file is wiped by the next -+ * expireover and never consumed, so a partial rmfile -+ * never lets expire drop history for live articles. */ -+ if (OVtombstonefile != NULL) { -+ fprintf(OVtombstonefile, "%s\n", TokenToText(token)); -+ if (ferror(OVtombstonefile)) { -+ syswarn("can't write tombstone log; ignoring it" -+ " for rest of run"); -+ fclose(OVtombstonefile); -+ OVtombstonefile = NULL; -+ } -+ } - return; -- fprintf(stderr, "Can't write to -z file, %s\n", strerror(errno)); -- fprintf(stderr, "(Will ignore it for rest of run.)\n"); -+ } -+ syswarn("can't write to -z file; ignoring it for rest of run"); - fclose(EXPunlinkfile); - EXPunlinkfile = NULL; - } -- if (!SMcancel(token) && SMerrno != SMERR_NOENT && SMerrno != SMERR_UNINIT) -- fprintf(stderr, "Can't unlink %s: %s\n", TokenToText(token), -- SMerrorstr); -+ /* Inline-cancel path. Tombstone is written only on a successful -+ * SMcancel (or NOENT, where the article is already gone): the log -+ * must be a strict subset of articles actually removed. Otherwise -+ * a non-NOENT/UNINIT failure (EIO, EBUSY, fs permission, etc.) -+ * would let the next expire drop the history entry while the -+ * article remains on disk: an undetectable permanent orphan. -+ * The inverse risk (cancelled but not logged on a crash between -+ * SMcancel and fprintf) leaves a history entry pointing at a -+ * missing article, which is recoverable by a periodic non-tombstone -+ * reconciliation run. */ -+ if (SMcancel(token) || SMerrno == SMERR_NOENT) { -+ if (OVtombstonefile != NULL) { -+ fprintf(OVtombstonefile, "%s\n", TokenToText(token)); -+ if (ferror(OVtombstonefile)) { -+ syswarn("can't write tombstone log; ignoring it for" -+ " rest of run"); -+ fclose(OVtombstonefile); -+ OVtombstonefile = NULL; -+ } -+ } -+ } else if (SMerrno != SMERR_UNINIT) { -+ warn("can't unlink %s: %s", TokenToText(token), SMerrorstr); -+ } - } - - /* -diff --git storage/interface.c storage/interface.c ---- storage/interface.c 2026-05-18 00:16:06.153708000 -0700 -+++ storage/interface.c 2026-05-18 00:16:11.127675000 -0700 -@@ -6,7 +6,9 @@ - - #include - #include -+#include - #include -+#include - - #include "conffile.h" - #include "inn/innconf.h" -@@ -815,6 +817,139 @@ - return false; - } - return storage_methods[typetoindex[token.type]].cancel(token); -+} -+ -+ -+/* -+** Append a token to the out-of-band cancel tombstone log. Best-effort: -+** failures are logged but do not affect the caller. Safe for concurrent -+** use across processes via inn_lock_file (fcntl POSIX locks; NFS-correct). -+** -+** Used by callers (innd, sm) that cancel articles outside the -+** expireover/expirerm pipeline. expire reads this log alongside the -+** expireover.tombstone so it can drop history entries for articles -+** cancelled out-of-band without per-article SMretrieve(RETR_STAT). -+** -+** No-op when innconf->expiretombstone is false. No-op for TOKEN_EMPTY. -+** -+** Locking model: shared (F_RDLCK) for appenders, exclusive (F_WRLCK) -+** for the consumer's read+rename phase (see tombstone_rename_for_ -+** processing). The lock type names "read" and "write" here describe -+** reader-vs-truncator coordination, not file access pattern; both -+** innd and sm write the file under the shared lock. POSIX guarantees -+** that each write(2) to a regular file opened with O_APPEND is -+** atomic with respect to other writers, so single-line tokens -+** interleave correctly without per-write serialization. POSIX -+** requires F_RDLCK only on a descriptor open for reading, so we -+** open O_RDWR even though we only write here. -+** -+** Lock is acquired non-blocking. innd's main loop must never wait -+** on filesystem locking (NFS lockd outages, expire's brief -+** rename hold). If the lock is contested, we skip this tombstone -+** write rather than block. Concurrent appenders do not contend -+** because shared locks coexist; only expire's exclusive lock -+** briefly excludes them, during the consumer's rename of cancels. -+** tombstone to .processing and during the post-consume header -+** re-seeding shift (tombstone_ensure_header). Both windows -+** scale with the file size at the moment they fire; for a -+** typical site (a few KiB of tokens) the combined hold is well -+** under a millisecond and the EAGAIN retry below absorbs it. -+*/ -+bool -+SMcanceltombstone(TOKEN token) -+{ -+ char *path; -+ int fd; -+ FILE *fp; -+ const char *text; -+ bool ok = true; -+ -+ /* Both knobs are required: expire only consumes the log when -+ * groupbaseexpiry is also true, so writing without that gate -+ * would leak entries forever and bloat nnrpd's per-connection -+ * cache. */ -+ if (!innconf->expiretombstone || !innconf->groupbaseexpiry) -+ return true; -+ if (token.type == TOKEN_EMPTY) -+ return true; -+ -+ path = concatpath(innconf->pathdb, "cancels.tombstone"); -+ /* O_RDWR (not O_WRONLY) so we can take F_RDLCK below; POSIX -+ * mandates lock-mode-matches-fd-mode. */ -+ fd = open(path, O_RDWR | O_CREAT | O_APPEND, 0664); -+ if (fd < 0) { -+ syswarn("can't open %s for tombstone append", path); -+ free(path); -+ return false; -+ } -+ /* Shared, non-blocking. Multiple appenders coexist; only blocked -+ * by expire's exclusive consumer lock during its brief rename. -+ * -+ * Race window: between our open() and our lock(), expire may -+ * rename(cancels.tombstone, cancels.tombstone.processing), -+ * leaving us with a fd pointing at the renamed (doomed) inode. -+ * If we acquired the lock on that fd and wrote, our entry would -+ * be consumed only by expire's already-loaded snapshot -- and -+ * our writes appended after the rename are not in that snapshot, -+ * so they would be unlinked when expire finishes. -+ * -+ * Resolution: on EAGAIN/EACCES (lock held by expire's rename -+ * window), close and reopen the path. After expire releases -+ * its exclusive lock, the path resolves to a fresh inode (or -+ * nothing, in which case O_CREAT makes a new one). Either way, -+ * the second open lands on the live file the next consumer -+ * will see. A 1ms sleep is more than enough for expire to -+ * complete its rename and release. */ -+ if (!inn_lock_file(fd, INN_LOCK_READ, false)) { -+ if (errno == EAGAIN || errno == EACCES) { -+ struct timespec ts = {0, 1000000L}; /* 1ms */ -+ close(fd); -+ nanosleep(&ts, NULL); -+ fd = open(path, O_RDWR | O_CREAT | O_APPEND, 0664); -+ if (fd < 0) { -+ syswarn("can't reopen %s for tombstone append", path); -+ free(path); -+ return false; -+ } -+ if (!inn_lock_file(fd, INN_LOCK_READ, false)) { -+ syswarn("can't lock %s after retry; skipping" -+ " tombstone entry", -+ path); -+ close(fd); -+ free(path); -+ return false; -+ } -+ } else { -+ syswarn("can't lock %s; skipping tombstone entry", path); -+ close(fd); -+ free(path); -+ return false; -+ } -+ } -+ fp = fdopen(fd, "a"); -+ if (fp == NULL) { -+ syswarn("can't fdopen %s", path); -+ inn_lock_file(fd, INN_LOCK_UNLOCK, true); -+ close(fd); -+ free(path); -+ return false; -+ } -+ text = TokenToText(token); -+ /* Single fprintf + fflush keeps the kernel-visible write to one -+ * syscall (~38-byte payload) so the per-write O_APPEND atomicity -+ * guarantee holds for concurrent shared-lock appenders. */ -+ if (fprintf(fp, "%s\n", text) < 0 || fflush(fp) == EOF) { -+ syswarn("can't write to %s", path); -+ ok = false; -+ } -+ /* fclose closes the underlying fd, which releases the fcntl -+ * POSIX lock; no separate INN_LOCK_UNLOCK needed. */ -+ if (fclose(fp) == EOF) { -+ syswarn("can't close %s", path); -+ ok = false; -+ } -+ free(path); -+ return ok; - } - - bool -diff --git storage/ovinterface.h storage/ovinterface.h ---- storage/ovinterface.h 2026-05-18 00:16:06.152455000 -0700 -+++ storage/ovinterface.h 2026-05-18 00:16:11.127767000 -0700 -@@ -5,6 +5,8 @@ - #ifndef OVINTERFACE_H - #define OVINTERFACE_H - -+#include -+ - #include "config.h" - #include "inn/history.h" - #include "inn/ov.h" -@@ -43,6 +45,17 @@ - extern struct bloom_filter *OVtokencache; - extern time_t OVnow; - extern FILE *EXPunlinkfile; -+ -+/* Tombstone log: when set, OVEXPremove appends the textual form of each -+ * successfully cancelled token to this stream. A subsequent expire run -+ * consumes the log to drop history entries for those articles without -+ * doing per-article SMretrieve(RETR_STAT) calls. Set by expireover (and -+ * by fastrm when called via expirerm in delayrm mode). Internal -- -+ * managed via direct extern access by expireover/fastrm, not via OVctl, -+ * because the file lifecycle (atomic .NEW -> final rename on success) -+ * does not fit the per-call OVctl pattern. */ -+extern FILE *OVtombstonefile; -+ - extern bool OVignoreselfexpire; - extern bool OVusepost; - extern bool OVkeep; -diff --git storage/tombstone.c storage/tombstone.c ---- storage/tombstone.c 1969-12-31 17:00:00.000000000 -0700 -+++ storage/tombstone.c 2026-05-18 00:16:11.127901000 -0700 -@@ -0,0 +1,277 @@ -+/* -+** Helpers for the expire-tombstone log files. -+** -+** See include/inn/tombstone.h for the API and inn.conf.pod under -+** "expiretombstone" for the file semantics. Shared between the -+** expire binary and the test suite so both exercise identical -+** parsing, hashing, and atomic-snapshot logic. -+*/ -+ -+#include "portable/system.h" -+ -+#include -+#include -+#include -+#include -+#include -+ -+#include "inn/hashtab.h" -+#include "inn/libinn.h" -+#include "inn/messages.h" -+#include "inn/storage.h" -+#include "inn/tombstone.h" -+ -+ -+/* -+** Hashset glue. Keys are 18-byte TOKEN structs (1 type + 1 class + -+** 16 token bytes) hashed and compared as raw bytes. hash_lookup2 -+** derives a stable unsigned long from the bytes; tombstone_equal -+** uses memcmp for an exact match. -+*/ -+ -+static unsigned long -+ts_hash(const void *p) -+{ -+ return hash_lookup2((const char *) p, sizeof(TOKEN), 0); -+} -+ -+static const void * -+ts_key(const void *p) -+{ -+ return p; -+} -+ -+static bool -+ts_equal(const void *a, const void *b) -+{ -+ return memcmp(a, b, sizeof(TOKEN)) == 0; -+} -+ -+ -+struct hash * -+tombstone_hash_create(size_t size) -+{ -+ return hash_create(size, ts_hash, ts_key, ts_equal, free); -+} -+ -+ -+unsigned long -+tombstone_read(struct hash *h, const char *path, bool *out_error) -+{ -+ FILE *f; -+ char line[SMBUF]; -+ size_t len; -+ unsigned long count = 0; -+ -+ if (out_error != NULL) -+ *out_error = false; -+ if (h == NULL || path == NULL) -+ return 0; -+ -+ f = fopen(path, "r"); -+ if (f == NULL) { -+ if (errno != ENOENT) { -+ syswarn("can't open %s; ignoring", path); -+ if (out_error != NULL) -+ *out_error = true; -+ } -+ return 0; -+ } -+ while (fgets(line, sizeof(line), f) != NULL) { -+ TOKEN t; -+ TOKEN *entry; -+ -+ len = strlen(line); -+ while (len > 0 && (line[len - 1] == '\n' || line[len - 1] == '\r')) -+ line[--len] = '\0'; -+ if (len == 0) -+ continue; -+ /* Comment lines (e.g., "# inn-tombstone v1") are reserved for -+ * future format markers and metadata. Readers ignore them -+ * silently so format evolution is graceful. */ -+ if (line[0] == '#') -+ continue; -+ if (!IsToken(line)) { -+ warn("malformed tombstone entry in %s: %s", path, line); -+ continue; -+ } -+ t = TextToToken(line); -+ entry = xmalloc(sizeof(TOKEN)); -+ *entry = t; -+ if (!hash_insert(h, entry, entry)) { -+ /* Already present (duplicate cancel), free the redundant -+ * allocation; the existing entry stays. */ -+ free(entry); -+ } -+ count++; -+ } -+ if (ferror(f)) { -+ syswarn("error reading %s; will use what was read", path); -+ if (out_error != NULL) -+ *out_error = true; -+ } -+ fclose(f); -+ return count; -+} -+ -+ -+char * -+tombstone_rename_for_processing(const char *path) -+{ -+ char *snapshot_path; -+ int fd; -+ -+ if (path == NULL) -+ return NULL; -+ -+ fd = open(path, O_RDWR); -+ if (fd < 0) { -+ if (errno != ENOENT) -+ syswarn("can't open %s", path); -+ return NULL; -+ } -+ if (!inn_lock_file(fd, INN_LOCK_WRITE, true)) { -+ syswarn("can't lock %s", path); -+ close(fd); -+ return NULL; -+ } -+ snapshot_path = concat(path, ".processing", (char *) 0); -+ if (rename(path, snapshot_path) < 0) { -+ syswarn("can't rename %s to %s", path, snapshot_path); -+ free(snapshot_path); -+ close(fd); -+ return NULL; -+ } -+ /* close releases the fcntl POSIX lock. */ -+ close(fd); -+ return snapshot_path; -+} -+ -+ -+bool -+tombstone_present(struct hash *h, const TOKEN *token) -+{ -+ if (h == NULL || token == NULL) -+ return false; -+ return hash_lookup(h, token) != NULL; -+} -+ -+ -+void -+tombstone_ensure_header(const char *path) -+{ -+ static const char header[] = TOMBSTONE_HEADER; -+ static const size_t header_len = sizeof(header) - 1; -+ /* Bounded working memory: shift the file right by header_len bytes -+ * in chunks read end-to-start. 64 KiB keeps us cache-warm without -+ * paying excessive syscall overhead. Memory is bounded regardless -+ * of file size, so a pathological never-consumed file cannot OOM -+ * the expire run. */ -+ static const size_t CHUNK = 64 * 1024; -+ int fd; -+ struct stat sb; -+ off_t size; -+ char *chunk_buf; -+ off_t pos; -+ -+ if (path == NULL) -+ return; -+ -+ /* No O_APPEND: we use explicit offsets via pread/pwrite, and on -+ * Linux O_APPEND would override pwrite's offset and force every -+ * write to EOF -- breaking the shift loop below. */ -+ fd = open(path, O_RDWR | O_CREAT, 0664); -+ if (fd < 0) { -+ syswarn("can't open %s for header restore", path); -+ return; -+ } -+ -+ /* Blocking exclusive lock. Appenders use shared locks, so we -+ * conflict; the operation runs for microseconds to milliseconds -+ * and only once per expire cycle, so blocking is fine. The -+ * serialization is what makes the read-modify-write safe -+ * against appenders. */ -+ if (!inn_lock_file(fd, INN_LOCK_WRITE, true)) { -+ syswarn("can't lock %s for header restore", path); -+ close(fd); -+ return; -+ } -+ -+ if (fstat(fd, &sb) < 0) { -+ syswarn("can't stat %s for header restore", path); -+ close(fd); -+ return; -+ } -+ size = sb.st_size; -+ -+ /* Idempotency fast path: if the file already starts with the -+ * header line, the rest is well-formed appended content and we -+ * have nothing to do. Without this, every call would prepend an -+ * additional header, growing the file by header_len each expire -+ * cycle on a site with zero cancels per cycle. */ -+ if (size >= (off_t) header_len) { -+ char prefix[sizeof(header)]; -+ -+ if (pread(fd, prefix, header_len, 0) == (ssize_t) header_len -+ && memcmp(prefix, header, header_len) == 0) { -+ close(fd); -+ return; -+ } -+ } -+ -+ /* Shift existing content right by header_len bytes, walking from -+ * end to start so we never read a region we have already -+ * overwritten. This works in place on the same fd; memory is -+ * bounded by CHUNK regardless of file size. A crash mid-shift -+ * leaves the file with some duplicated/partial lines, which -+ * tombstone_read tolerates (malformed lines warn and skip); no -+ * infinite escalation across crashes because the next run's -+ * idempotency check sees no header and shifts once more, -+ * producing a header-prefixed file the run after that leaves -+ * alone. */ -+ chunk_buf = xmalloc(CHUNK); -+ pos = size; -+ while (pos > 0) { -+ size_t this_chunk = (pos < (off_t) CHUNK) ? (size_t) pos : CHUNK; -+ off_t src = pos - this_chunk; -+ size_t io = 0; -+ ssize_t n; -+ -+ while (io < this_chunk) { -+ n = pread(fd, chunk_buf + io, this_chunk - io, src + io); -+ if (n <= 0) -+ break; -+ io += n; -+ } -+ if (io < this_chunk) { -+ syswarn("short read on %s during header restore", path); -+ free(chunk_buf); -+ close(fd); -+ return; -+ } -+ io = 0; -+ while (io < this_chunk) { -+ n = pwrite(fd, chunk_buf + io, this_chunk - io, -+ src + header_len + io); -+ if (n <= 0) { -+ syswarn("short write on %s during header restore", path); -+ free(chunk_buf); -+ close(fd); -+ return; -+ } -+ io += n; -+ } -+ pos = src; -+ } -+ free(chunk_buf); -+ -+ /* Header at offset 0. Single small pwrite; partial-write on a -+ * regular file is vanishingly rare but if it happens -+ * tombstone_read still treats the leading '#' bytes as a -+ * comment and skips them. */ -+ if (pwrite(fd, header, header_len, 0) != (ssize_t) header_len) -+ syswarn("can't write header to %s", path); -+ -+ /* close releases the fcntl POSIX lock. */ -+ close(fd); -+} -diff --git support/mkmanifest support/mkmanifest ---- support/mkmanifest 2026-05-18 00:16:06.151174000 -0700 -+++ support/mkmanifest 2026-05-18 00:16:11.128015000 -0700 -@@ -278,6 +278,8 @@ - tests/authprogs/ident.t - tests/clients/server-list - tests/docs/pod.t -+tests/expire/tombstone-hisexpire.t -+tests/expire/tombstone.t - tests/innd/artparse.t - tests/innd/chan.t - tests/lib/artnumber.t -@@ -335,4 +337,5 @@ - tests/overview/tradindexed.t - tests/overview/xref.t - tests/perl/minimum-version.t -+tests/storage/cancel-tombstone.t - tests/util/innbind.t -diff --git tests/Makefile tests/Makefile ---- tests/Makefile 2026-05-18 00:16:06.137090000 -0700 -+++ tests/Makefile 2026-05-18 00:18:06.443094000 -0700 -@@ -31,7 +31,9 @@ - lib/setenv.t lib/snprintf.t lib/strlcat.t \ - lib/strlcpy.t lib/tst.t lib/uwildmat.t lib/vector.t lib/wire.t \ - lib/xwrite.t nnrpd/auth-ext.t overview/api.t overview/buffindexed.t \ -- overview/ovsqlite.t overview/tradindexed.t overview/xref.t util/innbind.t -+ overview/ovsqlite.t overview/tradindexed.t overview/xref.t util/innbind.t \ -+ expire/tombstone.t expire/tombstone-hisexpire.t \ -+ storage/cancel-tombstone.t - - ## Extra stuff that needs to be built before tests can be run. - -@@ -322,6 +324,19 @@ - - overview/xref.t: overview/xref-t.o tap/basic.o $(STORAGEDEPS) - $(LINKDEPS) overview/xref-t.o tap/basic.o $(STORAGELIBS) $(LIBS) -+ -+expire/tombstone.t: expire/tombstone-t.o tap/basic.o $(STORAGEDEPS) -+ $(LINKDEPS) expire/tombstone-t.o tap/basic.o $(STORAGELIBS) $(LIBS) -+ -+expire/tombstone-hisexpire.t: expire/tombstone-hisexpire-t.o tap/basic.o \ -+ $(STORAGEDEPS) -+ $(LINKDEPS) expire/tombstone-hisexpire-t.o tap/basic.o \ -+ $(STORAGELIBS) $(LIBS) -+ -+storage/cancel-tombstone.t: storage/cancel-tombstone-t.o tap/basic.o \ -+ $(STORAGEDEPS) -+ $(LINKDEPS) storage/cancel-tombstone-t.o tap/basic.o \ -+ $(STORAGELIBS) $(LIBS) - - perl/minimum-version.t: perl/minimum-version.t.in $(FIXSCRIPT) - $(FIX) -i perl/minimum-version.t.in -diff --git tests/TESTS tests/TESTS ---- tests/TESTS 2026-05-18 00:16:06.136991000 -0700 -+++ tests/TESTS 2026-05-18 00:18:23.837362000 -0700 -@@ -54,6 +54,10 @@ - nnrpd/auth-ext - overview/api - overview/buffindexed -+expire/tombstone -+expire/tombstone-e2e -+expire/tombstone-hisexpire -+storage/cancel-tombstone - overview/ovsqlite - overview/ovsqlite-integ - overview/overchan -diff --git tests/expire/tombstone-e2e.t tests/expire/tombstone-e2e.t ---- tests/expire/tombstone-e2e.t 1969-12-31 17:00:00.000000000 -0700 -+++ tests/expire/tombstone-e2e.t 2026-05-18 00:16:11.128465000 -0700 -@@ -0,0 +1,174 @@ -+#! /bin/sh -+# -+# End-to-end test for the expire-tombstone feature. -+# -+# Exercises the integration between sm (writes to cancels.tombstone), -+# the tombstone library helpers (read/snapshot/lookup), and the -+# expiretombstone-related guards. Verifies: -+# - sm -r appends to ${pathdb}/cancels.tombstone when expiretombstone=true -+# - sm -r does NOT write when expiretombstone=false -+# - The written entries round-trip through tombstone_read into a -+# hashset that can answer membership queries -+# - The atomic-snapshot rename moves cancels.tombstone aside under -+# lock so a fresh file can capture writes that follow -+ -+count=1 -+printcount() { -+ echo "$1 $count $2" -+ count=$(expr $count + 1) -+} -+ -+# Find the right directory. -+sm="../../frontends/sm" -+dirs='../data data tests/data' -+for dir in $dirs; do -+ if [ -r "$dir/articles/1" ]; then -+ cd $dir -+ break -+ fi -+done -+if [ ! -x "$sm" ]; then -+ echo "Could not find sm" >&2 -+ exit 1 -+fi -+ -+# Use a fresh dedicated pathdb so we do not interfere with the other -+# storage tests (which use ./db) and so we can flip expiretombstone -+# without touching the shared etc/inn.conf. -+TMPDIR_E2E="tombstone-e2e.tmp" -+rm -rf "$TMPDIR_E2E" spool tradspool.map -+mkdir -p "$TMPDIR_E2E" spool -+ -+# Build a tweaked inn.conf that enables expiretombstone and points -+# pathdb at our temp dir. Other paths inherit defaults from the -+# tests/data tree. -+cat > "$TMPDIR_E2E/inn.conf" </dev/null 2>&1 -+lines=$(wc -l < "$CANCELS" | sed -e 's/[ \t]//g') -+if [ "$lines" = 2 ]; then -+ printcount "ok" -+else -+ printcount "not ok" "expected 2 lines after invalid-token rm, got $lines" -+fi -+ -+# 7. All entries are valid tokens (round-trip parseable). Use sm -i to -+# parse each line; sm -i succeeds only if the input is a valid token -+# that resolves to a known article. After cancellation the article -+# is gone, so sm -i exits non-zero with "Token not found", but it -+# does NOT exit with "Bad token" -- which is the format-validity -+# check we want. -+all_format_valid=true -+while read -r line; do -+ case "$line" in -+ '@'*'@') -+ # Format check: starts and ends with @ -+ ;; -+ *) -+ all_format_valid=false -+ ;; -+ esac -+done < "$CANCELS" -+if $all_format_valid; then -+ printcount "ok" -+else -+ printcount "not ok" "tombstone contains malformed lines" -+fi -+ -+# 8. flock contract: after sm finishes, the lock is released and another -+# writer (here, ourselves via sm) can acquire it. This is implicit -+# in tests 2, 4, etc., but we explicitly verify by storing+removing -+# a fourth article and confirming a clean append. -+token4=$($sm -s < articles/4) -+$sm -r "$token4" -+lines=$(wc -l < "$CANCELS" | sed -e 's/[ \t]//g') -+if [ "$lines" = 3 ] && grep -qF "$token4" "$CANCELS"; then -+ printcount "ok" -+else -+ printcount "not ok" "fourth cancel: expected 3 lines, got $lines" -+fi -+ -+# Cleanup. -+rm -rf "$TMPDIR_E2E" spool tradspool.map "$TMPDIR_E2E.bak" -+exit 0 -diff --git tests/expire/tombstone-hisexpire-t.c tests/expire/tombstone-hisexpire-t.c ---- tests/expire/tombstone-hisexpire-t.c 1969-12-31 17:00:00.000000000 -0700 -+++ tests/expire/tombstone-hisexpire-t.c 2026-05-18 00:16:11.128590000 -0700 -@@ -0,0 +1,298 @@ -+/* Integration test: HISexpire callback path consults the tombstone hashset. -+ * -+ * Builds a temporary history file with HISopen/HISwrite, populates a -+ * tombstone hashset for some of the tokens, runs HISexpire with a -+ * callback that mirrors EXPdoline's decision tree -- including the -+ * SELFEXPIRE branch (gap G). Verifies the resulting new history file -+ * contains exactly the tokens that should survive. -+ * -+ * The callback distinguishes four token categories: -+ * N_KEPT not tombstoned, not self-expiring -> keep -+ * N_TOMBSTONED in tombstone (fast path) -> drop -+ * N_SELFEXP_GONE self-expiring, simulated NOENT -> drop (slow path) -+ * N_SELFEXP_ALIVE self-expiring, simulated alive -> keep -+ * -+ * The slow path mirrors EXPdoline's behaviour for backends where -+ * SMprobe(SELFEXPIRE) is true: even with a tombstone hashset present, -+ * we still consult the simulated SMretrieve because articles can vanish -+ * via wrap-around without going through SMcancel. */ -+ -+#include "portable/system.h" -+ -+#include -+#include -+#include -+ -+#include "inn/hashtab.h" -+#include "inn/history.h" -+#include "inn/libinn.h" -+#include "inn/messages.h" -+#include "inn/storage.h" -+#include "inn/tombstone.h" -+#include "tap/basic.h" -+ -+ -+#define N_KEPT 5 -+#define N_TOMBSTONED 4 -+#define N_SELFEXP_GONE 3 -+#define N_SELFEXP_ALIVE 2 -+ -+#define N_TOTAL (N_KEPT + N_TOMBSTONED + N_SELFEXP_GONE + N_SELFEXP_ALIVE) -+ -+ -+/* Cookie threading two hashsets through HISexpire's callback: the -+ * tombstone (real EXPdoline checks this first) and a "gone" set used to -+ * simulate SMretrieve(RETR_STAT) == NULL for self-expiring backends. */ -+struct expire_cookie { -+ struct hash *tombstone; -+ struct hash *gone; -+}; -+ -+ -+/* Mirror of EXPdoline's decision tree for tokens that may be tombstoned -+ * or self-expiring. Returns true to keep, false to drop. -+ * -+ * Tokens are tagged via their type byte: -+ * type=1 - non-self-expiring backend -+ * type=2 - self-expiring backend (CNFS-like) -+ */ -+static bool -+test_expire_cb(void *cookie, time_t arrived UNUSED, time_t posted UNUSED, -+ time_t expires UNUSED, TOKEN *token) -+{ -+ struct expire_cookie *c = cookie; -+ -+ /* Fast path: tombstone hit -> drop, no further checks. */ -+ if (c->tombstone != NULL && tombstone_present(c->tombstone, token)) -+ return false; -+ -+ /* Self-expiring backend (gap G). Even with a tombstone in hand, we -+ * cannot trust "not in tombstone == alive" because wrap-around can -+ * silently delete articles. Simulate the SMretrieve(RETR_STAT) -+ * check via the `gone` hashset. */ -+ if (token->type == 2) { -+ if (tombstone_present(c->gone, token)) -+ return false; -+ return true; -+ } -+ -+ /* Non-self-expiring: trust the tombstone. Not in tombstone means -+ * still alive. */ -+ return true; -+} -+ -+ -+/* Build a synthetic token whose bytes encode n and whose type byte -+ * tags whether it is from a self-expiring backend. */ -+static TOKEN -+make_token(unsigned char type, unsigned long n) -+{ -+ TOKEN t; -+ -+ memset(&t, 0, sizeof(t)); -+ t.type = type; -+ t.class = 0; -+ t.token[0] = (n >> 24) & 0xff; -+ t.token[1] = (n >> 16) & 0xff; -+ t.token[2] = (n >> 8) & 0xff; -+ t.token[3] = n & 0xff; -+ return t; -+} -+ -+ -+static char * -+make_msgid(unsigned long n) -+{ -+ char buf[64]; -+ snprintf(buf, sizeof(buf), "", n); -+ return xstrdup(buf); -+} -+ -+ -+/* Count the number of non-empty lines in the history file (one per -+ * surviving entry). */ -+static unsigned long -+count_history_lines(const char *path) -+{ -+ FILE *f; -+ char line[SMBUF]; -+ unsigned long n = 0; -+ -+ f = fopen(path, "r"); -+ if (f == NULL) -+ return 0; -+ while (fgets(line, sizeof(line), f) != NULL) { -+ if (line[0] != '\n' && line[0] != '\0') -+ n++; -+ } -+ fclose(f); -+ return n; -+} -+ -+ -+/* Helper: insert a TOKEN copy into a hashset. */ -+static void -+hash_add_token(struct hash *h, const TOKEN *token) -+{ -+ TOKEN *p = xmalloc(sizeof(TOKEN)); -+ *p = *token; -+ if (!hash_insert(h, p, p)) -+ free(p); -+} -+ -+ -+int -+main(void) -+{ -+ struct history *h; -+ struct expire_cookie cookie; -+ char tmpdir[64]; -+ char histpath[128]; -+ char newhistpath[140]; -+ TOKEN tokens[N_TOTAL]; -+ unsigned long i; -+ unsigned long base; -+ bool expire_ok; -+ struct stat sb; -+ -+ test_init(9); -+ -+ strlcpy(tmpdir, "tombstone-hisexp-XXXXXX", sizeof(tmpdir)); -+ if (mkdtemp(tmpdir) == NULL) -+ sysbail("can't create temp directory"); -+ snprintf(histpath, sizeof(histpath), "%s/history", tmpdir); -+ -+ /* Layout of the tokens array: -+ [0 .. N_KEPT) non-selfexpire keep -+ [N_KEPT .. N_KEPT+N_TOMBSTONED) non-selfexpire tomb -+ [N_KEPT+N_TOMBSTONED .. + N_SELFEXP_GONE) selfexpire gone -+ [..rest] selfexpire alive -+ */ -+ base = 0; -+ for (i = 0; i < N_KEPT; i++) -+ tokens[base + i] = make_token(1, base + i); -+ base += N_KEPT; -+ for (i = 0; i < N_TOMBSTONED; i++) -+ tokens[base + i] = make_token(1, base + i); -+ base += N_TOMBSTONED; -+ for (i = 0; i < N_SELFEXP_GONE; i++) -+ tokens[base + i] = make_token(2, base + i); -+ base += N_SELFEXP_GONE; -+ for (i = 0; i < N_SELFEXP_ALIVE; i++) -+ tokens[base + i] = make_token(2, base + i); -+ -+ /* Populate a fresh history database with all N_TOTAL entries. */ -+ h = HISopen(histpath, "hisv6", HIS_CREAT | HIS_RDWR); -+ if (h == NULL) -+ bail("can't create history at %s", histpath); -+ for (i = 0; i < N_TOTAL; i++) { -+ char *msgid = make_msgid(i); -+ if (!HISwrite(h, msgid, (time_t) 1000000 + i, (time_t) 1000000 + i, -+ (time_t) 0, &tokens[i])) -+ bail("can't write history entry %lu: %s", i, HISerror(h)); -+ free(msgid); -+ } -+ HISsync(h); -+ HISclose(h); -+ ok(1, stat(histpath, &sb) == 0 && sb.st_size > 0); -+ -+ /* Build the tombstone hashset (only the N_TOMBSTONED slice). */ -+ cookie.tombstone = tombstone_hash_create(8); -+ for (i = N_KEPT; i < N_KEPT + N_TOMBSTONED; i++) -+ hash_add_token(cookie.tombstone, &tokens[i]); -+ ok(2, hash_count(cookie.tombstone) == N_TOMBSTONED); -+ -+ /* Build the "gone" hashset for self-expiring backends (only the -+ * N_SELFEXP_GONE slice; selfexpire-alive tokens are NOT added). */ -+ cookie.gone = tombstone_hash_create(8); -+ base = N_KEPT + N_TOMBSTONED; -+ for (i = 0; i < N_SELFEXP_GONE; i++) -+ hash_add_token(cookie.gone, &tokens[base + i]); -+ ok(3, hash_count(cookie.gone) == N_SELFEXP_GONE); -+ -+ /* Reopen read-only and run HISexpire with our SELFEXPIRE-aware -+ * callback. */ -+ h = HISopen(histpath, "hisv6", HIS_RDONLY); -+ if (h == NULL) -+ bail("can't reopen history at %s", histpath); -+ snprintf(newhistpath, sizeof(newhistpath), "%s/history.new", tmpdir); -+ /* High threshold so dropped entries do not survive as remember-only -+ * records; we want a clean count of survivors. */ -+ expire_ok = HISexpire(h, newhistpath, NULL, true, &cookie, -+ (time_t) 2000000000, test_expire_cb); -+ ok(4, expire_ok); -+ HISclose(h); -+ -+ /* The new history at .n should contain exactly the -+ * survivors: N_KEPT + N_SELFEXP_ALIVE. Tombstoned and selfexpire- -+ * gone entries should be absent. */ -+ { -+ char actual[160]; -+ unsigned long expected = N_KEPT + N_SELFEXP_ALIVE; -+ snprintf(actual, sizeof(actual), "%s.n", newhistpath); -+ ok(5, count_history_lines(actual) == expected); -+ } -+ -+ /* Per-category lookup checks: kept and selfexpire-alive present; -+ * tombstoned and selfexpire-gone absent. */ -+ { -+ char actual[160]; -+ struct history *hnew; -+ unsigned long kept_present = 0; -+ unsigned long tomb_present = 0; -+ unsigned long alive_present = 0; -+ unsigned long gone_present = 0; -+ TOKEN found; -+ -+ snprintf(actual, sizeof(actual), "%s.n", newhistpath); -+ hnew = HISopen(actual, "hisv6", HIS_RDONLY); -+ if (hnew == NULL) -+ bail("can't open new history %s", actual); -+ -+ for (i = 0; i < N_KEPT; i++) { -+ char *msgid = make_msgid(i); -+ if (HISlookup(hnew, msgid, NULL, NULL, NULL, &found)) -+ kept_present++; -+ free(msgid); -+ } -+ for (i = N_KEPT; i < N_KEPT + N_TOMBSTONED; i++) { -+ char *msgid = make_msgid(i); -+ if (HISlookup(hnew, msgid, NULL, NULL, NULL, &found)) -+ tomb_present++; -+ free(msgid); -+ } -+ base = N_KEPT + N_TOMBSTONED; -+ for (i = 0; i < N_SELFEXP_GONE; i++) { -+ char *msgid = make_msgid(base + i); -+ if (HISlookup(hnew, msgid, NULL, NULL, NULL, &found)) -+ gone_present++; -+ free(msgid); -+ } -+ base += N_SELFEXP_GONE; -+ for (i = 0; i < N_SELFEXP_ALIVE; i++) { -+ char *msgid = make_msgid(base + i); -+ if (HISlookup(hnew, msgid, NULL, NULL, NULL, &found)) -+ alive_present++; -+ free(msgid); -+ } -+ -+ ok(6, kept_present == N_KEPT); -+ ok(7, tomb_present == 0); -+ ok(8, gone_present == 0); /* SELFEXPIRE branch correctly drops */ -+ ok(9, alive_present == N_SELFEXP_ALIVE); -+ HISclose(hnew); -+ } -+ -+ hash_free(cookie.tombstone); -+ hash_free(cookie.gone); -+ -+ /* Cleanup. */ -+ { -+ char cmd[128]; -+ snprintf(cmd, sizeof(cmd), "/bin/rm -rf %s", tmpdir); -+ if (system(cmd) < 0) -+ sysdiag("can't clean up %s", tmpdir); -+ } -+ -+ return 0; -+} -diff --git tests/expire/tombstone-t.c tests/expire/tombstone-t.c ---- tests/expire/tombstone-t.c 1969-12-31 17:00:00.000000000 -0700 -+++ tests/expire/tombstone-t.c 2026-05-18 00:16:11.128733000 -0700 -@@ -0,0 +1,316 @@ -+/* Unit tests for the lib/tombstone helpers used by expire to consume -+ * the deletion logs written by expireover/expirerm and innd/sm. */ -+ -+#include "portable/system.h" -+ -+#include -+#include -+#include -+#include -+#include -+ -+#include "inn/hashtab.h" -+#include "inn/libinn.h" -+#include "inn/messages.h" -+#include "inn/storage.h" -+#include "inn/tombstone.h" -+#include "tap/basic.h" -+ -+ -+static TOKEN -+make_token(unsigned char type, unsigned char class, unsigned long n) -+{ -+ TOKEN t; -+ -+ memset(&t, 0, sizeof(t)); -+ t.type = type; -+ t.class = class; -+ t.token[0] = (n >> 24) & 0xff; -+ t.token[1] = (n >> 16) & 0xff; -+ t.token[2] = (n >> 8) & 0xff; -+ t.token[3] = n & 0xff; -+ return t; -+} -+ -+ -+/* Write a tombstone-format file containing the given tokens and any -+ * extra raw lines. Returns the path; caller must unlink and free. */ -+static char * -+write_tombstone_file(const char *prefix, const TOKEN *tokens, size_t ntokens, -+ const char *const *extra_lines, size_t nextra) -+{ -+ char tmpl[64]; -+ int fd; -+ FILE *f; -+ size_t i; -+ -+ snprintf(tmpl, sizeof(tmpl), "%s-XXXXXX", prefix); -+ fd = mkstemp(tmpl); -+ if (fd < 0) -+ sysbail("can't create %s", tmpl); -+ f = fdopen(fd, "w"); -+ if (f == NULL) -+ sysbail("can't fdopen %s", tmpl); -+ for (i = 0; i < ntokens; i++) -+ fprintf(f, "%s\n", TokenToText(tokens[i])); -+ for (i = 0; i < nextra; i++) -+ fprintf(f, "%s", extra_lines[i]); -+ fclose(f); -+ return xstrdup(tmpl); -+} -+ -+ -+int -+main(void) -+{ -+ TOKEN t1, t2, t3, t4, t_other; -+ char *text; -+ char *path; -+ char *path2; -+ struct hash *h; -+ unsigned long count; -+ TOKEN parsed; -+ char *roundtrip; -+ -+ test_init(25); -+ -+ /* TokenToText / TextToToken / IsToken round trip is the foundation -+ * of the file format. */ -+ t1 = make_token(3, 0, 0xdeadbeef); -+ text = xstrdup(TokenToText(t1)); -+ parsed = TextToToken(text); -+ ok(1, IsToken(text)); -+ ok(2, memcmp(&parsed, &t1, sizeof(TOKEN)) == 0); -+ roundtrip = xstrdup(TokenToText(parsed)); -+ ok(3, strcmp(text, roundtrip) == 0); -+ free(text); -+ free(roundtrip); -+ -+ /* tombstone_hash_create gives us a hash that handles TOKEN keys. */ -+ t2 = make_token(3, 0, 1); -+ t3 = make_token(3, 0, 2); -+ t_other = make_token(3, 0, 99); -+ h = tombstone_hash_create(4); -+ { -+ TOKEN *p = xmalloc(sizeof(TOKEN)); -+ *p = t2; -+ ok(4, hash_insert(h, p, p)); -+ } -+ { -+ TOKEN *p = xmalloc(sizeof(TOKEN)); -+ *p = t3; -+ ok(5, hash_insert(h, p, p)); -+ } -+ ok(6, tombstone_present(h, &t2)); -+ ok(7, tombstone_present(h, &t3)); -+ ok(8, !tombstone_present(h, &t_other)); -+ ok(9, hash_count(h) == 2); -+ hash_free(h); -+ -+ /* tombstone_read parses a single file with mixed valid/invalid -+ * lines, skips malformed entries, and silently ignores comment -+ * lines (used for the format version header and for future -+ * metadata). This locks in the writer/reader contract for -+ * "# inn-tombstone v1\n" written by expireover. */ -+ { -+ TOKEN tokens[] = {t1, t2, t3, t1}; /* duplicate t1 */ -+ const char *extras[] = { -+ "\n", /* blank */ -+ "# inn-tombstone v1\n", /* version header */ -+ "not-a-token\n", /* malformed */ -+ "# arbitrary metadata\n", /* future-format placeholder */ -+ "\r\n", /* CRLF blank */ -+ }; -+ path = write_tombstone_file("tombstone-read", tokens, 4, extras, 5); -+ h = tombstone_hash_create(4); -+ count = tombstone_read(h, path, NULL); -+ /* Three unique tokens (t1 dedup'd) plus one valid duplicate -+ * line counted but not inserted -> count returned is 4 -+ * (lines processed) but hash has 3 unique. Comment and -+ * blank lines are skipped silently and not counted. */ -+ ok(10, h != NULL); -+ ok(11, count == 4); -+ ok(12, hash_count(h) == 3); -+ ok(13, tombstone_present(h, &t1)); -+ ok(14, tombstone_present(h, &t2)); -+ ok(15, tombstone_present(h, &t3)); -+ ok(16, !tombstone_present(h, &t_other)); -+ hash_free(h); -+ unlink(path); -+ free(path); -+ } -+ -+ /* Two-file merge with overlap -- T-C4-B. expireover.tombstone has -+ * t1, t2; cancels.tombstone has t2, t3. Loaded into one hashset, -+ * we expect 3 unique entries with t2 merged. */ -+ t4 = make_token(3, 0, 4); -+ { -+ TOKEN expireover_tokens[] = {t1, t2}; -+ TOKEN cancels_tokens[] = {t2, t3, t4}; -+ path = write_tombstone_file("tombstone-expireover", -+ expireover_tokens, 2, NULL, 0); -+ path2 = write_tombstone_file("tombstone-cancels", -+ cancels_tokens, 3, NULL, 0); -+ h = tombstone_hash_create(4); -+ tombstone_read(h, path, NULL); -+ tombstone_read(h, path2, NULL); -+ ok(17, hash_count(h) == 4); /* t1, t2, t3, t4 */ -+ ok(18, tombstone_present(h, &t1) && tombstone_present(h, &t2) -+ && tombstone_present(h, &t3) -+ && tombstone_present(h, &t4)); -+ hash_free(h); -+ unlink(path); -+ unlink(path2); -+ free(path); -+ free(path2); -+ } -+ -+ /* tombstone_rename_for_processing -- T-C4-C. Atomically renames -+ * the source file to ".processing"; subsequent open of the -+ * original path finds nothing. */ -+ { -+ TOKEN tokens[] = {t1, t2}; -+ char *snapshot; -+ struct stat sb; -+ -+ path = write_tombstone_file("tombstone-rename", tokens, 2, NULL, 0); -+ snapshot = tombstone_rename_for_processing(path); -+ ok(19, snapshot != NULL); -+ if (snapshot != NULL) { -+ /* Original path no longer exists; snapshot does. */ -+ ok(20, stat(path, &sb) < 0 && errno == ENOENT -+ && stat(snapshot, &sb) == 0); -+ unlink(snapshot); -+ free(snapshot); -+ } else { -+ ok(20, false); -+ } -+ unlink(path); /* in case rename failed */ -+ free(path); -+ } -+ -+ /* tombstone_ensure_header on an absent file: creates a header-only -+ * file under exclusive lock. Read back as a hashset that holds -+ * no tokens (the header is a comment line, skipped by the -+ * reader). */ -+ { -+ char tmpl[] = "tombstone-ensure-XXXXXX"; -+ int tmp_fd; -+ struct stat sb; -+ -+ tmp_fd = mkstemp(tmpl); -+ if (tmp_fd < 0) -+ sysbail("can't create temp file"); -+ close(tmp_fd); -+ unlink(tmpl); /* mkstemp creates it; we want absent */ -+ path = xstrdup(tmpl); -+ tombstone_ensure_header(path); -+ h = tombstone_hash_create(4); -+ count = tombstone_read(h, path, NULL); -+ ok(21, stat(path, &sb) == 0 && sb.st_size > 0 && count == 0 -+ && hash_count(h) == 0); -+ hash_free(h); -+ unlink(path); -+ free(path); -+ } -+ -+ /* tombstone_ensure_header on a non-empty file (appender raced a -+ * cancel in between consumer's unlink and our recreate): the -+ * existing tokens are preserved verbatim below the new header. */ -+ { -+ TOKEN raced[] = {t1, t2}; -+ -+ path = write_tombstone_file("tombstone-prepend", raced, 2, NULL, 0); -+ tombstone_ensure_header(path); -+ h = tombstone_hash_create(4); -+ count = tombstone_read(h, path, NULL); -+ ok(22, count == 2 && hash_count(h) == 2 -+ && tombstone_present(h, &t1) && tombstone_present(h, &t2)); -+ hash_free(h); -+ unlink(path); -+ free(path); -+ } -+ -+ /* tombstone_ensure_header is idempotent: called twice in a row, -+ * the second call is a no-op because the file already starts -+ * with the header. Without this, every expire cycle would -+ * prepend an extra header line and grow the file 19 bytes per -+ * cycle on a site with zero cancels per cycle. */ -+ { -+ char tmpl[] = "tombstone-idem-XXXXXX"; -+ int tmp_fd; -+ struct stat sb1, sb2; -+ -+ tmp_fd = mkstemp(tmpl); -+ if (tmp_fd < 0) -+ sysbail("can't create temp file"); -+ close(tmp_fd); -+ unlink(tmpl); -+ path = xstrdup(tmpl); -+ tombstone_ensure_header(path); -+ stat(path, &sb1); -+ tombstone_ensure_header(path); -+ stat(path, &sb2); -+ ok(23, sb1.st_size == sb2.st_size && sb1.st_size > 0); -+ unlink(path); -+ free(path); -+ } -+ -+ /* Multi-chunk shift: build a file larger than the internal -+ * chunk size (64 KiB), call tombstone_ensure_header, and verify -+ * the shift loop's chunk-boundary arithmetic is correct. At -+ * ~38 bytes per token line, 3000 tokens is ~108 KiB, which -+ * forces the loop to run at least twice (one full CHUNK + one -+ * partial trailing chunk). */ -+ { -+ size_t nbig = 3000; -+ TOKEN *big = xmalloc(nbig * sizeof(TOKEN)); -+ size_t i; -+ bool all_present; -+ -+ for (i = 0; i < nbig; i++) -+ big[i] = make_token(3, 0, 0x10000 + i); -+ path = write_tombstone_file("tombstone-multichunk", big, nbig, -+ NULL, 0); -+ tombstone_ensure_header(path); -+ h = tombstone_hash_create(nbig * 2); -+ count = tombstone_read(h, path, NULL); -+ all_present = (count == nbig && hash_count(h) == nbig); -+ for (i = 0; i < nbig && all_present; i++) -+ all_present = tombstone_present(h, &big[i]); -+ ok(24, all_present); -+ hash_free(h); -+ unlink(path); -+ free(path); -+ free(big); -+ } -+ -+ /* Crash-recovery tolerance: simulate a file in a post-crash -+ * state (no header, some garbled lines interleaved with valid -+ * tokens). ensure_header should shift everything right by -+ * header_len without crashing; the subsequent tombstone_read -+ * should warn-and-skip garbled lines and parse the valid -+ * tokens. This pins down the recovery argument as actual -+ * behavior. */ -+ { -+ TOKEN valid[] = {t1, t2, t3}; -+ const char *garbled[] = { -+ "this-is-not-a-token\n", -+ "neither-is-this\n", -+ }; -+ -+ path = write_tombstone_file("tombstone-corrupt", valid, 3, -+ garbled, 2); -+ tombstone_ensure_header(path); -+ h = tombstone_hash_create(4); -+ count = tombstone_read(h, path, NULL); -+ ok(25, hash_count(h) == 3 && tombstone_present(h, &t1) -+ && tombstone_present(h, &t2) -+ && tombstone_present(h, &t3)); -+ hash_free(h); -+ unlink(path); -+ free(path); -+ } -+ -+ return 0; -+} -diff --git tests/storage/cancel-tombstone-t.c tests/storage/cancel-tombstone-t.c ---- tests/storage/cancel-tombstone-t.c 1969-12-31 17:00:00.000000000 -0700 -+++ tests/storage/cancel-tombstone-t.c 2026-05-18 00:16:11.128870000 -0700 -@@ -0,0 +1,233 @@ -+/* Unit tests for SMcanceltombstone (storage/interface.c). -+ * -+ * Verifies the multi-writer append protocol used by innd's ARTcancel -+ * and sm's -r path: the file format is one TokenToText() per line; the -+ * write is gated on innconf->expiretombstone; TOKEN_EMPTY is a no-op; -+ * concurrent appenders use fcntl POSIX locks via inn_lock_file. */ -+ -+#include "portable/system.h" -+ -+#include -+#include -+#include -+#include -+#include -+ -+#include "inn/innconf.h" -+#include "inn/libinn.h" -+#include "inn/messages.h" -+#include "inn/storage.h" -+#include "tap/basic.h" -+ -+ -+/* Build a synthetic token whose bytes encode n. Type/class chosen so -+ * IsToken accepts the textual form (the type byte must be a valid hex -+ * digit pair when round-tripped). */ -+static TOKEN -+make_token(unsigned char type, unsigned long n) -+{ -+ TOKEN t; -+ -+ memset(&t, 0, sizeof(t)); -+ t.type = type; -+ t.class = 0; -+ t.token[0] = (n >> 24) & 0xff; -+ t.token[1] = (n >> 16) & 0xff; -+ t.token[2] = (n >> 8) & 0xff; -+ t.token[3] = n & 0xff; -+ return t; -+} -+ -+ -+/* Read the entire file into a malloc'd buffer. Returns NULL on -+ * error. *out_size set to bytes read. */ -+static char * -+slurp(const char *path, size_t *out_size) -+{ -+ FILE *f; -+ struct stat sb; -+ char *buf; -+ -+ *out_size = 0; -+ if (stat(path, &sb) < 0) -+ return NULL; -+ f = fopen(path, "r"); -+ if (f == NULL) -+ return NULL; -+ buf = xmalloc(sb.st_size + 1); -+ if (fread(buf, 1, sb.st_size, f) != (size_t) sb.st_size) { -+ free(buf); -+ fclose(f); -+ return NULL; -+ } -+ buf[sb.st_size] = '\0'; -+ *out_size = sb.st_size; -+ fclose(f); -+ return buf; -+} -+ -+ -+/* Count lines in buf (number of '\n' bytes). */ -+static size_t -+count_lines(const char *buf, size_t len) -+{ -+ size_t i, n = 0; -+ for (i = 0; i < len; i++) -+ if (buf[i] == '\n') -+ n++; -+ return n; -+} -+ -+ -+int -+main(void) -+{ -+ char tmpdir[64]; -+ char *cancels_path; -+ TOKEN t1, t2, t3, empty; -+ char *contents; -+ size_t size; -+ struct stat sb; -+ -+ test_init(15); -+ -+ /* Set up a temporary pathdb. */ -+ strlcpy(tmpdir, "cancel-tombstone-XXXXXX", sizeof(tmpdir)); -+ if (mkdtemp(tmpdir) == NULL) -+ sysbail("can't create temp directory"); -+ -+ /* Initialize a minimal innconf so SMcanceltombstone has somewhere -+ * to write. We need the fields it consults: expiretombstone, -+ * groupbaseexpiry (both required), and pathdb. Other code paths -+ * in this binary read additional fields, so allocate via xcalloc -+ * to zero everything. */ -+ innconf = xcalloc(1, sizeof(*innconf)); -+ innconf->pathdb = xstrdup(tmpdir); -+ innconf->expiretombstone = true; -+ innconf->groupbaseexpiry = true; -+ -+ cancels_path = concatpath(tmpdir, "cancels.tombstone"); -+ -+ t1 = make_token(3, 0xdeadbeef); -+ t2 = make_token(3, 0xcafebabe); -+ t3 = make_token(3, 0x12345678); -+ memset(&empty, 0, sizeof(empty)); -+ empty.type = TOKEN_EMPTY; -+ -+ /* 1. First call creates the file with one line. */ -+ SMcanceltombstone(t1); -+ contents = slurp(cancels_path, &size); -+ ok(1, contents != NULL); -+ ok(2, count_lines(contents, size) == 1); -+ ok(3, contents != NULL && strstr(contents, TokenToText(t1)) != NULL); -+ free(contents); -+ -+ /* 2. Second call appends; file now has two distinct token lines. */ -+ SMcanceltombstone(t2); -+ contents = slurp(cancels_path, &size); -+ ok(4, contents != NULL && count_lines(contents, size) == 2); -+ ok(5, contents != NULL && strstr(contents, TokenToText(t1)) != NULL -+ && strstr(contents, TokenToText(t2)) != NULL); -+ free(contents); -+ -+ /* 3. expiretombstone=false: third call is a no-op. */ -+ innconf->expiretombstone = false; -+ SMcanceltombstone(t3); -+ contents = slurp(cancels_path, &size); -+ ok(6, contents != NULL && count_lines(contents, size) == 2); -+ ok(7, contents != NULL && strstr(contents, TokenToText(t3)) == NULL); -+ free(contents); -+ innconf->expiretombstone = true; -+ -+ /* 4. TOKEN_EMPTY: no-op (not a real article). */ -+ SMcanceltombstone(empty); -+ contents = slurp(cancels_path, &size); -+ ok(8, contents != NULL && count_lines(contents, size) == 2); -+ free(contents); -+ -+ /* 5. After write completes, the lock has been released; another -+ * process (here, just us reusing the same path) can acquire the -+ * exclusive write lock without contention. */ -+ { -+ int fd = open(cancels_path, O_RDWR); -+ ok(9, fd >= 0); -+ if (fd >= 0) { -+ ok(10, inn_lock_file(fd, INN_LOCK_WRITE, false)); -+ close(fd); -+ } else { -+ ok(10, false); -+ } -+ } -+ -+ /* 6. Each line round-trips: parsing each line through TextToToken -+ * yields the original TOKEN. Read the file and verify. */ -+ contents = slurp(cancels_path, &size); -+ if (contents != NULL) { -+ char *p = contents; -+ char *line_end; -+ TOKEN parsed; -+ bool found_t1 = false; -+ bool found_t2 = false; -+ -+ while ((line_end = strchr(p, '\n')) != NULL) { -+ *line_end = '\0'; -+ if (IsToken(p)) { -+ parsed = TextToToken(p); -+ if (memcmp(&parsed, &t1, sizeof(TOKEN)) == 0) -+ found_t1 = true; -+ if (memcmp(&parsed, &t2, sizeof(TOKEN)) == 0) -+ found_t2 = true; -+ } -+ p = line_end + 1; -+ } -+ ok(11, found_t1); -+ ok(12, found_t2); -+ free(contents); -+ } else { -+ ok(11, false); -+ ok(12, false); -+ } -+ -+ /* 7. File mode should be 0664 (created with O_CREAT, mode 0664). -+ * Check the stored mode bits. Allow umask-stripped variants. */ -+ if (stat(cancels_path, &sb) == 0) { -+ mode_t mode = sb.st_mode & 0777; -+ /* The file was created with mode 0664 but umask may have -+ * stripped write bits. Verify at least owner+group readable. */ -+ ok(13, (mode & 0640) == 0640); -+ } else { -+ ok(13, false); -+ } -+ -+ /* 8. Calling SMcanceltombstone with an unwritable pathdb does -+ * not crash (best-effort failure path), does not create any -+ * file outside the configured location, and returns false. -+ * Simulate by pointing pathdb at a non-existent directory. */ -+ { -+ char *saved_pathdb = innconf->pathdb; -+ char *bad_path; -+ struct stat sb_bad; -+ bool result; -+ innconf->pathdb = xstrdup("/nonexistent-tombstone-test-dir"); -+ result = SMcanceltombstone(t3); -+ bad_path = concatpath(innconf->pathdb, "cancels.tombstone"); -+ ok(14, !result && stat(bad_path, &sb_bad) < 0 -+ && errno == ENOENT); -+ free(bad_path); -+ free(innconf->pathdb); -+ innconf->pathdb = saved_pathdb; -+ } -+ -+ /* Cleanup. */ -+ if (unlink(cancels_path) < 0 && errno != ENOENT) -+ sysdiag("can't unlink %s", cancels_path); -+ free(cancels_path); -+ if (rmdir(tmpdir) < 0) -+ sysdiag("can't rmdir %s", tmpdir); -+ /* Verify cleanup actually removed the directory. */ -+ ok(15, stat(tmpdir, &sb) < 0 && errno == ENOENT); -+ free(innconf->pathdb); -+ free(innconf); -+ -+ return 0; -+} diff --git a/news/inn-current/pkg-plist b/news/inn-current/pkg-plist index 74391c16f80f..25f1ab2c9385 100644 --- a/news/inn-current/pkg-plist +++ b/news/inn-current/pkg-plist @@ -1,413 +1,414 @@ @owner news @group news @dir %%NEWSBASE%% %%CWDNEWSBASE%% bin/actmerge bin/actsync bin/actsyncd bin/archive @(root,,4550) bin/auth/passwd/ckpasswd %%KERBEROS%%bin/auth/passwd/auth_krb5 bin/auth/passwd/radius bin/auth/resolv/domain bin/auth/resolv/ident bin/batcher bin/buffchan bin/buffindexed_d bin/cnfsheadconf bin/cnfsstat bin/control/checkgroups.pl bin/control/ihave.pl bin/control/newgroup.pl bin/control/rmgroup.pl bin/control/sendme.pl bin/controlbatch bin/controlchan bin/convdate bin/ctlinnd bin/cvtbatch bin/delayer bin/docheckgroups bin/expire bin/expireover bin/expirerm bin/fastrm bin/filter/INN.py bin/filter/nnrpd.py @sample %%NEWSBASE%%/bin/filter/filter_innd.pl.sample @sample %%NEWSBASE%%/bin/filter/filter_innd.py.sample @sample %%NEWSBASE%%/bin/filter/filter_nnrpd.pl.sample @sample %%NEWSBASE%%/bin/filter/nnrpd_access.pl.sample @sample %%NEWSBASE%%/bin/filter/nnrpd_access.py.sample @sample %%NEWSBASE%%/bin/filter/nnrpd_auth.pl.sample @sample %%NEWSBASE%%/bin/filter/nnrpd_auth.py.sample @sample %%NEWSBASE%%/bin/filter/nnrpd_dynamic.py.sample @sample %%NEWSBASE%%/bin/filter/startup_innd.pl.sample bin/gencancel bin/getlist bin/grephistory bin/imapfeed @(,news,%%MODE_INEWS%%) bin/inews bin/inncheck bin/innconfval @(root,,4550) bin/innbind bin/innd bin/inndf bin/innfeed bin/innmail bin/innreport bin/innstat bin/innupgrade bin/innwatch bin/innxbatch bin/innxmit bin/mailpost bin/makedbz bin/makehistory bin/mod-active bin/news.daily bin/news2mail bin/ninpaths bin/nnrpd bin/nntpget bin/nntpsend bin/ovdb_init bin/ovdb_monitor bin/ovdb_server bin/ovdb_stat bin/overchan %%SQLITE%%bin/ovsqlite-server %%SQLITE%%bin/ovsqlite-util bin/perl-nocem bin/pgpverify bin/procbatch bin/prunehistory bin/pullnews bin/rc.news %%UUCP_RNEWS%%@(,uucp,4550) bin/rnews %%NO_UUCP_RNEWS%%bin/rnews bin/rnews.libexec/c7unbatch bin/rnews.libexec/decode bin/rnews.libexec/encode bin/rnews.libexec/bunbatch bin/rnews.libexec/gunbatch bin/scanlogs bin/scanspool bin/send-ihave bin/send-uucp bin/sendinpaths bin/sendxbatches bin/shlock bin/shrinkfile bin/simpleftp bin/sm bin/tally.control bin/tinyleaf bin/tdx-util bin/writelog doc/CONTRIBUTORS doc/GPL doc/HACKING doc/INSTALL doc/LICENSE doc/NEWS doc/README %%SNAPSHOT%%doc/README.snapshot doc/TODO doc/config-design doc/config-semantics doc/config-syntax doc/external-auth doc/history doc/history-innfeed doc/hook-perl doc/hook-python doc/sample-control http/innreport.css +include/inn/bloom.h include/inn/buffer.h include/inn/concat.h include/inn/confparse.h -%%EXPERIMENTAL%%include/inn/bloom.h include/inn/dbz.h include/inn/dispatch.h include/inn/fdflag.h include/inn/hashtab.h include/inn/history.h include/inn/innconf.h include/inn/inndcomm.h include/inn/libinn.h include/inn/list.h include/inn/macros.h include/inn/md5.h include/inn/messages.h include/inn/mmap.h include/inn/network-innbind.h include/inn/network.h include/inn/newsuser.h include/inn/nntp.h include/inn/options.h include/inn/ov.h include/inn/overview.h include/inn/paths.h include/inn/portable-getaddrinfo.h include/inn/portable-getnameinfo.h include/inn/portable-macros.h include/inn/portable-socket.h include/inn/portable-stdbool.h include/inn/qio.h include/inn/secrets.h include/inn/sequence.h include/inn/storage.h include/inn/system.h include/inn/timer.h +include/inn/tombstone.h include/inn/tst.h include/inn/utility.h include/inn/vector.h include/inn/version.h include/inn/wire.h include/inn/xmalloc.h include/inn/xwrite.h lib/perl/INN/Config.pm lib/perl/INN/ovsqlite_client.pm lib/perl/INN/Utils/Shlock.pm lib/innreport_inn.pm lib/innreport-display.conf lib/innshellvars lib/innshellvars.pl lib/innshellvars.tcl lib/libinn.a lib/libinn.so lib/libinn.so.%%INNLIB%% lib/libinn.so.%%INNLIB_LONG%% lib/libinnhist.a lib/libinnhist.so lib/libinnhist.so.%%LIBVER%% lib/libinnhist.so.%%LIBVER_LONG%% lib/libinnstorage.a lib/libinnstorage.so lib/libinnstorage.so.%%LIBSTORAGEVER%% lib/libinnstorage.so.%%LIBSTORAGEVER_LONG%% @comment -------------------------------------------------------------- @cwd share/man/man1/convdate.1.gz share/man/man1/fastrm.1.gz share/man/man1/gencancel.1.gz share/man/man1/getlist.1.gz share/man/man1/grephistory.1.gz share/man/man1/inews.1.gz share/man/man1/innconfval.1.gz share/man/man1/innmail.1.gz share/man/man1/nntpget.1.gz share/man/man1/pgpverify.1.gz share/man/man1/pullnews.1.gz share/man/man1/rnews.1.gz share/man/man1/shrinkfile.1.gz share/man/man1/simpleftp.1.gz share/man/man3/INN::Config.3pm.gz share/man/man3/INN::Utils::Shlock.3pm.gz share/man/man3/INN::ovsqlite_client.3pm.gz share/man/man3/libinn_clientlib.3.gz share/man/man3/libinn_dbz.3.gz share/man/man3/libinn_inndcomm.3.gz share/man/man3/libinn_list.3.gz share/man/man3/libinn_qio.3.gz share/man/man3/libinn_tst.3.gz share/man/man3/libinn_uwildmat.3.gz share/man/man3/libinnauth.3.gz share/man/man3/libinnstorage.3.gz share/man/man3/libinn.3.gz share/man/man3/libinnhist.3.gz share/man/man5/active.5.gz share/man/man5/active.times.5.gz share/man/man5/buffindexed.conf.5.gz share/man/man5/control.ctl.5.gz share/man/man5/cycbuff.conf.5.gz share/man/man5/distrib.pats.5.gz share/man/man5/distributions.5.gz share/man/man5/expire.ctl.5.gz share/man/man5/history.5.gz share/man/man5/incoming.conf.5.gz share/man/man5/inn-radius.conf.5.gz share/man/man5/inn-secrets.conf.5.gz share/man/man5/inn.conf.5.gz share/man/man5/innfeed.conf.5.gz share/man/man5/innreport.conf.5.gz share/man/man5/innwatch.ctl.5.gz share/man/man5/localgroups.5.gz share/man/man5/moderators.5.gz share/man/man5/motd.innd.5.gz share/man/man5/motd.news.5.gz share/man/man5/motd.nnrpd.5.gz share/man/man5/newsfeeds.5.gz share/man/man5/newsgroups.5.gz share/man/man5/newslog.5.gz share/man/man5/nnrpd.track.5.gz share/man/man5/nntpsend.ctl.5.gz share/man/man5/nocem.ctl.5.gz share/man/man5/ovdb.5.gz %%SQLITE%%share/man/man5/ovsqlite.5.gz share/man/man5/passwd.nntp.5.gz share/man/man5/readers.conf.5.gz share/man/man5/storage.conf.5.gz share/man/man5/subscriptions.5.gz share/man/man8/actsync.8.gz share/man/man8/actsyncd.8.gz share/man/man8/archive.8.gz %%KERBEROS%%share/man/man8/auth_krb5.8.gz share/man/man8/batcher.8.gz share/man/man8/buffchan.8.gz share/man/man8/ckpasswd.8.gz share/man/man8/cnfsheadconf.8.gz share/man/man8/cnfsstat.8.gz share/man/man8/controlchan.8.gz share/man/man8/ctlinnd.8.gz share/man/man8/cvtbatch.8.gz share/man/man8/delayer.8.gz share/man/man8/docheckgroups.8.gz share/man/man8/domain.8.gz share/man/man8/expire.8.gz share/man/man8/expireover.8.gz share/man/man8/expirerm.8.gz share/man/man8/ident.8.gz share/man/man8/imapfeed.8.gz share/man/man8/innbind.8.gz share/man/man8/inncheck.8.gz share/man/man8/innd.8.gz share/man/man8/inndf.8.gz share/man/man8/innfeed.8.gz share/man/man8/innreport.8.gz share/man/man8/innstat.8.gz share/man/man8/innupgrade.8.gz share/man/man8/innwatch.8.gz share/man/man8/innxbatch.8.gz share/man/man8/innxmit.8.gz share/man/man8/inpaths.8.gz share/man/man8/mailpost.8.gz share/man/man8/makedbz.8.gz share/man/man8/makehistory.8.gz share/man/man8/mod-active.8.gz share/man/man8/news.daily.8.gz share/man/man8/news2mail.8.gz share/man/man8/ninpaths.8.gz share/man/man8/nnrpd.8.gz share/man/man8/nntpsend.8.gz share/man/man8/ovdb_init.8.gz share/man/man8/ovdb_monitor.8.gz share/man/man8/ovdb_server.8.gz share/man/man8/ovdb_stat.8.gz %%SQLITE%%share/man/man8/ovsqlite-server.8.gz %%SQLITE%%share/man/man8/ovsqlite-util.8.gz share/man/man8/overchan.8.gz share/man/man8/perl-nocem.8.gz share/man/man8/procbatch.8.gz share/man/man8/prunehistory.8.gz share/man/man8/radius.8.gz share/man/man8/rc.news.8.gz share/man/man8/scanlogs.8.gz share/man/man8/scanspool.8.gz share/man/man8/send-ihave.8.gz share/man/man8/send-uucp.8.gz share/man/man8/sendinpaths.8.gz share/man/man8/sendxbatches.8.gz share/man/man8/shlock.8.gz share/man/man8/sm.8.gz share/man/man8/tally.control.8.gz share/man/man8/tdx-util.8.gz share/man/man8/tinyleaf.8.gz share/man/man8/writelog.8.gz @comment -------------------------------------------------------------- %%EXAMPLESDIR%%/actsync.cfg %%EXAMPLESDIR%%/actsync.ign %%EXAMPLESDIR%%/buffindexed.conf %%EXAMPLESDIR%%/control.ctl %%EXAMPLESDIR%%/control.ctl.local %%EXAMPLESDIR%%/cycbuff.conf %%EXAMPLESDIR%%/distrib.pats %%EXAMPLESDIR%%/distributions %%EXAMPLESDIR%%/expire.ctl %%EXAMPLESDIR%%/incoming.conf %%EXAMPLESDIR%%/inn-radius.conf %%EXAMPLESDIR%%/inn-secrets.conf %%EXAMPLESDIR%%/inn.conf %%EXAMPLESDIR%%/innfeed.conf %%EXAMPLESDIR%%/innreport.conf %%EXAMPLESDIR%%/innshellvars.local %%EXAMPLESDIR%%/innshellvars.pl.local %%EXAMPLESDIR%%/innshellvars.tcl.local %%EXAMPLESDIR%%/innwatch.ctl %%EXAMPLESDIR%%/localgroups %%EXAMPLESDIR%%/moderators %%EXAMPLESDIR%%/motd.innd.sample %%EXAMPLESDIR%%/motd.nnrpd.sample %%EXAMPLESDIR%%/news2mail.cf %%EXAMPLESDIR%%/newsfeeds %%EXAMPLESDIR%%/nnrpd.track %%EXAMPLESDIR%%/nntpsend.ctl %%EXAMPLESDIR%%/nocem.ctl %%EXAMPLESDIR%%/ovdb.conf %%SQLITE%%%%EXAMPLESDIR%%/ovsqlite.conf %%EXAMPLESDIR%%/passwd.nntp %%EXAMPLESDIR%%/readers.conf %%EXAMPLESDIR%%/send-uucp.cf %%EXAMPLESDIR%%/storage.conf %%EXAMPLESDIR%%/subscriptions @dir %%EXAMPLESDIR%%/db @comment -------------------------------------------------------------- @sample %%DBDIR%%/active.sample @sample %%DBDIR%%/active.times.sample @sample %%DBDIR%%/history.sample @sample %%DBDIR%%/newsgroups.sample @comment -------------------------------------------------------------- @sample %%DBDIR%%/history.dir.sample %%WITHOUT_TAGGED_HASH%%@sample %%DBDIR%%/history.hash.sample %%WITHOUT_TAGGED_HASH%%@sample %%DBDIR%%/history.index.sample %%WITH_TAGGED_HASH%%@sample %%DBDIR%%/history.pag.sample @sample %%CONFDIR%%/actsync.cfg.sample @sample %%CONFDIR%%/actsync.ign.sample @sample %%CONFDIR%%/buffindexed.conf.sample @sample %%CONFDIR%%/control.ctl.local.sample @sample %%CONFDIR%%/control.ctl.sample @sample %%CONFDIR%%/cycbuff.conf.sample @sample %%CONFDIR%%/distrib.pats.sample @sample %%CONFDIR%%/distributions.sample @sample %%CONFDIR%%/expire.ctl.sample @sample %%CONFDIR%%/incoming.conf.sample @sample %%CONFDIR%%/inn-radius.conf.sample @sample %%CONFDIR%%/inn-secrets.conf.sample @sample %%CONFDIR%%/inn.conf.sample @sample %%CONFDIR%%/innfeed.conf.sample @sample %%CONFDIR%%/innreport.conf.sample @sample %%CONFDIR%%/innshellvars.local.sample @sample %%CONFDIR%%/innshellvars.pl.local.sample @sample %%CONFDIR%%/innshellvars.tcl.local.sample @sample %%CONFDIR%%/innwatch.ctl.sample @sample %%CONFDIR%%/localgroups.sample @sample %%CONFDIR%%/moderators.sample @sample %%CONFDIR%%/news2mail.cf.sample @sample %%CONFDIR%%/newsfeeds.sample @sample %%CONFDIR%%/nnrpd.track.sample @sample %%CONFDIR%%/nntpsend.ctl.sample @sample %%CONFDIR%%/nocem.ctl.sample @sample %%CONFDIR%%/ovdb.conf.sample %%SQLITE%%@sample %%CONFDIR%%/ovsqlite.conf.sample @sample %%CONFDIR%%/passwd.nntp.sample @sample %%CONFDIR%%/readers.conf.sample @sample %%CONFDIR%%/send-uucp.cf.sample @sample %%CONFDIR%%/storage.conf.sample @sample %%CONFDIR%%/subscriptions.sample @comment -------------------------------------------------------------- @cwd / @dir %%TMPDIR%% @dir %%SHAREDIR%% @dir %%NEWSSPOOL%%/overview @dir %%NEWSSPOOL%%/outgoing @dir %%NEWSSPOOL%%/innfeed @dir %%NEWSSPOOL%%/incoming/bad @dir %%NEWSSPOOL%%/incoming @dir %%NEWSSPOOL%%/articles @dir %%NEWSSPOOL%%/archive @dir %%NEWSSPOOL%% @dir %%RUNDIR%% @dir %%DBDIR%% @dir %%LOGDIR%%/OLD @dir %%LOGDIR%% diff --git a/news/inn/Makefile b/news/inn/Makefile index e54a57cf5e58..bf73d128880b 100644 --- a/news/inn/Makefile +++ b/news/inn/Makefile @@ -1,246 +1,232 @@ PORTNAME?= inn PORTVERSION?= 2.7.3 PORTREVISION?= 3 CATEGORIES= news MASTER_SITES= ISC MASTER_SITE_SUBDIR?= ${PORTNAME} MAINTAINER= kbowling@FreeBSD.org COMMENT= InterNetNews -- the Internet meets Netnews WWW= https://www.eyrie.org/~eagle/software/inn/ LICENSE= GPLv2 ISCL LICENSE_COMB= multi LICENSE_FILE_ISCL= ${WRKSRC}/LICENSE BUILD_DEPENDS= p5-GD>=0:graphics/p5-GD \ p5-MIME-Tools>=0:mail/p5-MIME-Tools RUN_DEPENDS:= ${BUILD_DEPENDS} USES= cpe gmake libtool perl5 uidfix CPE_VENDOR= isc USE_RC_SUBR= innd nnrpd .ifnmake portclippy VARBASE?= /var .endif .include "Makefile.layout" GNU_CONFIGURE= yes GNU_CONFIGURE_PREFIX= ${INN_NEWSBASE} # Point to base system mailwrapper to avoid configure failure in jails CONFIGURE_ARGS+= --datarootdir=${INN_SHAREDIR} \ --sysconfdir=${INN_CONFDIR} \ --with-db-dir=${INN_DBDIR} \ --with-log-dir=${INN_LOGDIR} \ --with-perl \ --with-run-dir=${INN_RUNDIR} \ --with-sendmail=/usr/sbin/sendmail \ --with-spool-dir=${INN_NEWSSPOOL} \ --with-tmp-dir=${INN_TMPDIR} CONFIGURE_ENV+= LOCALBASE=${LOCALBASE} CONFIGURE_ENV+= ac_cv_prog_AWK="${AWK}" CONFLICTS?= inn-current SUB_FILES= pkg-install SUB_LIST+= EGDIR="${EXAMPLESDIR:C,^${PREFIX},\\$\\${PREFIX},}" INNLIB_LONG?= 9.0.2 LIBVER_LONG?= 3.0.9 LIBSTORAGEVER_LONG?= 3.1.3 CONFIG_FILES+= actsync.cfg actsync.ign buffindexed.conf control.ctl \ control.ctl.local cycbuff.conf distrib.pats distributions \ expire.ctl inn-radius.conf inn-secrets.conf incoming.conf \ inn.conf innfeed.conf innreport.conf innshellvars.local \ innshellvars.pl.local innshellvars.tcl.local innwatch.ctl \ localgroups moderators news2mail.cf newsfeeds nocem.ctl \ nnrpd.track nntpsend.ctl ovdb.conf passwd.nntp readers.conf \ send-uucp.cf storage.conf subscriptions .if ${BRANCH:U} == 2.8 PLIST_SUB+= SNAPSHOT="" .else PLIST_SUB+= SNAPSHOT="@comment " .endif PLIST_SUB+= ETCFILES="${CONFIG_FILES}" \ INNLIB=${INNLIB_LONG:R:R} \ INNLIB_LONG=${INNLIB_LONG} \ LIBSTORAGEVER=${LIBSTORAGEVER_LONG:R:R} \ LIBSTORAGEVER_LONG=${LIBSTORAGEVER_LONG} \ LIBVER=${LIBVER_LONG:R:R} \ LIBVER_LONG=${LIBVER_LONG} PORTDOCS= CONTRIBUTORS HACKING INSTALL LICENSE MANIFEST NEWS README TODO OPTIONS_DEFINE= BERKELEYDB BLOCKLISTD CANLOCK DOCS EXAMPLES GNUPG \ KERBEROS KEYWORDS LARGE_FILES OPENSSL PYTHON SASL \ SETGID_INEWS SQLITE TAGGED_HASH UUCP_RNEWS OPTIONS_DEFAULT= BERKELEYDB BLOCKLISTD CANLOCK GNUPG KEYWORDS \ LARGE_FILES OPENSSL PYTHON SETGID_INEWS SQLITE \ UUCP_RNEWS -.if ${BRANCH:U} == 2.8 -OPTIONS_DEFINE+= EXPERIMENTAL -.endif OPTIONS_SUB= yes BERKELEYDB_DESC= Enable BerkeleyDB (for ovdb overview method) BLOCKLISTD_DESC= Enable blocklistd support CANLOCK_DESC= Enable Cancel-Lock support via libcanlock -EXPERIMENTAL_DESC= Experimental upstream patches (ovsqlite, bloom filter) GNUPG_DESC= GnuPG support (for pgpverify control message) KERBEROS_DESC= Enable Kerberos v5 (for auth_krb5) KEYWORDS_DESC= Automatic keyword generation support LARGE_FILES_DESC= Support for files larger than 2GB OPENSSL_DESC= Enable OpenSSL (for NNTP over TLS/SSL support) PYTHON_DESC= Enable support for embedded Python SASL_DESC= Enable SASL (for imapfeed authentication) SETGID_INEWS_DESC= Enable posting articles locally for all users SQLITE_DESC= Enable SQLite (for ovsqlite overview method) TAGGED_HASH_DESC= Use tagged hash table for history UUCP_RNEWS_DESC= Install rnews suitable for use with net/freebsd-uucp BERKELEYDB_USES= bdb BERKELEYDB_CONFIGURE_ON= --with-bdb-include=${BDB_INCLUDE_DIR} \ --with-bdb-lib=${LOCALBASE}/lib \ --with-bdb=${LOCALBASE} .if ${BRANCH:U} == 2.8 && ${OSVERSION:U} >= 1500000 BLOCKLISTD_CONFIGURE_WITH= blocklist .else BLOCKLISTD_CONFIGURE_WITH= blacklist .endif CANLOCK_LIB_DEPENDS= libcanlock.so:news/libcanlock CANLOCK_CONFIGURE_ON= --with-canlock=${LOCALBASE} CANLOCK_CONFIGURE_OFF= --without-canlock # Although pgpverify needs gpgv/gpgv2, # checking 'gpg' is the shortest way to check any version of GnuPG installed. GNUPG_BUILD_DEPENDS= gpg:security/gnupg GNUPG_RUN_DEPENDS= gpg:security/gnupg KERBEROS_USES= gssapi KERBEROS_CONFIGURE_ON= --with-krb5=${GSSAPIBASEDIR} KERBEROS_CONFIGURE_OFF= --without-krb5 KEYWORDS_CONFIGURE_ENABLE= keywords LARGE_FILES_PREVENTS= TAGGED_HASH LARGE_FILES_PREVENTS_MSG= cannot be compiled with both LARGE_FILES and \ TAGGED_HASH LARGE_FILES_CONFIGURE_ENABLE= largefiles OPENSSL_USES= ssl OPENSSL_CONFIGURE_ON= --with-openssl=${OPENSSLBASE} PYTHON_USES= gettext-runtime python PYTHON_CONFIGURE_ON= --with-python SASL_LIB_DEPENDS= libsasl2.so:security/cyrus-sasl2 SASL_CONFIGURE_ON= --with-sasl=${LOCALBASE} SETGID_INEWS_CONFIGURE_ENABLE= setgid-inews SETGID_INEWS_PLIST_SUB= MODE_INEWS="2555" SETGID_INEWS_PLIST_SUB_OFF= MODE_INEWS="" SQLITE_USES= sqlite SQLITE_CONFIGURE_ON= --with-sqlite3=${LOCALBASE} SQLITE_CONFIGURE_OFF= --without-sqlite3 SQLITE_VARS= CONFIG_FILES+=ovsqlite.conf TAGGED_HASH_CONFIGURE_ENABLE= tagged-hash TAGGED_HASH_SUB_LIST= DBZ_FILES="dir pag" TAGGED_HASH_SUB_LIST_OFF= DBZ_FILES="dir hash index" TAGGED_HASH_PLIST_SUB= WITH_TAGGED_HASH="" \ WITHOUT_TAGGED_HASH="@comment " TAGGED_HASH_PLIST_SUB_OFF= WITH_TAGGED_HASH="@comment " \ WITHOUT_TAGGED_HASH="" UUCP_RNEWS_RUN_DEPENDS= uucp:net/freebsd-uucp UUCP_RNEWS_CONFIGURE_ENABLE= uucp-rnews -EXPERIMENTAL_EXTRA_PATCHES= ${PKGDIR}/files/pr337.patch \ - ${PKGDIR}/files/pr338.patch \ - ${PKGDIR}/files/pr339.patch \ - ${PKGDIR}/files/pr340.patch -EXPERIMENTAL_PLIST_SUB= EXPERIMENTAL="" -EXPERIMENTAL_PLIST_SUB_OFF= EXPERIMENTAL="@comment " - .include GID?= ${.MAKE.GID} -post-extract-EXPERIMENTAL-on: - @${CP} ${PKGDIR}/files/gitignore ${WRKSRC}/.gitignore - post-patch: @${REINPLACE_CMD} -E 's!\$$[{(]PATHETC[})]!${EXAMPLESDIR}!g' \ ${WRKSRC}/site/Makefile .if ${PORT_OPTIONS:MBERKELEYDB} @${REINPLACE_CMD} -e 's!-ldb!-ldb-${BDB_VER}!' ${WRKSRC}/configure .endif .if ${BRANCH:U} == 2.8 . if ${OPSYS} == FreeBSD && ${OSVERSION} < 1500000 @${FIND} ${WRKSRC} -type f -exec ${SED} -i '' \ -e 's/BLOCKLIST/BLACKLIST/g' \ -e 's/Blocklist/Blacklist/g' \ -e 's/blocklist/blacklist/g' {} + . endif .else . if ${OPSYS} == FreeBSD && ${OSVERSION} >= 1500000 @${FIND} ${WRKSRC} -type f -exec ${SED} -i '' \ -e 's/BLACKLIST/BLOCKLIST/g' \ -e 's/Blacklist/Blocklist/g' \ -e 's/blacklist/blocklist/g' {} + . endif .endif pre-install: @${MKDIR} ${STAGEDIR}${INN_DBDIR} \ ${STAGEDIR}${EXAMPLESDIR} \ ${STAGEDIR}${EXAMPLESDIR}/db @${REINPLACE_CMD} -e '/^RUNASUSER *=/s/=.*/= ${UID}/' \ -e '/^RUNASGROUP *=/s/=.*/= ${GID}/' \ -e '/^RNEWSGROUP *=/s/=.*/= ${GID}/' \ ${WRKSRC}/Makefile.global post-build: @${FIND} ${WRKSRC} -name "inn.conf" \ | ${XARGS} ${REINPLACE_CMD} -e 's/^pathhost:.*/pathhost: host.example.com/' post-install: ${FIND} ${STAGEDIR}${INN_NEWSBASE} -type f \ \( -perm +111 -o -name '*.so.*' \) ! -name '*.a' \ -exec ${STRIP_CMD} {} + 2>/dev/null || ${TRUE} ${MKDIR} ${STAGEDIR}${INN_NEWSSPOOL} ${MKDIR} ${STAGEDIR}${INN_CONFDIR} .for FILE in ${CONFIG_FILES} ${CP} -p ${STAGEDIR}${EXAMPLESDIR}/${FILE} ${STAGEDIR}${INN_CONFDIR}/${FILE}.sample .endfor ${MKDIR} ${STAGEDIR}${INN_DBDIR} .for FILE in active active.times history newsgroups ${MV} ${STAGEDIR}${INN_DBDIR}/${FILE} \ ${STAGEDIR}${INN_DBDIR}/${FILE}.sample .endfor ${TOUCH} ${STAGEDIR}${INN_DBDIR}/history.dir.sample .if ${PORT_OPTIONS:MTAGGED_HASH} ${TOUCH} ${STAGEDIR}${INN_DBDIR}/history.pag.sample .else ${TOUCH} ${STAGEDIR}${INN_DBDIR}/history.hash.sample ${TOUCH} ${STAGEDIR}${INN_DBDIR}/history.index.sample .endif .for FILE in filter_innd.pl filter_innd.py filter_nnrpd.pl nnrpd_access.pl \ nnrpd_access.py nnrpd_auth.pl nnrpd_auth.py nnrpd_dynamic.py \ startup_innd.pl ${MV} ${STAGEDIR}${INN_NEWSBASE}/bin/filter/${FILE} \ ${STAGEDIR}${INN_NEWSBASE}/bin/filter/${FILE}.sample .endfor ${MKDIR} ${STAGEDIR}${DOCSDIR} .for FILE in ${PORTDOCS} ${INSTALL_DATA} ${WRKSRC}/${FILE} ${STAGEDIR}${DOCSDIR} .endfor .include