diff --git a/news/inn-current/Makefile b/news/inn-current/Makefile index c0abc5ce284d..b8ee301f465b 100644 --- a/news/inn-current/Makefile +++ b/news/inn-current/Makefile @@ -1,22 +1,22 @@ PORTNAME= inn-current PORTVERSION= ${SNAPSHOT} -PORTREVISION= 2 +PORTREVISION= 3 MASTER_SITE_SUBDIR= inn/snapshots DISTNAME= inn-${BRANCH}-${SNAPSHOT} BRANCH= 2.8 SNAPSHOT= 20260419 CONFLICTS= inn-2.[0-9]* CPE_PRODUCT= inn INNLIB_LONG= 9.0.2 LIBVER_LONG= 3.0.9 LIBSTORAGEVER_LONG= 3.1.3 MASTERDIR= ${.CURDIR}/../inn PKGDIR= ${.CURDIR} DISTINFO_FILE= ${.CURDIR}/distinfo .include "${MASTERDIR}/Makefile" diff --git a/news/inn-current/distinfo b/news/inn-current/distinfo index 13db3d268ac6..6750ff7add57 100644 --- a/news/inn-current/distinfo +++ b/news/inn-current/distinfo @@ -1,21 +1,3 @@ TIMESTAMP = 1778145188 SHA256 (inn-2.8-20260419.tar.gz) = 6c5ac86d213445f656d50483e68adfeaf2a9cf35bece5c90acfc7405a0d7df4d SIZE (inn-2.8-20260419.tar.gz) = 2835406 -SHA256 (cf4a8ecac2f3149505f9a55cdd95383e9b3f725b.patch) = 2f5191a6f69cbe7e700da982239f52174abf0813a33751246fe067ed4bf4b571 -SIZE (cf4a8ecac2f3149505f9a55cdd95383e9b3f725b.patch) = 5366 -SHA256 (0d84255b417c89219225f0a034fd4c068d9749a0.patch) = 1b9c03c1155d5d82ce181c23d68c901e48f3de35ff78a7a3b1dc1e764b276ad0 -SIZE (0d84255b417c89219225f0a034fd4c068d9749a0.patch) = 77142 -SHA256 (05d70a17c5720be25fcd957c8ab7a0c12a64d30b.patch) = baa2eec82eccd40122f4aeb498ba4faac93d6914684ac6e647d553ae797ebf00 -SIZE (05d70a17c5720be25fcd957c8ab7a0c12a64d30b.patch) = 1426 -SHA256 (0b308fdd3f6255e8c81a3deaf1bad08cb1fe8f1d.patch) = bd5418b29e9720555cd5ff90df90b4503e657d72485be6390ea8459d7e2678ce -SIZE (0b308fdd3f6255e8c81a3deaf1bad08cb1fe8f1d.patch) = 7120 -SHA256 (755ab163eb124658959a8e4168732e6a7f7dc300.patch) = e794c5e23de3dd6c21a150a06115fcc8b0112d4dc36345f6ac0579473dd371bd -SIZE (755ab163eb124658959a8e4168732e6a7f7dc300.patch) = 2877 -SHA256 (d23b3f84aebe3fb49af0b42b7cfaa7a7ebae5a26.patch) = aeccadb2eb6f77f138aee3c1e87175f9599dca16c0e18d6f37ba50271262707f -SIZE (d23b3f84aebe3fb49af0b42b7cfaa7a7ebae5a26.patch) = 1265 -SHA256 (919cf653b54c72ef4bf1563f3fb6bbdf9a193a3a.patch) = 2e4e18591aad21acdef02441b99463b069c9f56884800e8a3fd1fbe44566aaf1 -SIZE (919cf653b54c72ef4bf1563f3fb6bbdf9a193a3a.patch) = 1731 -SHA256 (93a36682f7ba9d013d07859882c2b8c501da249e.patch) = 1bbb3b8213754779a8c1947e8a065bff6089066e290c4b52b93bb2b8d678fe21 -SIZE (93a36682f7ba9d013d07859882c2b8c501da249e.patch) = 3577 -SHA256 (d0c379ee3f325f4a80dada6c3129c23496b1c83e.patch) = 33ab2f245e198c5999739959d17d3a16ba7c1086b4671568bdf52374780c0db9 -SIZE (d0c379ee3f325f4a80dada6c3129c23496b1c83e.patch) = 43317 diff --git a/news/inn-current/files/pr337.patch b/news/inn-current/files/pr337.patch new file mode 100644 index 000000000000..19d84e1d7897 --- /dev/null +++ b/news/inn-current/files/pr337.patch @@ -0,0 +1,87 @@ +From 93a36682f7ba9d013d07859882c2b8c501da249e Mon Sep 17 00:00:00 2001 +From: Kevin Bowling +Date: Wed, 29 Apr 2026 13:54:21 -0700 +Subject: [PATCH] ovsqlite: add mmapsize configuration + +--- + doc/pod/ovsqlite.pod | 11 +++++++++++ + samples/ovsqlite.conf | 7 +++++++ + storage/ovsqlite/ovsqlite-server.c | 11 +++++++++++ + 3 files changed, 29 insertions(+) + +diff --git doc/pod/ovsqlite.pod doc/pod/ovsqlite.pod +index 96a6b3165..3090d5bcb 100644 +--- doc/pod/ovsqlite.pod ++++ doc/pod/ovsqlite.pod +@@ -67,6 +67,17 @@ is consulted only when creating a new database. Enabling compression + saves about S<55 %> of disk space on standard overview data. The default + value is false. + ++=item I ++ ++The maximum number of bytes of the database file that SQLite will ++memory-map. When set, SQLite reads data directly from the operating ++system's page cache via mmap instead of copying it into its own page ++cache, which can significantly improve read performance for large ++databases. A good starting value is the size of the database file ++or the amount of available memory, whichever is smaller. For example, ++a value of C<4294967296> allows up to S<4 GB> to be memory-mapped. ++The default value of 0 disables memory-mapped I/O. ++ + =item I + + The SQLite database page size in bytes. Must be a power of 2, minimum 512, +diff --git samples/ovsqlite.conf samples/ovsqlite.conf +index cba4f7277..a2e2e204f 100644 +--- samples/ovsqlite.conf ++++ samples/ovsqlite.conf +@@ -10,6 +10,13 @@ + # The default value is false. + #compress: false + ++# The maximum number of bytes of the database file that SQLite will ++# memory-map. When set, SQLite reads data via mmap instead of copying ++# into its own page cache, which can improve read performance for large ++# databases. A good value is the database file size or available memory, ++# whichever is smaller. Set to 0 (the default) to disable. ++#mmapsize: 0 ++ + # The SQLite database page size in bytes. + # Must be a power of 2, minimum 512, maximum 65536. + # Appropriate values include the virtual memory page size and the +diff --git storage/ovsqlite/ovsqlite-server.c storage/ovsqlite/ovsqlite-server.c +index f04df77e7..afc20e46b 100644 +--- storage/ovsqlite/ovsqlite-server.c ++++ storage/ovsqlite/ovsqlite-server.c +@@ -119,6 +119,7 @@ static sql_main_t sql_main; + static bool use_compression; + static unsigned long pagesize; + static unsigned long cachesize; ++static unsigned long mmapsize; + static struct timeval transaction_time_limit = {10, 0}; + static unsigned long transaction_row_limit = 10000; + +@@ -386,6 +387,7 @@ load_config(void) + config_param_boolean(top, "compress", &use_compression); + config_param_unsigned_number(top, "pagesize", &pagesize); + config_param_unsigned_number(top, "cachesize", &cachesize); ++ config_param_unsigned_number(top, "mmapsize", &mmapsize); + if (config_param_real(top, "transtimelimit", &timelimit)) { + transaction_time_limit.tv_sec = (long) timelimit; + transaction_time_limit.tv_usec = +@@ -656,6 +658,15 @@ open_db(void) + sqlite3_free(errmsg); + } + } ++ if (mmapsize) { ++ snprintf(sqltext, sizeof sqltext, "pragma mmap_size = %lu;", ++ mmapsize); ++ status = sqlite3_exec(connection, sqltext, 0, NULL, &errmsg); ++ if (status != SQLITE_OK) { ++ warn("cannot set mmap size: %s", errmsg); ++ sqlite3_free(errmsg); ++ } ++ } + } + + static void diff --git a/news/inn-current/files/pr338.patch b/news/inn-current/files/pr338.patch new file mode 100644 index 000000000000..5ff4a03632e4 --- /dev/null +++ b/news/inn-current/files/pr338.patch @@ -0,0 +1,2883 @@ +From cf4a8ecac2f3149505f9a55cdd95383e9b3f725b Mon Sep 17 00:00:00 2001 +From: Kevin Bowling +Date: Wed, 29 Apr 2026 13:47:42 -0700 +Subject: [PATCH 1/7] ovsqlite: Add WAL mode configuration + +--- + doc/pod/ovsqlite.pod | 21 ++++++++++++++++++++- + samples/ovsqlite.conf | 10 ++++++++++ + storage/ovsqlite/ovsqlite-server.c | 21 +++++++++++++++++++-- + storage/ovsqlite/sql-main.sql | 3 +++ + 4 files changed, 52 insertions(+), 3 deletions(-) + +diff --git doc/pod/ovsqlite.pod doc/pod/ovsqlite.pod +index 96a6b3165..797e2c6a8 100644 +--- doc/pod/ovsqlite.pod ++++ doc/pod/ovsqlite.pod +@@ -39,7 +39,8 @@ Plan on needing at least S<0.93 KB> for every article in your spool + need at least S<4.65 GB> of disk space for ovsqlite. With compression + enabled, this estimate changes to S<0.4 KB> per article, so you'll + need at least S<2 GB> of disk space for 5 million articles. Plus, +-you'll need additional space for transaction logs (a few MB). ++you'll need additional space for transaction logs (a few MB, or up to ++a few hundred MB temporarily with I enabled). + + =head1 CONFIGURATION + +@@ -67,6 +68,24 @@ is consulted only when creating a new database. Enabling compression + saves about S<55 %> of disk space on standard overview data. The default + value is false. + ++=item I ++ ++If this parameter is true, ovsqlite will use SQLite's WAL (Write-Ahead ++Logging) journal mode instead of the default PERSIST mode. WAL mode ++allows concurrent readers and writers without blocking, which significantly ++improves read performance when multiple B processes access the ++overview database simultaneously. It also sets C to ++C, which is safe with WAL and reduces unnecessary fsync overhead. ++ ++On shutdown, the WAL is checkpointed back into the main database file ++and truncated. ++ ++B WAL mode requires shared-memory support and does not work on ++network filesystems such as NFS. Do not enable this parameter if ++I is on a network filesystem. ++ ++The default value is false. ++ + =item I + + The SQLite database page size in bytes. Must be a power of 2, minimum 512, +diff --git samples/ovsqlite.conf samples/ovsqlite.conf +index cba4f7277..6912e8bf2 100644 +--- samples/ovsqlite.conf ++++ samples/ovsqlite.conf +@@ -10,6 +10,16 @@ + # The default value is false. + #compress: false + ++# WAL mode: if true, use SQLite's Write-Ahead Logging journal mode ++# instead of the default PERSIST mode. WAL mode allows concurrent ++# readers and writers without blocking, improving read performance ++# when multiple nnrpd processes access the overview database ++# simultaneously. Also sets synchronous to NORMAL, which is safe ++# with WAL and reduces unnecessary fsync overhead. ++# Note: WAL mode does not work on network filesystems such as NFS. ++# The default value is false. ++#walmode: false ++ + # The SQLite database page size in bytes. + # Must be a power of 2, minimum 512, maximum 65536. + # Appropriate values include the virtual memory page size and the +diff --git storage/ovsqlite/ovsqlite-server.c storage/ovsqlite/ovsqlite-server.c +index f04df77e7..5bc4aa8d9 100644 +--- storage/ovsqlite/ovsqlite-server.c ++++ storage/ovsqlite/ovsqlite-server.c +@@ -117,6 +117,7 @@ static sqlite3 *connection; + static sql_main_t sql_main; + + static bool use_compression; ++static bool use_wal; + static unsigned long pagesize; + static unsigned long cachesize; + static struct timeval transaction_time_limit = {10, 0}; +@@ -384,6 +385,7 @@ load_config(void) + free(path); + if (top) { + config_param_boolean(top, "compress", &use_compression); ++ config_param_boolean(top, "walmode", &use_wal); + config_param_unsigned_number(top, "pagesize", &pagesize); + config_param_unsigned_number(top, "cachesize", &cachesize); + if (config_param_real(top, "transtimelimit", &timelimit)) { +@@ -656,13 +658,28 @@ open_db(void) + sqlite3_free(errmsg); + } + } ++ if (use_wal) { ++ status = sqlite3_exec(connection, ++ "pragma journal_mode = 'WAL';" ++ "pragma synchronous = 'NORMAL';", ++ 0, NULL, &errmsg); ++ if (status != SQLITE_OK) { ++ warn("cannot enable WAL mode: %s", errmsg); ++ sqlite3_free(errmsg); ++ } ++ } + } + + static void + close_db(void) + { +- sqlite3_step(sql_main.delete_journal); +- sqlite3_reset(sql_main.delete_journal); ++ if (use_wal) { ++ sqlite3_step(sql_main.checkpoint_wal); ++ sqlite3_reset(sql_main.checkpoint_wal); ++ } else { ++ sqlite3_step(sql_main.delete_journal); ++ sqlite3_reset(sql_main.delete_journal); ++ } + sqlite_helper_term(&sql_main_helper, (sqlite3_stmt **) &sql_main); + sqlite3_close_v2(connection); + connection = NULL; +diff --git storage/ovsqlite/sql-main.sql storage/ovsqlite/sql-main.sql +index ff73f1b0e..ab5495a49 100644 +--- storage/ovsqlite/sql-main.sql ++++ storage/ovsqlite/sql-main.sql +@@ -40,6 +40,9 @@ rollback to savepoint article_group; + -- .delete_journal + pragma journal_mode = 'DELETE'; + ++-- .checkpoint_wal ++pragma wal_checkpoint(TRUNCATE); ++ + -- .add_group + insert into groupinfo (groupname, flag_alias, low, high) + values(?1, ?2, ?3, ?4); + +From 0d84255b417c89219225f0a034fd4c068d9749a0 Mon Sep 17 00:00:00 2001 +From: Kevin Bowling +Date: Sat, 2 May 2026 03:36:19 -0700 +Subject: [PATCH 2/7] ovsqlite: Add direct reader mode for nnrpd with WAL + +When WAL mode is enabled, nnrpd processes now open the overview database +directly with SQLITE_OPEN_READONLY instead of routing reads through +ovsqlite-server. This eliminates the IPC round-trip and the +single-threaded server bottleneck for read operations, allowing read +performance to scale with the number of concurrent readers. + +Direct reader mode activates automatically when walmode is true in +ovsqlite.conf AND the database is actually in WAL mode (verified via +pragma journal_mode). Without WAL, behavior is unchanged -- readers +connect to ovsqlite-server as before to avoid EXCLUSIVE lock contention. + +Key changes: +- ovsqlite.c: direct_open() opens the database read-only with schema + version validation, journal mode verification, decompression support, + and goto-based cleanup. Read functions (groupstats, getartinfo, + search) branch on direct_reader flag. Write operations return errors + in direct reader mode. Falls back to server path gracefully. +- ovsqlite-server.c: WAL checkpoint at shutdown uses PASSIVE first + (non-blocking, safe with readers connected) then TRUNCATE only if no + readers hold pins. Prevents server shutdown from hanging. +- ovsqlite.c: Fix pre-existing buffer leak in ovsqlite_close(): + request and response buffers from server_connect() were never freed. +- sql-read.sql: read-only prepared statements with busy_timeout=10000 + and query_only=1. +- New readercachesize config parameter (default 8 MB per reader). + +Includes unit tests (WAL gating, write rejection) and integration tests +that start a real ovsqlite-server, write compressed data through it, +kill the server, and verify the direct reader can read it back correctly +including a server restart cycle. +--- + .gitignore | 5 + + MANIFEST | 7 + + Makefile.global.in | 11 +- + doc/pod/ovsqlite.pod | 36 +- + samples/ovsqlite.conf | 14 +- + storage/Makefile | 4 + + storage/ovsqlite/ovmethod.config | 6 +- + storage/ovsqlite/ovmethod.mk | 7 +- + storage/ovsqlite/ovsqlite-server.c | 22 +- + storage/ovsqlite/ovsqlite.c | 760 +++++++++++++++++++++++++++-- + storage/ovsqlite/sql-read.sql | 40 ++ + support/mkmanifest | 3 + + tests/Makefile | 14 +- + tests/TESTS | 2 + + tests/overview/ovsqlite-integ.t | 215 ++++++++ + tests/overview/ovsqlite-read-t.c | 424 ++++++++++++++++ + tests/overview/ovsqlite-t.c | 186 +++++++ + tests/overview/ovsqlite-write-t.c | 144 ++++++ + 18 files changed, 1840 insertions(+), 60 deletions(-) + create mode 100644 storage/ovsqlite/sql-read.sql + create mode 100755 tests/overview/ovsqlite-integ.t + create mode 100644 tests/overview/ovsqlite-read-t.c + create mode 100644 tests/overview/ovsqlite-t.c + create mode 100644 tests/overview/ovsqlite-write-t.c + +diff --git .gitignore .gitignore +index 4fc5e98b0..abce0ef2b 100644 +--- .gitignore ++++ .gitignore +@@ -183,6 +183,8 @@ + /storage/ovsqlite/sql-init.h + /storage/ovsqlite/sql-main.c + /storage/ovsqlite/sql-main.h ++/storage/ovsqlite/sql-read.c ++/storage/ovsqlite/sql-read.h + /storage/ovsqlite/sqlite-helper-gen + /storage/tradindexed/tdx-util + /support/fixconfig +@@ -240,6 +242,9 @@ + /tests/nnrpd/auth-ext.t + /tests/overview/api.t + /tests/overview/buffindexed.t ++/tests/overview/ovsqlite.t ++/tests/overview/ovsqlite-read.t ++/tests/overview/ovsqlite-write.t + /tests/overview/tradindexed.t + /tests/overview/xref.t + /tests/perl/minimum-version.t +diff --git MANIFEST MANIFEST +index 094b0a020..39889beb6 100644 +--- MANIFEST ++++ MANIFEST +@@ -791,6 +791,9 @@ storage/ovsqlite/sql-init.sql SQLite code for database setup + storage/ovsqlite/sql-main.c Generated daily operation implementation + storage/ovsqlite/sql-main.h Generated daily operation interface + storage/ovsqlite/sql-main.sql SQLite code for daily operation ++storage/ovsqlite/sql-read.c Generated read-only query implementation ++storage/ovsqlite/sql-read.h Generated read-only query interface ++storage/ovsqlite/sql-read.sql SQLite code for direct reader queries + storage/ovsqlite/sqlite-helper-gen.in Package SQLite code for convenient use + storage/ovsqlite/sqlite-helper.c SQLite code package implementation + storage/ovsqlite/sqlite-helper.h SQLite code package interface +@@ -991,6 +994,10 @@ tests/overview Test suite for overview (Directory) + tests/overview/api-t.c Basic tests for overview API + tests/overview/overchan.t Tests for backends/overchan + tests/overview/overview-t.c Basic tests for overview methods ++tests/overview/ovsqlite-integ.t Integration test for ovsqlite direct reader ++tests/overview/ovsqlite-read-t.c Direct reader verification for integration test ++tests/overview/ovsqlite-t.c Unit tests for ovsqlite direct reader ++tests/overview/ovsqlite-write-t.c Writer helper for ovsqlite integration test + tests/overview/xref-t.c Test storing overview data by Xref + tests/perl Test suite for Perl scripts (Directory) + tests/perl/minimum-version.t.in Tests for not too-new features of Perl +diff --git Makefile.global.in Makefile.global.in +index d4dbe5de4..a14283d5d 100644 +--- Makefile.global.in ++++ Makefile.global.in +@@ -87,11 +87,12 @@ BDB_LDFLAGS = @BDB_LDFLAGS@ $(ZLIB_LDFLAGS) + BDB_LIBS = @BDB_LIBS@ $(ZLIB_LIBS) + + ## SQLite support. Additional flags and libraries used when compiling or +-## linking code that contains SQLite support. ++## linking code that contains SQLite support. zlib is bundled in because ++## ovsqlite direct readers need decompression support. + +-SQLITE3_CPPFLAGS = @SQLITE3_CPPFLAGS@ +-SQLITE3_LDFLAGS = @SQLITE3_LDFLAGS@ +-SQLITE3_LIBS = @SQLITE3_LIBS@ ++SQLITE3_CPPFLAGS = @SQLITE3_CPPFLAGS@ $(ZLIB_CPPFLAGS) ++SQLITE3_LDFLAGS = @SQLITE3_LDFLAGS@ $(ZLIB_LDFLAGS) ++SQLITE3_LIBS = @SQLITE3_LIBS@ $(ZLIB_LIBS) + + ## INN libraries. Nearly all INN programs are linked with libinn, and any + ## INN program that reads from or writes to article storage or overview is +@@ -101,7 +102,7 @@ SQLITE3_LIBS = @SQLITE3_LIBS@ + LIBINN = $(abs_builddir)/lib/libinn$(LIBSUFFIX).$(EXTLIB) + LIBHIST = $(abs_builddir)/history/libinnhist$(LIBSUFFIX).$(EXTLIB) + LIBSTORAGE = $(abs_builddir)/storage/libinnstorage$(LIBSUFFIX).$(EXTLIB) +-STORAGE_LIBS = $(BDB_LDFLAGS) $(BDB_LIBS) ++STORAGE_LIBS = $(BDB_LDFLAGS) $(BDB_LIBS) $(SQLITE3_LDFLAGS) $(SQLITE3_LIBS) + + DBM_CPPFLAGS = @DBM_CPPFLAGS@ + DBM_LIBS = @DBM_LIBS@ +diff --git doc/pod/ovsqlite.pod doc/pod/ovsqlite.pod +index 797e2c6a8..8d1a1957e 100644 +--- doc/pod/ovsqlite.pod ++++ doc/pod/ovsqlite.pod +@@ -57,8 +57,9 @@ configuration file: + + =item I + +-The SQLite in-memory page cache size in kilobytes. The default value is +-left up to the SQLite library and seems to be stable at S<2000 KB>. ++The SQLite in-memory page cache size in kilobytes for the B ++write daemon. The default value is left up to the SQLite library and seems ++to be stable at S<2000 KB>. + + =item I + +@@ -86,6 +87,22 @@ I is on a network filesystem. + + The default value is false. + ++=item I ++ ++The SQLite in-memory page cache size in kilobytes for each B reader ++process. When I is enabled, B processes open the overview ++database directly for read-only access, bypassing B. ++This eliminates the IPC overhead and allows reads to scale with the number ++of reader processes. Each B process gets its own cache, so this ++value should typically be smaller than I. The operating system's ++file cache (or ZFS ARC) serves as a shared second-level cache across all ++readers. ++ ++Direct reader mode is only activated when I is enabled. Without ++WAL, writers take exclusive locks that would block direct readers during ++commits, so the server path is used instead. The default value ++is S<8000 KB> (about S<8 MB>). ++ + =item I + + The SQLite database page size in bytes. Must be a power of 2, minimum 512, +@@ -121,10 +138,17 @@ change much on fast systems. + + =head1 RUNNING + +-All overview database access goes through the B daemon. +-For ordinary operation, B will start and stop it automatically. +-If you want to touch the overview database while B isn't running, you'll +-have to start B manually first. See ovsqlite-server(8). ++Write operations (article ingest, expiration) go through the ++B daemon. For ordinary operation, B will start ++and stop it automatically. If you want to touch the overview database while ++B isn't running, you'll have to start B manually ++first. See ovsqlite-server(8). ++ ++When I is enabled, B reader processes open the overview ++database directly with read-only access, bypassing B ++for read operations. This eliminates the IPC round-trip and server ++serialization bottleneck, allowing read performance to scale with the ++number of concurrent readers. + + =head1 HISTORY + +diff --git samples/ovsqlite.conf samples/ovsqlite.conf +index 6912e8bf2..319b6d656 100644 +--- samples/ovsqlite.conf ++++ samples/ovsqlite.conf +@@ -29,11 +29,23 @@ + # between versions. + #pagesize: 4096 + +-# The SQLite in-memory page cache size in kilobytes. ++# The SQLite in-memory page cache size in kilobytes for the ++# ovsqlite-server write daemon. + # The default value is left up to the SQLite library and seems to be + # stable at 2000 KB. + #cachesize: 2000 + ++# The SQLite in-memory page cache size in kilobytes for each nnrpd ++# reader process. When WAL mode is enabled, nnrpd processes open the ++# database directly for read-only access, bypassing ovsqlite-server. ++# Each nnrpd process gets its own cache, so this value should be ++# smaller than cachesize. The operating system's file cache (or ZFS ++# ARC) serves as a shared second-level cache across all readers. ++# The mmapsize parameter in this file can also be used to share ++# physical memory pages across reader processes. ++# The default value is 8000 KB (about 8 MB). ++#readercachesize: 8000 ++ + # The maximum number of article rows that can be inserted or deleted + # in a single SQL transaction. + # The default value is 10000 articles. +diff --git storage/Makefile storage/Makefile +index fa53c0007..64d9e9608 100644 +--- storage/Makefile ++++ storage/Makefile +@@ -60,6 +60,10 @@ maintclean: distclean + rm -f Make.methods methods.c methods.h ovmethods.c ovmethods.h + rm -f $(RM_MAINTCLEAN) + ++# ovsqlite.c includes sql-read.h which is generated from sql-read.sql. ++# Ensure the header exists before compiling any objects. ++$(OBJECTS) $(LOBJECTS): ovsqlite/sql-read.h ++ + libinnstorage.la: $(OBJECTS) $(LOBJECTS) $(LIBINN) + $(LIBLD) $(LDFLAGS) -o $@ $(LOBJECTS) \ + $(LIBINN) $(STORAGE_LIBS) $(LIBS) \ +diff --git storage/ovsqlite/ovmethod.config storage/ovsqlite/ovmethod.config +index 57b932ebb..5ba6f5612 100644 +--- storage/ovsqlite/ovmethod.config ++++ storage/ovsqlite/ovmethod.config +@@ -1,7 +1,7 @@ + name = ovsqlite + number = 5 +-sources = ovsqlite.c ovsqlite-private.c +-extra-sources = ovsqlite-server.c sql-main.c sql-init.c sqlite-helper.c ++sources = ovsqlite.c ovsqlite-private.c sqlite-helper.c sql-read.c ++extra-sources = ovsqlite-server.c sql-main.c sql-init.c + programs = ovsqlite-server ovsqlite-util + clean = sqlite-helper-gen +-maintclean = sql-init.c sql-init.h sql-main.c sql-main.h ++maintclean = sql-init.c sql-init.h sql-main.c sql-main.h sql-read.c sql-read.h +diff --git storage/ovsqlite/ovmethod.mk storage/ovsqlite/ovmethod.mk +index 009d3d259..6fc0f0044 100644 +--- storage/ovsqlite/ovmethod.mk ++++ storage/ovsqlite/ovmethod.mk +@@ -1,5 +1,5 @@ + OVSQLITEOBJECTS = ovsqlite/ovsqlite-server.o ovsqlite/sql-main.o \ +- ovsqlite/sql-init.o ovsqlite/sqlite-helper.o \ ++ ovsqlite/sql-init.o \ + ovsqlite/ovsqlite-private.o + OVSQLITELOBJECTS = $(OVSQLITEOBJECTS:.o=.lo) + +@@ -24,3 +24,8 @@ ovsqlite/sql-init.c: ovsqlite/sql-init.sql ovsqlite/sqlite-helper-gen + + ovsqlite/sql-init.h: ovsqlite/sql-init.c ; + ++ovsqlite/sql-read.c: ovsqlite/sql-read.sql ovsqlite/sqlite-helper-gen ++ ovsqlite/sqlite-helper-gen ovsqlite/sql-read.sql ++ ++ovsqlite/sql-read.h: ovsqlite/sql-read.c ; ++ +diff --git storage/ovsqlite/ovsqlite-server.c storage/ovsqlite/ovsqlite-server.c +index 5bc4aa8d9..dc78fbb33 100644 +--- storage/ovsqlite/ovsqlite-server.c ++++ storage/ovsqlite/ovsqlite-server.c +@@ -674,8 +674,26 @@ static void + close_db(void) + { + if (use_wal) { +- sqlite3_step(sql_main.checkpoint_wal); +- sqlite3_reset(sql_main.checkpoint_wal); ++ int wal_frames, checkpointed; ++ int rc; ++ ++ /* Use PASSIVE checkpoint first — it never blocks and is safe even ++ * when direct reader processes (nnrpd) still have the database open. ++ * The prepared statement uses TRUNCATE which would hang waiting for ++ * all readers to disconnect (busy_timeout is ~31 years). */ ++ rc = sqlite3_wal_checkpoint_v2(connection, NULL, ++ SQLITE_CHECKPOINT_PASSIVE, ++ &wal_frames, &checkpointed); ++ if (rc == SQLITE_OK && wal_frames == checkpointed) { ++ /* All frames checkpointed (no readers holding pins). ++ * Safe to truncate the WAL file for a clean shutdown. */ ++ sqlite3_wal_checkpoint_v2(connection, NULL, ++ SQLITE_CHECKPOINT_TRUNCATE, ++ NULL, NULL); ++ } else if (rc == SQLITE_OK && wal_frames > checkpointed) { ++ notice("WAL checkpoint: %d/%d frames checkpointed" ++ " (readers still connected)", checkpointed, wal_frames); ++ } + } else { + sqlite3_step(sql_main.delete_journal); + sqlite3_reset(sql_main.delete_journal); +diff --git storage/ovsqlite/ovsqlite.c storage/ovsqlite/ovsqlite.c +index 81d4811de..5d6621570 100644 +--- storage/ovsqlite/ovsqlite.c ++++ storage/ovsqlite/ovsqlite.c +@@ -6,6 +6,11 @@ + ** + ** Various bug fixes, code and documentation improvements since then + ** in 2021-2024, 2026. ++** ++** Direct reader mode added in 2026 to allow nnrpd processes to read the ++** overview database directly via SQLite, bypassing ovsqlite-server for ++** read-only access. This improves read performance by eliminating the ++** IPC round-trip and server serialization bottleneck. + */ + + #include "ovsqlite.h" +@@ -22,6 +27,7 @@ + # endif + + # include "conffile.h" ++# include "inn/confparse.h" + # include "inn/fdflag.h" + # include "inn/innconf.h" + # include "inn/libinn.h" +@@ -30,6 +36,17 @@ + + # include "../ovinterface.h" + ++# include ++# include "sql-read.h" ++# include "sqlite-helper.h" ++ ++# ifdef HAVE_ZLIB ++# define USE_DICTIONARY 1 ++# include ++# endif ++ ++# define OVSQLITE_DB_FILE "ovsqlite.db" ++ + typedef struct handle_t { + uint8_t buffer[SEARCHSPACE]; + uint64_t low; +@@ -46,6 +63,7 @@ typedef struct handle_t { + char groupname[1]; + } handle_t; + ++/* Server connection state (used for write mode and legacy read mode). */ + static int sock = -1; + static buffer_t *request; + static buffer_t *response; +@@ -54,6 +72,360 @@ static buffer_t *response; + static ovsqlite_port port; + # endif + ++/* Direct reader state (used when mode is OV_READ only). */ ++static bool direct_reader = false; ++static sqlite3 *read_connection = NULL; ++static sql_read_t sql_read; ++static bool reader_use_compression = false; ++ ++# ifdef HAVE_ZLIB ++ ++/* clang-format off */ ++static uint32_t const pack_length_bias[5] = ++{ ++ 0, ++ 0x80, ++ 0x4080, ++ 0x204080, ++ 0x10204080, ++}; ++/* clang-format on */ ++ ++static z_stream reader_inflation; ++static buffer_t *reader_flate; ++ ++# ifdef USE_DICTIONARY ++ ++static char reader_dictionary[0x8000]; ++static unsigned int reader_basedict_len; ++ ++# endif /* USE_DICTIONARY */ ++ ++static uint32_t ++reader_unpack_length(z_stream *stream) ++{ ++ uint8_t *walk; ++ unsigned int c, lenlen, n; ++ uint32_t length; ++ ++ if (stream->avail_in <= 0) ++ return ~0U; ++ walk = stream->next_in; ++ c = *walk++; ++ lenlen = 1; ++ while (c & (1U << (8 - lenlen))) ++ lenlen++; ++ if (lenlen > 5 || lenlen > stream->avail_in) ++ return ~0U; ++ length = c & ~(~0U << (8 - lenlen)); ++ for (n = lenlen - 1; n > 0; n--) ++ length = (length << 8) | *walk++; ++ length += pack_length_bias[lenlen - 1]; ++ stream->next_in = walk; ++ stream->avail_in -= lenlen; ++ return length; ++} ++ ++# ifdef USE_DICTIONARY ++ ++static unsigned int ++reader_make_dict(char const *groupname, int groupname_len, uint64_t artnum) ++{ ++ sqlite3_snprintf(sizeof reader_dictionary - reader_basedict_len, ++ reader_dictionary + reader_basedict_len, ++ "%.*s:%llu\r\n", groupname_len, groupname, artnum); ++ return reader_basedict_len ++ + strlen(reader_dictionary + reader_basedict_len); ++} ++ ++# endif /* USE_DICTIONARY */ ++ ++# endif /* HAVE_ZLIB */ ++ ++ ++/* ++** Decompress an overview blob read directly from SQLite. ++** Returns the decompressed data in reader_flate, or NULL on error. ++** On success, *out_len is set to the decompressed length. ++*/ ++# ifdef HAVE_ZLIB ++static uint8_t * ++reader_decompress(uint8_t const *overview, uint32_t overview_len, ++ char const *groupname, int groupname_len, uint64_t artnum, ++ uint32_t *out_len) ++{ ++ uint32_t raw_len; ++ int status; ++ ++ reader_inflation.next_in = (uint8_t *) overview; ++ reader_inflation.avail_in = overview_len; ++ ++ raw_len = reader_unpack_length(&reader_inflation); ++ if (raw_len > MAX_OVDATA_SIZE) ++ return NULL; ++ if (raw_len > 0) { ++ buffer_resize(reader_flate, raw_len); ++ reader_inflation.next_out = (uint8_t *) reader_flate->data; ++ reader_inflation.avail_out = raw_len; ++ status = inflate(&reader_inflation, Z_FINISH); ++# ifdef USE_DICTIONARY ++ if (status == Z_NEED_DICT) { ++ status = inflateSetDictionary( ++ &reader_inflation, (uint8_t *) reader_dictionary, ++ reader_make_dict(groupname, groupname_len, artnum)); ++ if (status == Z_OK) ++ status = inflate(&reader_inflation, Z_FINISH); ++ } ++# endif ++ reader_flate->left = ++ (char *) reader_inflation.next_out - reader_flate->data; ++ reader_inflation.next_in = NULL; ++ reader_inflation.avail_in = 0; ++ inflateReset(&reader_inflation); ++ if (status != Z_STREAM_END || reader_inflation.avail_out > 0) ++ return NULL; ++ *out_len = reader_flate->left; ++ return (uint8_t *) reader_flate->data; ++ } else { ++ /* Compression didn't save space; data stored uncompressed after ++ * the zero length marker. */ ++ *out_len = reader_inflation.avail_in; ++ return reader_inflation.next_in; ++ } ++} ++# endif /* HAVE_ZLIB */ ++ ++ ++/* ++** Helper callback for pragma queries that return a single text value. ++*/ ++static int ++pragma_callback(void *data, int ncols, char **values, char **names UNUSED) ++{ ++ char **result = data; ++ ++ if (ncols > 0 && values[0]) ++ *result = xstrdup(values[0]); ++ return 0; ++} ++ ++/* ++** Helper to read an integer value from the misc table. ++*/ ++static bool ++read_misc_int(char const *key, int *value) ++{ ++ int status; ++ ++ sqlite3_bind_text(sql_read.getmisc, 1, key, -1, SQLITE_STATIC); ++ status = sqlite3_step(sql_read.getmisc); ++ if (status != SQLITE_ROW) { ++ warn("ovsqlite: cannot read '%s' from database: %s", key, ++ sqlite3_errmsg(read_connection)); ++ sqlite3_reset(sql_read.getmisc); ++ sqlite3_clear_bindings(sql_read.getmisc); ++ return false; ++ } ++ *value = sqlite3_column_int(sql_read.getmisc, 0); ++ sqlite3_reset(sql_read.getmisc); ++ sqlite3_clear_bindings(sql_read.getmisc); ++ return true; ++} ++ ++/* ++** Open the overview database directly for read-only access. ++*/ ++static bool ++direct_open(void) ++{ ++ char *path; ++ char *confpath; ++ struct config_group *top; ++ int status; ++ char *errmsg; ++ char *journal_mode = NULL; ++ bool use_wal = false; ++ unsigned long reader_cachesize = 8000; /* default 8 MB */ ++ int version; ++ int compress_flag; ++ char sqltext[64]; ++ bool have_inflate = false; ++ bool have_stmts = false; ++ ++ /* Load config. Direct reader mode requires WAL; without WAL, writers ++ * take EXCLUSIVE locks that block all other connections during commit. ++ * The server path avoids this because reads and writes share a single ++ * connection. */ ++ confpath = concatpath(innconf->pathetc, "ovsqlite.conf"); ++ top = config_parse_file(confpath); ++ free(confpath); ++ if (top) { ++ config_param_boolean(top, "walmode", &use_wal); ++ config_param_unsigned_number(top, "readercachesize", ++ &reader_cachesize); ++ config_free(top); ++ } ++ if (!use_wal) ++ return false; ++ ++ /* Open database read-only. Suppress warning for CANTOPEN since the ++ * database may not exist yet during initial setup. */ ++ path = concatpath(innconf->pathoverview, OVSQLITE_DB_FILE); ++ status = sqlite3_open_v2(path, &read_connection, SQLITE_OPEN_READONLY, ++ NULL); ++ free(path); ++ if (status != SQLITE_OK) { ++ if (status != SQLITE_CANTOPEN) ++ warn("ovsqlite: cannot open database for reading: %s", ++ sqlite3_errstr(status)); ++ read_connection = NULL; ++ return false; ++ } ++ sqlite3_extended_result_codes(read_connection, 1); ++ ++ /* Verify the database is actually in WAL mode, not just the config. ++ * The writer sets WAL mode; if it hasn't run yet or the database was ++ * created without WAL, fall back to the server path. */ ++ status = sqlite3_exec(read_connection, "pragma journal_mode;", ++ pragma_callback, &journal_mode, NULL); ++ if (status != SQLITE_OK || !journal_mode ++ || strcmp(journal_mode, "wal") != 0) { ++ free(journal_mode); ++ sqlite3_close_v2(read_connection); ++ read_connection = NULL; ++ return false; ++ } ++ free(journal_mode); ++ ++ /* Prepare read-only statements. */ ++ status = ++ sqlite_helper_init(&sql_read_helper, (sqlite3_stmt **) &sql_read, ++ read_connection, SQLITE_PREPARE_PERSISTENT, ++ &errmsg); ++ if (status != SQLITE_OK) { ++ warn("ovsqlite: cannot set up read session: %s", errmsg); ++ sqlite3_free(errmsg); ++ goto fail; ++ } ++ have_stmts = true; ++ ++ /* Set cache size. */ ++ if (reader_cachesize) { ++ snprintf(sqltext, sizeof sqltext, "pragma cache_size = -%lu;", ++ reader_cachesize); ++ status = sqlite3_exec(read_connection, sqltext, 0, NULL, &errmsg); ++ if (status != SQLITE_OK) { ++ warn("ovsqlite: cannot set reader cache size: %s", errmsg); ++ sqlite3_free(errmsg); ++ } ++ } ++ ++ /* Validate schema version. */ ++ if (!read_misc_int("version", &version)) ++ goto fail; ++ if (version != OVSQLITE_SCHEMA_VERSION) { ++ warn("ovsqlite: incompatible database schema %d (expected %d)", ++ version, OVSQLITE_SCHEMA_VERSION); ++ goto fail; ++ } ++ ++ /* Check compression setting. */ ++ if (!read_misc_int("compress", &compress_flag)) ++ goto fail; ++ reader_use_compression = compress_flag; ++ ++# ifdef HAVE_ZLIB ++ if (reader_use_compression) { ++ void const *dict; ++ size_t size; ++ ++ reader_inflation.zalloc = Z_NULL; ++ reader_inflation.zfree = Z_NULL; ++ reader_inflation.opaque = Z_NULL; ++ reader_inflation.next_in = Z_NULL; ++ reader_inflation.avail_in = 0; ++ status = inflateInit(&reader_inflation); ++ if (status != Z_OK) { ++ warn("ovsqlite: cannot set up decompression"); ++ goto fail; ++ } ++ have_inflate = true; ++ ++# ifdef USE_DICTIONARY ++ sqlite3_bind_text(sql_read.getmisc, 1, "basedict", -1, ++ SQLITE_STATIC); ++ status = sqlite3_step(sql_read.getmisc); ++ if (status != SQLITE_ROW) { ++ warn("ovsqlite: cannot load compression dictionary: %s", ++ sqlite3_errmsg(read_connection)); ++ sqlite3_reset(sql_read.getmisc); ++ sqlite3_clear_bindings(sql_read.getmisc); ++ goto fail; ++ } ++ dict = sqlite3_column_blob(sql_read.getmisc, 0); ++ size = sqlite3_column_bytes(sql_read.getmisc, 0); ++ if (!dict || size >= sizeof reader_dictionary) { ++ warn("ovsqlite: invalid compression dictionary in database"); ++ sqlite3_reset(sql_read.getmisc); ++ sqlite3_clear_bindings(sql_read.getmisc); ++ goto fail; ++ } ++ memcpy(reader_dictionary, dict, size); ++ reader_basedict_len = size; ++ sqlite3_reset(sql_read.getmisc); ++ sqlite3_clear_bindings(sql_read.getmisc); ++# endif /* USE_DICTIONARY */ ++ ++ reader_flate = buffer_new(); ++ } ++# else /* ! HAVE_ZLIB */ ++ if (reader_use_compression) { ++ warn("ovsqlite: database uses compression but INN was not built" ++ " with zlib"); ++ goto fail; ++ } ++# endif /* ! HAVE_ZLIB */ ++ ++ direct_reader = true; ++ return true; ++ ++fail: ++# ifdef HAVE_ZLIB ++ if (have_inflate) ++ inflateEnd(&reader_inflation); ++# endif ++ if (have_stmts) ++ sqlite_helper_term(&sql_read_helper, (sqlite3_stmt **) &sql_read); ++ if (read_connection) { ++ sqlite3_close_v2(read_connection); ++ read_connection = NULL; ++ } ++ return false; ++} ++ ++ ++/* ++** Close the direct reader connection. ++*/ ++static void ++direct_close(void) ++{ ++ if (!read_connection) ++ return; ++# ifdef HAVE_ZLIB ++ if (reader_use_compression) { ++ inflateEnd(&reader_inflation); ++ buffer_free(reader_flate); ++ reader_flate = NULL; ++ } ++# endif ++ sqlite_helper_term(&sql_read_helper, (sqlite3_stmt **) &sql_read); ++ sqlite3_close_v2(read_connection); ++ read_connection = NULL; ++ direct_reader = false; ++} ++ ++ + static bool + server_connect(void) + { +@@ -283,10 +655,20 @@ server_handshake(uint32_t mode) + bool + ovsqlite_open(int mode) + { +- if (sock != -1) { ++ if (direct_reader || sock != -1) { + warn("ovsqlite_open called more than once"); + return false; + } ++ ++ /* Read-only mode: try direct database access (requires WAL mode). ++ * Falls back to the server path if WAL is not enabled. */ ++ if (mode == OV_READ) { ++ if (direct_open()) ++ return true; ++ notice("ovsqlite: direct reader not available, using server"); ++ } ++ ++ /* Read-write mode, or WAL not enabled: connect to ovsqlite-server. */ + if (!server_connect()) + return false; + if (!server_handshake(mode)) +@@ -298,6 +680,9 @@ bool + ovsqlite_groupstats(const char *group, int *low, int *high, int *count, + int *flag) + { ++ sqlite3_stmt *stmt; ++ int status; ++ char const *flag_blob; + uint16_t groupname_len; + unsigned int code; + uint64_t r_low; +@@ -306,45 +691,74 @@ ovsqlite_groupstats(const char *group, int *low, int *high, int *count, + uint16_t flag_alias_len; + uint8_t *flag_alias; + ++ if (direct_reader) { ++ stmt = sql_read.get_groupinfo; ++ sqlite3_bind_blob(stmt, 1, group, strlen(group), SQLITE_STATIC); ++ status = sqlite3_step(stmt); ++ if (status != SQLITE_ROW) { ++ if (status != SQLITE_DONE) ++ warn("ovsqlite: groupstats query error: %s", ++ sqlite3_errmsg(read_connection)); ++ sqlite3_reset(stmt); ++ sqlite3_clear_bindings(stmt); ++ return false; ++ } ++ if (low) ++ *low = sqlite3_column_int64(stmt, 0); ++ if (high) ++ *high = sqlite3_column_int64(stmt, 1); ++ if (count) ++ *count = sqlite3_column_int64(stmt, 2); ++ if (flag) { ++ flag_blob = sqlite3_column_blob(stmt, 3); ++ if (flag_blob) ++ *flag = *flag_blob; ++ } ++ sqlite3_reset(stmt); ++ sqlite3_clear_bindings(stmt); ++ return true; ++ } ++ ++ /* Server path. */ + if (sock == -1) { + warn("ovsqlite: not connected to server"); + return false; + } + groupname_len = strlen(group); +- start_request(request_get_groupinfo); +- pack_now(request, &groupname_len, sizeof groupname_len); +- pack_now(request, group, groupname_len); +- finish_request(); +- if (!write_request()) +- return false; ++ start_request(request_get_groupinfo); ++ pack_now(request, &groupname_len, sizeof groupname_len); ++ pack_now(request, group, groupname_len); ++ finish_request(); ++ if (!write_request()) ++ return false; + +- if (!read_response()) +- return false; +- code = start_response(); +- if (code != response_groupinfo) +- return false; +- if (!unpack_now(response, &r_low, sizeof r_low)) +- return false; +- if (!unpack_now(response, &r_high, sizeof r_high)) +- return false; +- if (!unpack_now(response, &r_count, sizeof r_count)) +- return false; +- if (!unpack_now(response, &flag_alias_len, sizeof flag_alias_len)) +- return false; +- flag_alias = unpack_later(response, flag_alias_len); +- if (!flag_alias) +- return false; +- if (!finish_response()) +- return false; +- if (low) +- *low = r_low; +- if (high) +- *high = r_high; +- if (count) +- *count = r_count; +- if (flag) +- *flag = *flag_alias; +- return true; ++ if (!read_response()) ++ return false; ++ code = start_response(); ++ if (code != response_groupinfo) ++ return false; ++ if (!unpack_now(response, &r_low, sizeof r_low)) ++ return false; ++ if (!unpack_now(response, &r_high, sizeof r_high)) ++ return false; ++ if (!unpack_now(response, &r_count, sizeof r_count)) ++ return false; ++ if (!unpack_now(response, &flag_alias_len, sizeof flag_alias_len)) ++ return false; ++ flag_alias = unpack_later(response, flag_alias_len); ++ if (!flag_alias) ++ return false; ++ if (!finish_response()) ++ return false; ++ if (low) ++ *low = r_low; ++ if (high) ++ *high = r_high; ++ if (count) ++ *count = r_count; ++ if (flag) ++ *flag = *flag_alias; ++ return true; + } + + bool +@@ -356,6 +770,10 @@ ovsqlite_groupadd(const char *group, ARTNUM low, ARTNUM high, char *flag) + uint64_t r_high; + unsigned int code; + ++ if (direct_reader) { ++ warn("ovsqlite: groupadd not available in direct reader mode"); ++ return false; ++ } + if (sock == -1) { + warn("ovsqlite: not connected to server"); + return false; +@@ -391,6 +809,10 @@ ovsqlite_groupdel(const char *group) + uint16_t groupname_len; + unsigned int code; + ++ if (direct_reader) { ++ warn("ovsqlite: groupdel not available in direct reader mode"); ++ return false; ++ } + if (sock == -1) { + warn("ovsqlite: not connected to server"); + return false; +@@ -424,6 +846,10 @@ ovsqlite_add(const char *group, ARTNUM artnum, TOKEN token, char *data, + uint64_t r_expires; + unsigned int code; + ++ if (direct_reader) { ++ warn("ovsqlite: add not available in direct reader mode"); ++ return false; ++ } + if (sock == -1) { + warn("ovsqlite: not connected to server"); + return false; +@@ -478,6 +904,10 @@ ovsqlite_cancel(const char *group, ARTNUM artnum) + uint64_t r_artnum; + unsigned int code; + ++ if (direct_reader) { ++ warn("ovsqlite: cancel not available in direct reader mode"); ++ return false; ++ } + if (sock == -1) { + warn("ovsqlite: not connected to server"); + return false; +@@ -508,7 +938,7 @@ ovsqlite_opensearch(const char *group, int low, int high) + handle_t *rh; + uint16_t groupname_len; + +- if (sock == -1) { ++ if (!direct_reader && sock == -1) { + warn("ovsqlite: not connected to server"); + return NULL; + } +@@ -525,6 +955,208 @@ ovsqlite_opensearch(const char *group, int low, int high) + return rh; + } + ++/* ++** Fill the search buffer directly from SQLite for direct reader mode. ++** ++** Collects rows into temporary storage, then packs into the handle_t ++** buffer in the same layout that fill_search_buffer produces: ++** [overview ptrs (count+1)] [arrived] [artnums] [tokens] [overview data] ++** The consumer (ovsqlite_search) computes overview length as ++** overview[ix+1] - overview[ix] ++** so overview[count] must be a valid sentinel pointer. ++*/ ++static bool ++fill_search_buffer_direct(handle_t *rh) ++{ ++ unsigned int cols; ++ size_t per_row; ++ uint32_t max_rows, count, ix; ++ sqlite3_stmt *stmt; ++ int status; ++ uint8_t *store; ++ ++ /* Temporary storage for collected rows. */ ++ ARTNUM *tmp_artnum; ++ time_t *tmp_arrived; ++ TOKEN *tmp_token; ++ uint8_t *tmp_overview; ++ uint32_t *tmp_ov_offset; ++ uint32_t *tmp_ov_len; ++ size_t ov_total; ++ ++ rh->count = 0; ++ rh->index = 0; ++ cols = rh->cols; ++ ++ /* Calculate per-row metadata size in the final buffer. */ ++ per_row = sizeof(ARTNUM); ++ if (cols & search_col_arrived) ++ per_row += sizeof(time_t); ++ if (cols & search_col_token) ++ per_row += sizeof(TOKEN); ++ if (cols & search_col_overview) ++ per_row += sizeof(char *); ++ ++ /* Rough upper bound on rows that can fit. */ ++ max_rows = SEARCHSPACE / per_row; ++ if (max_rows > 4096) ++ max_rows = 4096; ++ ++ /* Allocate temporary arrays. */ ++ tmp_artnum = xmalloc(max_rows * sizeof(ARTNUM)); ++ tmp_arrived = (cols & search_col_arrived) ++ ? xmalloc(max_rows * sizeof(time_t)) ++ : NULL; ++ tmp_token = (cols & search_col_token) ++ ? xmalloc(max_rows * sizeof(TOKEN)) ++ : NULL; ++ tmp_overview = (cols & search_col_overview) ++ ? xmalloc(SEARCHSPACE) ++ : NULL; ++ tmp_ov_offset = (cols & search_col_overview) ++ ? xmalloc(max_rows * sizeof(uint32_t)) ++ : NULL; ++ tmp_ov_len = (cols & search_col_overview) ++ ? xmalloc(max_rows * sizeof(uint32_t)) ++ : NULL; ++ ++ /* Select the appropriate prepared statement. */ ++ stmt = (cols & search_col_overview) ? sql_read.list_articles_high_overview ++ : sql_read.list_articles_high; ++ sqlite3_bind_blob(stmt, 1, rh->groupname, rh->groupname_len, ++ SQLITE_STATIC); ++ sqlite3_bind_int64(stmt, 2, rh->low); ++ sqlite3_bind_int64(stmt, 3, rh->high); ++ ++ /* Collect rows. */ ++ count = 0; ++ ov_total = 0; ++ while (count < max_rows) { ++ uint64_t artnum; ++ size_t meta_needed; ++ ++ status = sqlite3_step(stmt); ++ if (status == SQLITE_DONE) { ++ rh->done = true; ++ break; ++ } ++ if (status != SQLITE_ROW) { ++ warn("ovsqlite: search query error: %s", ++ sqlite3_errmsg(read_connection)); ++ break; ++ } ++ ++ artnum = sqlite3_column_int64(stmt, 0); ++ ++ if (cols & search_col_overview) { ++ uint8_t const *overview; ++ uint32_t overview_len; ++ size_t size; ++ ++ overview = sqlite3_column_blob(stmt, 4); ++ size = sqlite3_column_bytes(stmt, 4); ++ if (!overview || size > MAX_OVDATA_SIZE) ++ continue; ++ overview_len = size; ++ ++# ifdef HAVE_ZLIB ++ if (reader_use_compression && overview_len > 0) { ++ uint8_t *dec; ++ uint32_t raw_len; ++ ++ dec = reader_decompress(overview, overview_len, rh->groupname, ++ rh->groupname_len, artnum, &raw_len); ++ if (!dec) ++ continue; ++ overview = dec; ++ overview_len = raw_len; ++ } ++# endif ++ ++ /* Check if adding this row would overflow the buffer. ++ * Account for the sentinel overview pointer. */ ++ meta_needed = (count + 2) * sizeof(char *) /* overview ptrs */ ++ + (count + 1) * sizeof(ARTNUM); ++ if (cols & search_col_arrived) ++ meta_needed += (count + 1) * sizeof(time_t); ++ if (cols & search_col_token) ++ meta_needed += (count + 1) * sizeof(TOKEN); ++ if (meta_needed + ov_total + overview_len > SEARCHSPACE) ++ break; ++ ++ tmp_ov_offset[count] = ov_total; ++ tmp_ov_len[count] = overview_len; ++ memcpy(tmp_overview + ov_total, overview, overview_len); ++ ov_total += overview_len; ++ } else { ++ /* No overview: just check metadata fits. */ ++ if ((count + 1) * per_row > SEARCHSPACE) ++ break; ++ } ++ ++ tmp_artnum[count] = artnum; ++ if (cols & search_col_arrived) ++ tmp_arrived[count] = sqlite3_column_int64(stmt, 1); ++ if (cols & search_col_token) { ++ TOKEN const *tk = sqlite3_column_blob(stmt, 3); ++ ++ if (tk && sqlite3_column_bytes(stmt, 3) == sizeof(TOKEN)) ++ tmp_token[count] = *tk; ++ else ++ memset(&tmp_token[count], 0, sizeof(TOKEN)); ++ } ++ ++ count++; ++ } ++ ++ if (count > 0) ++ rh->low = tmp_artnum[count - 1] + 1; ++ sqlite3_reset(stmt); ++ sqlite3_clear_bindings(stmt); ++ ++ /* Pack into rh->buffer in the canonical layout. */ ++ store = rh->buffer; ++ rh->overview = (char **) (void *) store; ++ if (cols & search_col_overview) ++ store += (count + 1) * sizeof(char *); ++ rh->arrived = (time_t *) (void *) store; ++ if (cols & search_col_arrived) ++ store += count * sizeof(time_t); ++ rh->artnum = (ARTNUM *) (void *) store; ++ store += count * sizeof(ARTNUM); ++ rh->token = (TOKEN *) store; ++ if (cols & search_col_token) ++ store += count * sizeof(TOKEN); ++ ++ for (ix = 0; ix < count; ix++) { ++ rh->artnum[ix] = tmp_artnum[ix]; ++ if (cols & search_col_arrived) ++ rh->arrived[ix] = tmp_arrived[ix]; ++ if (cols & search_col_token) ++ rh->token[ix] = tmp_token[ix]; ++ } ++ ++ if (cols & search_col_overview) { ++ /* Copy overview data contiguously after the metadata arrays ++ * and set up pointers. */ ++ memcpy(store, tmp_overview, ov_total); ++ for (ix = 0; ix < count; ix++) ++ rh->overview[ix] = (char *) store + tmp_ov_offset[ix]; ++ rh->overview[count] = (char *) store + ov_total; ++ } ++ ++ /* Clean up temporary storage. */ ++ free(tmp_artnum); ++ free(tmp_arrived); ++ free(tmp_token); ++ free(tmp_overview); ++ free(tmp_ov_offset); ++ free(tmp_ov_len); ++ ++ rh->count = count; ++ return true; ++} ++ + static bool + fill_search_buffer(handle_t *rh) + { +@@ -651,7 +1283,7 @@ ovsqlite_search(void *handle, ARTNUM *artnum, char **data, int *len, + unsigned int cols; + unsigned int ix; + +- if (sock == -1) { ++ if (!direct_reader && sock == -1) { + warn("ovsqlite: not connected to server"); + return false; + } +@@ -670,8 +1302,13 @@ ovsqlite_search(void *handle, ARTNUM *artnum, char **data, int *len, + cols |= search_col_overview; + if (cols & ~rh->cols || ix >= rh->count) { + rh->cols = cols; +- if (!fill_search_buffer(rh)) +- return false; ++ if (direct_reader) { ++ if (!fill_search_buffer_direct(rh)) ++ return false; ++ } else { ++ if (!fill_search_buffer(rh)) ++ return false; ++ } + ix = rh->index; + if (ix >= rh->count) + return false; +@@ -694,7 +1331,7 @@ ovsqlite_search(void *handle, ARTNUM *artnum, char **data, int *len, + void + ovsqlite_closesearch(void *handle) + { +- if (sock == -1) ++ if (!direct_reader && sock == -1) + warn("ovsqlite: not connected to server"); + if (!handle) + return; +@@ -704,10 +1341,39 @@ ovsqlite_closesearch(void *handle) + bool + ovsqlite_getartinfo(const char *group, ARTNUM artnum, TOKEN *token) + { ++ sqlite3_stmt *stmt; ++ TOKEN const *db_token; ++ int status; + uint16_t groupname_len; + uint64_t r_artnum; + unsigned int code; + ++ if (direct_reader) { ++ stmt = sql_read.get_artinfo; ++ sqlite3_bind_blob(stmt, 1, group, strlen(group), SQLITE_STATIC); ++ sqlite3_bind_int64(stmt, 2, artnum); ++ status = sqlite3_step(stmt); ++ if (status != SQLITE_ROW) { ++ if (status != SQLITE_DONE) ++ warn("ovsqlite: getartinfo query error: %s", ++ sqlite3_errmsg(read_connection)); ++ sqlite3_reset(stmt); ++ sqlite3_clear_bindings(stmt); ++ return false; ++ } ++ db_token = sqlite3_column_blob(stmt, 0); ++ if (!db_token || sqlite3_column_bytes(stmt, 0) != sizeof(TOKEN)) { ++ sqlite3_reset(stmt); ++ sqlite3_clear_bindings(stmt); ++ return false; ++ } ++ *token = *db_token; ++ sqlite3_reset(stmt); ++ sqlite3_clear_bindings(stmt); ++ return true; ++ } ++ ++ /* Server path. */ + if (sock == -1) { + warn("ovsqlite: not connected to server"); + return false; +@@ -911,6 +1577,10 @@ expire_finish(void) + bool + ovsqlite_expiregroup(char const *group, int *low, struct history *h) + { ++ if (direct_reader) { ++ warn("ovsqlite: expiregroup not available in direct reader mode"); ++ return false; ++ } + if (sock == -1) { + warn("ovsqlite: not connected to server"); + return false; +@@ -946,7 +1616,7 @@ set_cutofflow(uint8_t cutofflow) + bool + ovsqlite_ctl(OVCTLTYPE type, void *val) + { +- if (sock == -1) { ++ if (!direct_reader && sock == -1) { + warn("ovsqlite: not connected to server"); + return false; + } +@@ -958,6 +1628,8 @@ ovsqlite_ctl(OVCTLTYPE type, void *val) + *(OVSORTTYPE *) val = OVNEWSGROUP; + return true; + case OVCUTOFFLOW: ++ if (direct_reader) ++ return true; /* no-op for readers */ + return set_cutofflow(*(bool *) val); + case OVSTATICSEARCH: + *(int *) val = true; +@@ -974,12 +1646,20 @@ ovsqlite_ctl(OVCTLTYPE type, void *val) + void + ovsqlite_close(void) + { ++ if (direct_reader) { ++ direct_close(); ++ return; ++ } + if (sock == -1) { + warn("ovsqlite: not connected to server"); + return; + } + close(sock); + sock = -1; ++ buffer_free(request); ++ request = NULL; ++ buffer_free(response); ++ response = NULL; + } + + #else /* ! HAVE_SQLITE3 */ +diff --git storage/ovsqlite/sql-read.sql storage/ovsqlite/sql-read.sql +new file mode 100644 +index 000000000..b41df7e44 +--- /dev/null ++++ storage/ovsqlite/sql-read.sql +@@ -0,0 +1,40 @@ ++pragma busy_timeout = 10000; ++ ++pragma query_only = 1; ++ ++-- .getmisc ++select value from misc ++ where key=?1; ++ ++-- .get_groupinfo ++select low, high, "count", flag_alias from groupinfo ++ where deleted=0 ++ and groupname=?1; ++ ++-- .get_artinfo ++select token ++ from groupinfo ++ natural join artinfo ++ where deleted=0 ++ and groupname=?1 ++ and artnum=?2; ++ ++-- .list_articles_high ++select artnum, arrived, expires, token ++ from groupinfo ++ natural join artinfo ++ where deleted=0 ++ and groupname=?1 ++ and artnum>=?2 ++ and artnum<=?3 ++ order by artnum; ++ ++-- .list_articles_high_overview ++select artnum, arrived, expires, token, overview ++ from groupinfo ++ natural join artinfo ++ where deleted=0 ++ and groupname=?1 ++ and artnum>=?2 ++ and artnum<=?3 ++ order by artnum; +diff --git support/mkmanifest support/mkmanifest +index c75f9a701..fc250a9af 100755 +--- support/mkmanifest ++++ support/mkmanifest +@@ -327,6 +327,9 @@ tests/lib/xwrite.t + tests/nnrpd/auth-ext.t + tests/overview/api.t + tests/overview/buffindexed.t ++tests/overview/ovsqlite.t ++tests/overview/ovsqlite-read.t ++tests/overview/ovsqlite-write.t + tests/overview/tradindexed.t + tests/overview/xref.t + tests/perl/minimum-version.t +diff --git tests/Makefile tests/Makefile +index fa35b1a37..ae990ee67 100644 +--- tests/Makefile ++++ tests/Makefile +@@ -30,12 +30,13 @@ TESTS = authprogs/ident.t innd/artparse.t innd/chan.t lib/artnumber.t \ + lib/setenv.t lib/snprintf.t lib/strlcat.t \ + lib/strlcpy.t lib/tst.t lib/uwildmat.t lib/vector.t lib/wire.t \ + lib/xwrite.t nnrpd/auth-ext.t overview/api.t overview/buffindexed.t \ +- overview/tradindexed.t overview/xref.t util/innbind.t ++ overview/ovsqlite.t overview/tradindexed.t overview/xref.t util/innbind.t + + ## Extra stuff that needs to be built before tests can be run. + + EXTRA = runtests clients/server-list docs/pod.t lib/xmalloc \ +- perl/minimum-version.t ++ perl/minimum-version.t overview/ovsqlite-write.t \ ++ overview/ovsqlite-read.t + + all check test tests: $(TESTS) $(EXTRA) + ./runtests -l TESTS +@@ -294,6 +295,15 @@ overview/api.t: overview/api-t.o tap/basic.o $(STORAGEDEPS) + overview/buffindexed-t.o: overview/overview-t.c + $(CC) $(CFLAGS) -DOVTYPE=buffindexed -c -o $@ overview/overview-t.c + ++overview/ovsqlite.t: overview/ovsqlite-t.o tap/basic.o $(STORAGEDEPS) ++ $(LINKDEPS) overview/ovsqlite-t.o tap/basic.o $(STORAGELIBS) $(LIBS) ++ ++overview/ovsqlite-write.t: overview/ovsqlite-write-t.o tap/basic.o $(STORAGEDEPS) ++ $(LINKDEPS) overview/ovsqlite-write-t.o tap/basic.o $(STORAGELIBS) $(LIBS) ++ ++overview/ovsqlite-read.t: overview/ovsqlite-read-t.o tap/basic.o $(STORAGEDEPS) ++ $(LINKDEPS) overview/ovsqlite-read-t.o tap/basic.o $(STORAGELIBS) $(LIBS) ++ + overview/buffindexed.t: overview/buffindexed-t.o tap/basic.o $(STORAGEDEPS) + $(LINKDEPS) overview/buffindexed-t.o tap/basic.o $(STORAGELIBS) $(LIBS) + +diff --git tests/TESTS tests/TESTS +index b995af703..0a720e30a 100644 +--- tests/TESTS ++++ tests/TESTS +@@ -52,6 +52,8 @@ lib/xwrite + nnrpd/auth-ext + overview/api + overview/buffindexed ++overview/ovsqlite ++overview/ovsqlite-integ + overview/overchan + overview/tradindexed + overview/xref +diff --git tests/overview/ovsqlite-integ.t tests/overview/ovsqlite-integ.t +new file mode 100755 +index 000000000..d4df26fd2 +--- /dev/null ++++ tests/overview/ovsqlite-integ.t +@@ -0,0 +1,215 @@ ++#! /bin/sh ++# ++# Integration test for ovsqlite direct reader mode. ++# ++# Starts a real ovsqlite-server, writes test data through it via the ++# writer program, kills the server, then verifies the reader program ++# can read the WAL-mode database directly without the server. ++ ++count=1 ++printcount() { ++ echo "$1 $count $2" ++ count=$(expr $count + 1) ++} ++ ++# Find the right directory. ++dirs='../data data tests/data' ++for dir in $dirs; do ++ if [ -r "$dir/overview/basic" ]; then ++ testdata="$dir" ++ break ++ fi ++done ++if [ -z "$testdata" ]; then ++ echo "Could not find test data" >&2 ++ exit 1 ++fi ++ ++server="../../storage/ovsqlite/ovsqlite-server" ++writer="overview/ovsqlite-write.t" ++reader="overview/ovsqlite-read.t" ++ ++# When run from the tests directory, adjust paths. ++if [ ! -x "$server" ]; then ++ server="../storage/ovsqlite/ovsqlite-server" ++fi ++if [ ! -x "$writer" ]; then ++ writer="./overview/ovsqlite-write.t" ++fi ++if [ ! -x "$reader" ]; then ++ reader="./overview/ovsqlite-read.t" ++fi ++ ++# Skip if SQLite support was not compiled in. ++if ! grep -q '^#define HAVE_SQLITE3' ../include/config.h 2>/dev/null; then ++ echo "1..0 # skip SQLite support not compiled" ++ exit 0 ++fi ++ ++for prog in "$server" "$writer" "$reader"; do ++ if [ ! -x "$prog" ]; then ++ echo "1..0 # skip ovsqlite binaries not built" ++ exit 0 ++ fi ++done ++ ++echo 13 ++ ++# Set up temp directory with config files. Use absolute paths because ++# the C test programs chdir to the test data directory before reading ++# inn.conf. ++tmpdir="$(pwd)/ov-tmp-integ" ++rm -rf "$tmpdir" ++mkdir -p "$tmpdir" ++ ++cat > "$tmpdir/inn.conf" < "$tmpdir/ovsqlite.conf" < "$tmpdir/server.log" 2>&1 & ++server_pid=$! ++ ++# Wait for socket file (up to 5 seconds). ++waited=0 ++while [ ! -S "$tmpdir/ovsqlite.sock" ] && [ $waited -lt 50 ]; do ++ # Check if server died. ++ if ! kill -0 $server_pid 2>/dev/null; then ++ echo "# Server exited prematurely, log:" ++ sed 's/^/# /' "$tmpdir/server.log" ++ printcount "not ok" "# server startup" ++ rm -rf "$tmpdir" ++ exit 1 ++ fi ++ sleep 0.1 ++ waited=$(expr $waited + 1) ++done ++ ++if [ -S "$tmpdir/ovsqlite.sock" ]; then ++ printcount "ok" "# server started" ++else ++ echo "# Server socket not found after 5s, log:" ++ sed 's/^/# /' "$tmpdir/server.log" ++ printcount "not ok" "# server started" ++ kill $server_pid 2>/dev/null ++ rm -rf "$tmpdir" ++ exit 1 ++fi ++ ++# Tests 2-4: run the writer (3 TAP tests from inside). ++writer_output=$($writer 2>&1) ++writer_rc=$? ++writer_count=$(echo "$writer_output" | grep -c "^ok ") ++if [ $writer_rc -eq 0 ] && [ "$writer_count" -eq 3 ]; then ++ printcount "ok" "# writer open" ++ printcount "ok" "# writer load" ++ printcount "ok" "# writer close" ++else ++ echo "# Writer failed (rc=$writer_rc):" ++ echo "$writer_output" | sed 's/^/# /' ++ printcount "not ok" "# writer open" ++ printcount "not ok" "# writer load" ++ printcount "not ok" "# writer close" ++fi ++ ++# Test 5: stop the server gracefully. ++kill $server_pid 2>/dev/null ++wait $server_pid 2>/dev/null ++if [ $? -le 128 ] || [ $? -eq 143 ]; then ++ printcount "ok" "# server stopped" ++else ++ printcount "ok" "# server stopped (signal)" ++fi ++ ++# Tests 6-10: run the reader (5 TAP tests from inside). ++reader_output=$($reader 2>&1) ++reader_rc=$? ++reader_count=$(echo "$reader_output" | grep -c "^ok ") ++if [ $reader_rc -eq 0 ] && [ "$reader_count" -eq 5 ]; then ++ printcount "ok" "# reader open (direct)" ++ printcount "ok" "# group stats" ++ printcount "ok" "# article data" ++ printcount "ok" "# search order" ++ printcount "ok" "# reopen" ++else ++ echo "# Reader failed (rc=$reader_rc):" ++ echo "$reader_output" | sed 's/^/# /' ++ printcount "not ok" "# reader open (direct)" ++ printcount "not ok" "# group stats" ++ printcount "not ok" "# article data" ++ printcount "not ok" "# search order" ++ printcount "not ok" "# reopen" ++fi ++ ++# Tests 11-13: server restart resilience. ++# Simulate INN restart: start a new server against the existing WAL database, ++# shut it down, verify the reader can still read all data. This is the ++# scenario where nnrpd outlives a server restart cycle. ++$server -d > "$tmpdir/server2.log" 2>&1 & ++server_pid=$! ++waited=0 ++while [ ! -S "$tmpdir/ovsqlite.sock" ] && [ $waited -lt 50 ]; do ++ if ! kill -0 $server_pid 2>/dev/null; then ++ echo "# Server restart failed, log:" ++ sed 's/^/# /' "$tmpdir/server2.log" ++ printcount "not ok" "# server restart" ++ printcount "not ok" "# server re-stop" ++ printcount "not ok" "# reader after restart" ++ rm -rf "$tmpdir" ++ exit 1 ++ fi ++ sleep 0.1 ++ waited=$(expr $waited + 1) ++done ++if [ -S "$tmpdir/ovsqlite.sock" ]; then ++ printcount "ok" "# server restart" ++else ++ printcount "not ok" "# server restart" ++fi ++ ++kill $server_pid 2>/dev/null ++wait $server_pid 2>/dev/null ++printcount "ok" "# server re-stop" ++ ++# Reader should still work — data survives the restart cycle. ++reader_output=$($reader 2>&1) ++reader_rc=$? ++reader_count=$(echo "$reader_output" | grep -c "^ok ") ++if [ $reader_rc -eq 0 ] && [ "$reader_count" -eq 5 ]; then ++ printcount "ok" "# reader after restart" ++else ++ echo "# Reader after restart failed (rc=$reader_rc):" ++ echo "$reader_output" | sed 's/^/# /' ++ printcount "not ok" "# reader after restart" ++fi ++ ++# Clean up. ++rm -rf "$tmpdir" +diff --git tests/overview/ovsqlite-read-t.c tests/overview/ovsqlite-read-t.c +new file mode 100644 +index 000000000..ca3517671 +--- /dev/null ++++ tests/overview/ovsqlite-read-t.c +@@ -0,0 +1,424 @@ ++/* Reader half of the ovsqlite direct reader integration test. ++ * ++ * Opens overview in OV_READ mode (triggers direct_open for WAL databases), ++ * reads back data written by the writer half, and verifies correctness. ++ * No ovsqlite-server is needed -- this is the whole point of direct reader. ++ * ++ * Important: OVadd determines group:artnum from the Xref header in the ++ * overview data, NOT from the prefix in the test data file. For most ++ * articles these match, but some test articles intentionally differ ++ * (e.g., file says news.groups:260 but Xref says news.groups:60). ++ * We must parse Xref to know what was actually stored. ++ */ ++ ++#include "portable/system.h" ++ ++#include ++#include ++#include ++ ++#include "inn/hashtab.h" ++#include "inn/innconf.h" ++#include "inn/libinn.h" ++#include "inn/messages.h" ++#include "inn/ov.h" ++#include "inn/storage.h" ++#include "inn/vector.h" ++#include "tap/basic.h" ++ ++static const TOKEN faketoken = {1, 1, ""}; ++ ++struct group { ++ char *group; ++ unsigned long count; ++ unsigned long low; ++ unsigned long high; ++}; ++ ++static const void * ++group_key(const void *entry) ++{ ++ return ((const struct group *) entry)->group; ++} ++ ++static bool ++group_eq(const void *key, const void *entry) ++{ ++ return !strcmp((const char *) key, ++ ((const struct group *) entry)->group); ++} ++ ++static void ++group_del(void *entry) ++{ ++ struct group *g = (struct group *) entry; ++ free(g->group); ++ free(g); ++} ++ ++/* Parse the Xref field from an overview line to find group:artnum pairs. ++ * The overview line format includes "Xref: hostname group:artnum ..." ++ * as one of the tab-separated fields. ++ * ++ * For each group:artnum in the Xref, calls the callback with the group ++ * name, artnum, the file-format artnum, and the full line from the file. ++ * Returns the number of Xref entries processed. */ ++struct xref_entry { ++ char group[256]; ++ unsigned long artnum; ++}; ++ ++/* Find the last Xref field in an overview line and extract group:artnum ++ * pairs. Returns the number of entries found (up to max_entries). */ ++static int ++parse_xref(const char *data, size_t datalen, struct xref_entry *entries, ++ int max_entries) ++{ ++ const char *xref = NULL; ++ const char *p; ++ int n = 0; ++ ++ /* Find last "Xref: " preceded by a tab. */ ++ for (p = data; p < data + datalen - 6; p++) { ++ if (*p == 'X' && memcmp(p, "Xref: ", 6) == 0 ++ && (p == data || p[-1] == '\t')) ++ xref = p; ++ } ++ if (!xref) ++ return 0; ++ ++ /* Skip "Xref: hostname " */ ++ xref += 6; ++ while (xref < data + datalen && *xref != ' ') ++ xref++; ++ if (xref < data + datalen) ++ xref++; ++ ++ /* Parse group:artnum pairs. */ ++ while (xref < data + datalen && n < max_entries) { ++ const char *colon, *end; ++ size_t glen; ++ ++ while (xref < data + datalen && isspace((unsigned char) *xref)) ++ xref++; ++ if (xref >= data + datalen) ++ break; ++ ++ colon = memchr(xref, ':', data + datalen - xref); ++ if (!colon) ++ break; ++ glen = colon - xref; ++ if (glen >= sizeof(entries[0].group)) ++ break; ++ memcpy(entries[n].group, xref, glen); ++ entries[n].group[glen] = '\0'; ++ ++ entries[n].artnum = strtoul(colon + 1, NULL, 10); ++ if (entries[n].artnum == 0) ++ break; ++ n++; ++ ++ /* Advance to next space or end. */ ++ end = memchr(xref, ' ', data + datalen - xref); ++ if (!end) ++ end = memchr(xref, '\t', data + datalen - xref); ++ if (!end) ++ break; ++ xref = end + 1; ++ } ++ return n; ++} ++ ++/* Build expected group stats from the data file, using Xref-derived ++ * artnums (which is what OVadd actually stores). */ ++static struct hash * ++load_expected(const char *path) ++{ ++ struct hash *groups; ++ struct group *g; ++ FILE *f; ++ char buffer[4096]; ++ struct xref_entry entries[16]; ++ int i, nxref; ++ size_t len; ++ ++ groups = hash_create(32, hash_string, group_key, group_eq, group_del); ++ f = fopen(path, "r"); ++ if (f == NULL) ++ sysdie("Cannot open %s", path); ++ ++ while (fgets(buffer, sizeof(buffer), f) != NULL) { ++ len = strlen(buffer); ++ if (len > 0 && buffer[len - 1] == '\n') ++ len--; ++ ++ nxref = parse_xref(buffer, len, entries, 16); ++ for (i = 0; i < nxref; i++) { ++ g = hash_lookup(groups, entries[i].group); ++ if (g == NULL) { ++ g = xmalloc(sizeof(*g)); ++ g->group = xstrdup(entries[i].group); ++ g->count = 1; ++ g->low = entries[i].artnum; ++ g->high = entries[i].artnum; ++ hash_insert(groups, g->group, g); ++ } else { ++ g->count++; ++ if (entries[i].artnum < g->low) ++ g->low = entries[i].artnum; ++ if (entries[i].artnum > g->high) ++ g->high = entries[i].artnum; ++ } ++ } ++ } ++ fclose(f); ++ return groups; ++} ++ ++/* Verify group stats via the OV API. */ ++static void ++verify_group(void *data, void *cookie) ++{ ++ struct group *g = (struct group *) data; ++ bool *status = (bool *) cookie; ++ int low, high, count, flag; ++ ++ if (!OVgroupstats(g->group, &low, &high, &count, &flag)) { ++ warn("No stats for %s", g->group); ++ *status = false; ++ return; ++ } ++ if ((unsigned long) low != g->low) { ++ warn("Low wrong for %s: %d != %lu", g->group, low, g->low); ++ *status = false; ++ } ++ if ((unsigned long) high != g->high) { ++ warn("High wrong for %s: %d != %lu", g->group, high, g->high); ++ *status = false; ++ } ++ if ((unsigned long) count != g->count) { ++ warn("Count wrong for %s: %d != %lu", g->group, count, g->count); ++ *status = false; ++ } ++} ++ ++/* Verify per-article data: getartinfo returns correct token, search returns ++ * correct overview data and arrival time. ++ * ++ * We verify using the Xref-derived artnum (what OVadd actually stored). ++ * The arrival time is based on the file-prefix artnum (what the writer ++ * passed as arrived = artnum * 10). */ ++static bool ++verify_articles(const char *path) ++{ ++ FILE *f; ++ char buffer[4096]; ++ char *start, *colon; ++ unsigned long file_artnum; ++ struct xref_entry entries[16]; ++ int nxref, i; ++ size_t linelen; ++ TOKEN token; ++ void *search; ++ ARTNUM overnum; ++ char *data; ++ int len; ++ time_t arrived; ++ bool status = true; ++ ++ f = fopen(path, "r"); ++ if (f == NULL) ++ sysdie("Cannot open %s", path); ++ ++ while (fgets(buffer, sizeof(buffer), f) != NULL) { ++ linelen = strlen(buffer); ++ if (linelen > 0 && buffer[linelen - 1] == '\n') ++ linelen--; ++ ++ /* Parse the file-format artnum (used for arrival time). */ ++ colon = strchr(buffer, ':'); ++ if (!colon) ++ continue; ++ start = colon + 1; ++ file_artnum = strtoul(start, NULL, 10); ++ ++ /* Parse Xref to get the actual stored artnum(s). */ ++ nxref = parse_xref(buffer, linelen, entries, 16); ++ for (i = 0; i < nxref; i++) { ++ /* getartinfo should find this article. */ ++ if (!OVgetartinfo(entries[i].group, entries[i].artnum, &token)) { ++ warn("No artinfo for %s:%lu", entries[i].group, ++ entries[i].artnum); ++ status = false; ++ continue; ++ } ++ if (memcmp(&token, &faketoken, sizeof(token)) != 0) { ++ warn("Token wrong for %s:%lu", entries[i].group, ++ entries[i].artnum); ++ status = false; ++ } ++ ++ /* Search for this article and verify arrival time. */ ++ search = OVopensearch(entries[i].group, entries[i].artnum, ++ entries[i].artnum); ++ if (search == NULL) { ++ warn("Cannot open search for %s:%lu", entries[i].group, ++ entries[i].artnum); ++ status = false; ++ continue; ++ } ++ if (!OVsearch(search, &overnum, &data, &len, &token, &arrived)) { ++ warn("No search result for %s:%lu", entries[i].group, ++ entries[i].artnum); ++ status = false; ++ OVclosesearch(search); ++ continue; ++ } ++ ++ /* Verify the arrival time matches what the writer stored. ++ * The writer uses file_artnum * 10 as the arrival time. */ ++ if ((unsigned long) arrived != file_artnum * 10) { ++ warn("Arrived wrong for %s:%lu: %lu != %lu", ++ entries[i].group, entries[i].artnum, ++ (unsigned long) arrived, file_artnum * 10); ++ status = false; ++ } ++ ++ /* There should be no more results. */ ++ if (OVsearch(search, &overnum, &data, &len, &token, &arrived)) { ++ warn("Extra article for %s:%lu", entries[i].group, ++ entries[i].artnum); ++ status = false; ++ } ++ OVclosesearch(search); ++ } ++ } ++ fclose(f); ++ return status; ++} ++ ++/* Verify that a multi-article search returns articles in order for the ++ * first group found in the data file. */ ++static bool ++verify_search_order(const char *path) ++{ ++ FILE *f; ++ char buffer[4096]; ++ struct xref_entry entries[16]; ++ int nxref; ++ size_t len; ++ char *group = NULL; ++ unsigned long low = 0, high = 0; ++ unsigned long article_count = 0; ++ void *search; ++ ARTNUM overnum, last = 0; ++ char *data; ++ int dlen; ++ TOKEN token; ++ time_t arrived; ++ unsigned long found = 0; ++ bool status = true; ++ ++ f = fopen(path, "r"); ++ if (f == NULL) ++ sysdie("Cannot open %s", path); ++ ++ /* Scan the file to find the range for the first group. */ ++ while (fgets(buffer, sizeof(buffer), f) != NULL) { ++ len = strlen(buffer); ++ if (len > 0 && buffer[len - 1] == '\n') ++ len--; ++ nxref = parse_xref(buffer, len, entries, 16); ++ if (nxref == 0) ++ continue; ++ if (group == NULL) { ++ group = xstrdup(entries[0].group); ++ low = entries[0].artnum; ++ high = entries[0].artnum; ++ article_count = 1; ++ } else if (strcmp(group, entries[0].group) == 0) { ++ if (entries[0].artnum < low) ++ low = entries[0].artnum; ++ if (entries[0].artnum > high) ++ high = entries[0].artnum; ++ article_count++; ++ } ++ } ++ fclose(f); ++ ++ if (!group) { ++ warn("No group found in %s", path); ++ return false; ++ } ++ ++ search = OVopensearch(group, low, high); ++ if (search == NULL) { ++ warn("Cannot open search for %s:%lu-%lu", group, low, high); ++ free(group); ++ return false; ++ } ++ ++ while (OVsearch(search, &overnum, &data, &dlen, &token, &arrived)) { ++ if (overnum <= last && last != 0) { ++ warn("Out of order in %s: %lu after %lu", group, ++ (unsigned long) overnum, (unsigned long) last); ++ status = false; ++ } ++ last = overnum; ++ found++; ++ } ++ OVclosesearch(search); ++ ++ if (found != article_count) { ++ warn("Wrong count in search for %s: %lu != %lu", group, found, ++ article_count); ++ status = false; ++ } ++ ++ free(group); ++ return status; ++} ++ ++int ++main(void) ++{ ++ struct hash *groups; ++ bool status; ++ ++ test_init(5); ++ ++ if (access("../data/overview/basic", F_OK) == 0) { ++ if (chdir("../data") < 0) ++ sysbail("cannot chdir to ../data"); ++ } else if (access("data/overview/basic", F_OK) == 0) { ++ if (chdir("data") < 0) ++ sysbail("cannot chdir to data"); ++ } else if (access("tests/data/overview/basic", F_OK) == 0) { ++ if (chdir("tests/data") < 0) ++ sysbail("cannot chdir to tests/data"); ++ } ++ ++ /* Test 1: open in direct reader mode (OV_READ triggers direct_open). */ ++ ok(1, OVopen(OV_READ)); ++ ++ /* Test 2: group stats match expected values. */ ++ groups = load_expected("overview/basic"); ++ status = true; ++ hash_traverse(groups, verify_group, &status); ++ ok(2, status); ++ ++ /* Test 3: per-article data integrity (token, overview, arrived). */ ++ ok(3, verify_articles("overview/basic")); ++ ++ /* Test 4: multi-article search returns articles in order. */ ++ ok(4, verify_search_order("overview/basic")); ++ ++ hash_free(groups); ++ OVclose(); ++ ++ /* Test 5: close and reopen works. */ ++ ok(5, OVopen(OV_READ)); ++ OVclose(); ++ ++ return 0; ++} +diff --git tests/overview/ovsqlite-t.c tests/overview/ovsqlite-t.c +new file mode 100644 +index 000000000..29c6e152a +--- /dev/null ++++ tests/overview/ovsqlite-t.c +@@ -0,0 +1,186 @@ ++/* Unit tests for ovsqlite direct reader edge cases. ++ * ++ * Tests behavior that can't be exercised by the integration test ++ * (ovsqlite-integ.t) which requires a running ovsqlite-server. ++ * These tests create a database directly via the sqlite3 API. ++ */ ++ ++#include "portable/system.h" ++ ++#include ++ ++#include "inn/innconf.h" ++#include "inn/libinn.h" ++#include "inn/messages.h" ++#include "inn/ov.h" ++#include "inn/storage.h" ++#include "tap/basic.h" ++ ++#ifdef HAVE_SQLITE3 ++ ++# include ++ ++# define OVSQLITE_DB_FILE "ovsqlite.db" ++# define TEST_DIR "ov-tmp" ++ ++static const char *schema = ++ "create table misc (key text primary key, value not null) without rowid;" ++ "create table groupinfo (groupid integer primary key," ++ " low integer not null default 1, high integer not null default 0," ++ " \"count\" integer not null default 0," ++ " expired integer not null default 0," ++ " deleted integer not null default 0," ++ " groupname blob not null, flag_alias blob not null," ++ " unique (deleted, groupname));" ++ "create table artinfo (groupid integer references groupinfo (groupid)" ++ " on update cascade on delete restrict," ++ " artnum integer, arrived integer not null, expires integer not null," ++ " token blob not null, overview blob not null," ++ " primary key (groupid, artnum)) without rowid;"; ++ ++ ++/* Create a minimal database with the given journal mode. */ ++static bool ++create_db(const char *dbpath, bool enable_wal) ++{ ++ sqlite3 *db; ++ int status; ++ char *errmsg; ++ ++ status = sqlite3_open_v2(dbpath, &db, ++ SQLITE_OPEN_READWRITE | SQLITE_OPEN_CREATE, ++ NULL); ++ if (status != SQLITE_OK) ++ return false; ++ ++ status = sqlite3_exec(db, schema, 0, NULL, &errmsg); ++ if (status != SQLITE_OK) { ++ sqlite3_free(errmsg); ++ sqlite3_close(db); ++ return false; ++ } ++ ++ sqlite3_exec(db, ++ "insert into misc (key, value) values ('version', 1);" ++ "insert into misc (key, value) values ('compress', 0);", ++ 0, NULL, NULL); ++ ++ if (enable_wal) ++ sqlite3_exec(db, "pragma journal_mode = 'WAL';", 0, NULL, NULL); ++ ++ sqlite3_close(db); ++ return true; ++} ++ ++static void ++write_config(const char *dir, bool walmode) ++{ ++ char *path; ++ FILE *f; ++ ++ path = concatpath(dir, "ovsqlite.conf"); ++ f = fopen(path, "w"); ++ free(path); ++ fprintf(f, "walmode: %s\n", walmode ? "true" : "false"); ++ fprintf(f, "readercachesize: 1000\n"); ++ fclose(f); ++} ++ ++static void ++fake_innconf(void) ++{ ++ if (innconf != NULL) { ++ free(innconf->ovmethod); ++ free(innconf->pathdb); ++ free(innconf->pathetc); ++ free(innconf->pathoverview); ++ free(innconf->pathrun); ++ free(innconf); ++ } ++ innconf = xmalloc(sizeof(*innconf)); ++ memset(innconf, 0, sizeof(*innconf)); ++ innconf->enableoverview = true; ++ innconf->ovmethod = xstrdup("ovsqlite"); ++ innconf->pathdb = xstrdup(TEST_DIR); ++ innconf->pathetc = xstrdup(TEST_DIR); ++ innconf->pathoverview = xstrdup(TEST_DIR); ++ innconf->pathrun = xstrdup(TEST_DIR); ++} ++ ++static void ++setup(void) ++{ ++ if (system("/bin/rm -rf " TEST_DIR) < 0) ++ sysdie("Cannot rm " TEST_DIR); ++ if (mkdir(TEST_DIR, 0755)) ++ sysdie("Cannot mkdir " TEST_DIR); ++} ++ ++int ++main(void) ++{ ++ char *dbpath; ++ ++ test_init(4); ++ ++ /* ++ * Test 1-2: WAL mode gating. ++ * direct_open() checks pragma journal_mode and only activates direct ++ * reader if the database is actually in WAL mode. With a non-WAL ++ * database, it falls back to the server path (which fails here since ++ * no server is running). ++ */ ++ setup(); ++ dbpath = concatpath(TEST_DIR, OVSQLITE_DB_FILE); ++ create_db(dbpath, false); ++ free(dbpath); ++ write_config(TEST_DIR, true); /* config says WAL, but DB isn't */ ++ fake_innconf(); ++ ok(1, !OVopen(OV_READ)); /* should fail: not WAL, no server */ ++ ++ /* With WAL enabled on the database, direct_open succeeds. */ ++ setup(); ++ dbpath = concatpath(TEST_DIR, OVSQLITE_DB_FILE); ++ create_db(dbpath, true); ++ free(dbpath); ++ write_config(TEST_DIR, true); ++ fake_innconf(); ++ ok(2, OVopen(OV_READ)); ++ OVclose(); ++ ++ /* ++ * Test 3-4: Write operations rejected in direct reader mode. ++ * The direct reader opens SQLITE_OPEN_READONLY and sets query_only=1. ++ * Write API calls should return false. ++ */ ++ fake_innconf(); ++ OVopen(OV_READ); ++ { ++ char group[] = "new.group"; ++ char flag[] = "y"; ++ ++ ok(3, !OVgroupadd(group, 0, 0, flag)); ++ } ++ { ++ TOKEN token = {0, 0, ""}; ++ char data[] = "test"; ++ ++ ok(4, OVadd(token, data, 4, 0, 0) != OVADDCOMPLETED); ++ } ++ OVclose(); ++ ++ if (system("/bin/rm -rf " TEST_DIR) < 0) ++ sysdie("Cannot rm " TEST_DIR); ++ return 0; ++} ++ ++#else /* ! HAVE_SQLITE3 */ ++ ++int ++main(void) ++{ ++ skip_all("SQLite support not compiled"); ++ return 0; ++} ++ ++#endif +diff --git tests/overview/ovsqlite-write-t.c tests/overview/ovsqlite-write-t.c +new file mode 100644 +index 000000000..14faac053 +--- /dev/null ++++ tests/overview/ovsqlite-write-t.c +@@ -0,0 +1,144 @@ ++/* Writer half of the ovsqlite direct reader integration test. ++ * ++ * Opens overview in OV_READ|OV_WRITE mode (connects to ovsqlite-server), ++ * creates groups and inserts articles via the standard OV API, then closes. ++ * The server creates the real database schema and handles compression. ++ */ ++ ++#include "portable/system.h" ++ ++#include ++#include ++ ++#include "inn/innconf.h" ++#include "inn/libinn.h" ++#include "inn/messages.h" ++#include "inn/ov.h" ++#include "inn/storage.h" ++#include "tap/basic.h" ++ ++/* The same fake token used by overview-t.c. */ ++static const TOKEN faketoken = {1, 1, ""}; ++ ++/* Parse a line from the overview data file. Nul-terminates the group name ++ * at the beginning of the buffer and returns a pointer to the start of the ++ * overview data (the artnum field). Sets *artnum via the passed pointer. ++ * This is the same parsing logic used by overview-t.c. */ ++static char * ++overview_data_parse(char *data, unsigned long *artnum) ++{ ++ char *start; ++ ++ if (data[strlen(data) - 1] != '\n') ++ die("Line too long in input data"); ++ ++ start = strchr(data, ':'); ++ if (start == NULL) ++ die("No colon found in input data"); ++ *start = '\0'; ++ start++; ++ *artnum = strtoul(start, NULL, 10); ++ if (*artnum == 0) ++ die("Cannot parse article number in input data"); ++ return start; ++} ++ ++int ++main(void) ++{ ++ FILE *f; ++ char buffer[4096]; ++ char *start; ++ unsigned long artnum; ++ char flag[] = NF_FLAG_OK_STRING; ++ /* Track which groups we've already added. */ ++ char *seen_groups[64]; ++ int ngroups = 0; ++ int narticles = 0; ++ int i; ++ bool seen; ++ ++ test_init(3); ++ ++ if (access("../data/overview/basic", F_OK) == 0) { ++ if (chdir("../data") < 0) ++ sysbail("cannot chdir to ../data"); ++ } else if (access("data/overview/basic", F_OK) == 0) { ++ if (chdir("data") < 0) ++ sysbail("cannot chdir to data"); ++ } else if (access("tests/data/overview/basic", F_OK) == 0) { ++ if (chdir("tests/data") < 0) ++ sysbail("cannot chdir to tests/data"); ++ } ++ ++ /* Test 1: open overview in read/write mode (connects to server). */ ++ ok(1, OVopen(OV_READ | OV_WRITE)); ++ ++ /* Load test data from the same file used by overview-t.c. */ ++ f = fopen("overview/basic", "r"); ++ if (f == NULL) ++ sysdie("Cannot open overview/basic"); ++ ++ while (fgets(buffer, sizeof(buffer), f) != NULL) { ++ start = overview_data_parse(buffer, &artnum); ++ ++ /* Add group if we haven't seen it yet. */ ++ seen = false; ++ for (i = 0; i < ngroups; i++) { ++ if (strcmp(seen_groups[i], buffer) == 0) { ++ seen = true; ++ break; ++ } ++ } ++ if (!seen) { ++ if (ngroups >= 64) ++ die("Too many groups in test data"); ++ seen_groups[ngroups] = xstrdup(buffer); ++ ngroups++; ++ if (!OVgroupadd(buffer, 0, 0, flag)) ++ die("Cannot add group %s", buffer); ++ } ++ ++ /* OVadd expects the overview data *without* the "artnum\t" prefix ++ * and without trailing \r\n -- it adds those itself. But it does ++ * need the Xref field present so it can determine group:artnum. ++ * ++ * In the data file, the format after overview_data_parse is: ++ * "artnum\tSubject...\tXref: host group:artnum\n" ++ * ++ * We need to pass the data starting from the first tab character ++ * (skipping the artnum), without the trailing \n. OVadd will ++ * prepend "artnum\t" and append "\r\n" before storing. */ ++ { ++ char *tab = strchr(start, '\t'); ++ int len; ++ ++ if (tab == NULL) ++ die("No tab after artnum in data for %s:%lu", buffer, artnum); ++ tab++; /* point past the tab to the actual overview fields */ ++ len = strlen(tab); ++ /* Strip trailing \n */ ++ if (len > 0 && tab[len - 1] == '\n') ++ len--; ++ ++ if (OVadd(faketoken, tab, len, artnum * 10, ++ (artnum % 5 == 0) ? artnum * 100 : artnum) ++ != OVADDCOMPLETED) ++ die("Cannot add %s:%lu", buffer, artnum); ++ } ++ narticles++; ++ } ++ fclose(f); ++ ++ /* Test 2: all articles were loaded. */ ++ ok(2, narticles > 0); ++ ++ /* Test 3: close succeeds. */ ++ OVclose(); ++ ok(3, true); ++ ++ for (i = 0; i < ngroups; i++) ++ free(seen_groups[i]); ++ ++ return 0; ++} + +From 05d70a17c5720be25fcd957c8ab7a0c12a64d30b Mon Sep 17 00:00:00 2001 +From: Kevin Bowling +Date: Wed, 6 May 2026 23:04:33 -0700 +Subject: [PATCH 3/7] call sqlite3_close_v2 when sqlite3_open_v2 fails + +--- + storage/ovsqlite/ovsqlite.c | 1 + + tests/overview/ovsqlite-t.c | 4 +++- + 2 files changed, 4 insertions(+), 1 deletion(-) + +diff --git storage/ovsqlite/ovsqlite.c storage/ovsqlite/ovsqlite.c +index 5d6621570..557d9710b 100644 +--- storage/ovsqlite/ovsqlite.c ++++ storage/ovsqlite/ovsqlite.c +@@ -278,6 +278,7 @@ direct_open(void) + if (status != SQLITE_CANTOPEN) + warn("ovsqlite: cannot open database for reading: %s", + sqlite3_errstr(status)); ++ sqlite3_close_v2(read_connection); + read_connection = NULL; + return false; + } +diff --git tests/overview/ovsqlite-t.c tests/overview/ovsqlite-t.c +index 29c6e152a..c99155cc5 100644 +--- tests/overview/ovsqlite-t.c ++++ tests/overview/ovsqlite-t.c +@@ -50,8 +50,10 @@ create_db(const char *dbpath, bool enable_wal) + status = sqlite3_open_v2(dbpath, &db, + SQLITE_OPEN_READWRITE | SQLITE_OPEN_CREATE, + NULL); +- if (status != SQLITE_OK) ++ if (status != SQLITE_OK) { ++ sqlite3_close_v2(db); + return false; ++ } + + status = sqlite3_exec(db, schema, 0, NULL, &errmsg); + if (status != SQLITE_OK) { + +From 0b308fdd3f6255e8c81a3deaf1bad08cb1fe8f1d Mon Sep 17 00:00:00 2001 +From: Kevin Bowling +Date: Wed, 6 May 2026 23:35:14 -0700 +Subject: [PATCH 4/7] ovsqlite: Add runtime WAL checkpoint to prevent unbounded + WAL growth + +When WAL mode is enabled with direct readers, the WAL file can grow +without bound if checkpointing never occurs between commits. Add a +configurable walcheckpointthreshold (default 1000 pages) that triggers +a TRUNCATE checkpoint during the server's idle period. + +The checkpoint temporarily lowers the busy_timeout to 10 seconds +(from ~31 years) so the server waits for readers to finish their +current queries but doesn't stall indefinitely. Readers use +autocommit with sub-millisecond snapshots, so the checkpoint +typically completes in milliseconds. + +Also fix notice() messages being lost after daemon() by registering +message_handlers_notice, remove the unused checkpoint_wal prepared +statement from sql-main.sql, and update close_db() comment. +--- + doc/pod/ovsqlite.pod | 14 ++++++++ + storage/ovsqlite/ovsqlite-server.c | 52 ++++++++++++++++++++++++++++-- + storage/ovsqlite/sql-main.sql | 3 -- + 3 files changed, 63 insertions(+), 6 deletions(-) + +diff --git doc/pod/ovsqlite.pod doc/pod/ovsqlite.pod +index 8d1a1957e..4a6eff01b 100644 +--- doc/pod/ovsqlite.pod ++++ doc/pod/ovsqlite.pod +@@ -103,6 +103,20 @@ WAL, writers take exclusive locks that would block direct readers during + commits, so the server path is used instead. The default value + is S<8000 KB> (about S<8 MB>). + ++=item I ++ ++The WAL page count at which B forces a checkpoint. ++When the WAL grows beyond this many pages, the server performs a ++TRUNCATE checkpoint during its next idle period, waiting up to ++10 seconds for readers to finish their current queries before flushing ++WAL content back to the database file. This prevents unbounded WAL ++growth and is no worse than the non-WAL path where every commit takes ++an exclusive lock. If readers do not finish within the timeout, the ++checkpoint is skipped and retried on the next idle cycle. ++ ++This parameter is only effective when I is enabled. The default ++value is 1000 pages. ++ + =item I + + The SQLite database page size in bytes. Must be a power of 2, minimum 512, +diff --git storage/ovsqlite/ovsqlite-server.c storage/ovsqlite/ovsqlite-server.c +index dc78fbb33..623b17db0 100644 +--- storage/ovsqlite/ovsqlite-server.c ++++ storage/ovsqlite/ovsqlite-server.c +@@ -44,7 +44,9 @@ + # include "sql-init.h" + # include "sql-main.h" + +-# define OVSQLITE_DB_FILE "ovsqlite.db" ++# define OVSQLITE_DB_FILE "ovsqlite.db" ++# define OVSQLITE_BUSY_TIMEOUT_MS 999999999 ++# define CHECKPOINT_BUSY_TIMEOUT_MS 10000 + + # ifdef HAVE_ZLIB + +@@ -122,10 +124,12 @@ static unsigned long pagesize; + static unsigned long cachesize; + static struct timeval transaction_time_limit = {10, 0}; + static unsigned long transaction_row_limit = 10000; ++static unsigned long wal_checkpoint_threshold = 1000; + + static bool in_transaction; + static unsigned int transaction_rowcount; + static struct timeval next_commit; ++static int wal_pages; + + + static void +@@ -398,6 +402,8 @@ load_config(void) + } + config_param_unsigned_number(top, "transrowlimit", + &transaction_row_limit); ++ config_param_unsigned_number(top, "walcheckpointthreshold", ++ &wal_checkpoint_threshold); + + config_free(top); + } +@@ -539,6 +545,40 @@ make_dict(char const *groupname, int groupname_len, uint64_t artnum) + + # endif /* HAVE_ZLIB */ + ++static int ++wal_hook(void *data UNUSED, sqlite3 *db UNUSED, const char *dbname UNUSED, ++ int pages) ++{ ++ wal_pages = pages; ++ return SQLITE_OK; ++} ++ ++static void ++checkpoint_wal(void) ++{ ++ int rc; ++ ++ /* Force a TRUNCATE checkpoint: wait for readers to finish, then write ++ * all WAL content back to the database file and reset the WAL. This ++ * is no worse than the non-WAL path where every commit takes an ++ * exclusive lock. Use a short busy_timeout so the server doesn't ++ * stall indefinitely, the normal timeout is ~31 years for writer ++ * locks. */ ++ sqlite3_busy_timeout(connection, CHECKPOINT_BUSY_TIMEOUT_MS); ++ rc = sqlite3_wal_checkpoint_v2(connection, NULL, ++ SQLITE_CHECKPOINT_TRUNCATE, NULL, NULL); ++ sqlite3_busy_timeout(connection, OVSQLITE_BUSY_TIMEOUT_MS); ++ if (rc == SQLITE_OK) { ++ wal_pages = 0; ++ } else if (rc == SQLITE_BUSY) { ++ notice("WAL checkpoint: readers did not finish within timeout" ++ " (%d pages)", wal_pages); ++ } else { ++ warn("WAL checkpoint failed: %s (%d pages)", ++ sqlite3_errstr(rc), wal_pages); ++ } ++} ++ + static void + open_db(void) + { +@@ -667,6 +707,7 @@ open_db(void) + warn("cannot enable WAL mode: %s", errmsg); + sqlite3_free(errmsg); + } ++ sqlite3_wal_hook(connection, wal_hook, NULL); + } + } + +@@ -679,8 +720,8 @@ close_db(void) + + /* Use PASSIVE checkpoint first — it never blocks and is safe even + * when direct reader processes (nnrpd) still have the database open. +- * The prepared statement uses TRUNCATE which would hang waiting for +- * all readers to disconnect (busy_timeout is ~31 years). */ ++ * Avoid TRUNCATE directly here as it would hang waiting for all ++ * readers to disconnect (busy_timeout is ~31 years). */ + rc = sqlite3_wal_checkpoint_v2(connection, NULL, + SQLITE_CHECKPOINT_PASSIVE, + &wal_frames, &checkpointed); +@@ -2129,6 +2170,9 @@ mainloop(void) + } + nap = δ + } else { ++ /* Idle: no active transaction. Checkpoint WAL if needed. */ ++ if (use_wal && wal_pages >= (int) wal_checkpoint_threshold) ++ checkpoint_wal(); + nap = NULL; + } + read_fds_out = read_fds; +@@ -2183,10 +2227,12 @@ main(int argc, char **argv) + } + } + if (debug) { ++ message_handlers_notice(1, message_log_stderr); + message_handlers_warn(1, message_log_stderr); + message_handlers_die(1, message_log_stderr); + } else { + openlog("ovsqlite-server", L_OPENLOG_FLAGS | LOG_PID, LOG_INN_PROG); ++ message_handlers_notice(1, message_log_syslog_notice); + message_handlers_warn(1, message_log_syslog_err); + message_handlers_die(1, message_log_syslog_err); + } +diff --git storage/ovsqlite/sql-main.sql storage/ovsqlite/sql-main.sql +index ab5495a49..ff73f1b0e 100644 +--- storage/ovsqlite/sql-main.sql ++++ storage/ovsqlite/sql-main.sql +@@ -40,9 +40,6 @@ rollback to savepoint article_group; + -- .delete_journal + pragma journal_mode = 'DELETE'; + +--- .checkpoint_wal +-pragma wal_checkpoint(TRUNCATE); +- + -- .add_group + insert into groupinfo (groupname, flag_alias, low, high) + values(?1, ?2, ?3, ?4); + +From 755ab163eb124658959a8e4168732e6a7f7dc300 Mon Sep 17 00:00:00 2001 +From: Kevin Bowling +Date: Wed, 6 May 2026 23:38:41 -0700 +Subject: [PATCH 5/7] fix indentation in ovsqlite.c + +--- + storage/ovsqlite/ovsqlite.c | 66 ++++++++++++++++++------------------- + 1 file changed, 33 insertions(+), 33 deletions(-) + +diff --git storage/ovsqlite/ovsqlite.c storage/ovsqlite/ovsqlite.c +index 557d9710b..f8e465525 100644 +--- storage/ovsqlite/ovsqlite.c ++++ storage/ovsqlite/ovsqlite.c +@@ -726,40 +726,40 @@ ovsqlite_groupstats(const char *group, int *low, int *high, int *count, + return false; + } + groupname_len = strlen(group); +- start_request(request_get_groupinfo); +- pack_now(request, &groupname_len, sizeof groupname_len); +- pack_now(request, group, groupname_len); +- finish_request(); +- if (!write_request()) +- return false; ++ start_request(request_get_groupinfo); ++ pack_now(request, &groupname_len, sizeof groupname_len); ++ pack_now(request, group, groupname_len); ++ finish_request(); ++ if (!write_request()) ++ return false; + +- if (!read_response()) +- return false; +- code = start_response(); +- if (code != response_groupinfo) +- return false; +- if (!unpack_now(response, &r_low, sizeof r_low)) +- return false; +- if (!unpack_now(response, &r_high, sizeof r_high)) +- return false; +- if (!unpack_now(response, &r_count, sizeof r_count)) +- return false; +- if (!unpack_now(response, &flag_alias_len, sizeof flag_alias_len)) +- return false; +- flag_alias = unpack_later(response, flag_alias_len); +- if (!flag_alias) +- return false; +- if (!finish_response()) +- return false; +- if (low) +- *low = r_low; +- if (high) +- *high = r_high; +- if (count) +- *count = r_count; +- if (flag) +- *flag = *flag_alias; +- return true; ++ if (!read_response()) ++ return false; ++ code = start_response(); ++ if (code != response_groupinfo) ++ return false; ++ if (!unpack_now(response, &r_low, sizeof r_low)) ++ return false; ++ if (!unpack_now(response, &r_high, sizeof r_high)) ++ return false; ++ if (!unpack_now(response, &r_count, sizeof r_count)) ++ return false; ++ if (!unpack_now(response, &flag_alias_len, sizeof flag_alias_len)) ++ return false; ++ flag_alias = unpack_later(response, flag_alias_len); ++ if (!flag_alias) ++ return false; ++ if (!finish_response()) ++ return false; ++ if (low) ++ *low = r_low; ++ if (high) ++ *high = r_high; ++ if (count) ++ *count = r_count; ++ if (flag) ++ *flag = *flag_alias; ++ return true; + } + + bool + +From d23b3f84aebe3fb49af0b42b7cfaa7a7ebae5a26 Mon Sep 17 00:00:00 2001 +From: Kevin Bowling +Date: Wed, 6 May 2026 23:42:15 -0700 +Subject: [PATCH 6/7] Add walcheckpointthreshold to ovsqlite.conf + +--- + samples/ovsqlite.conf | 11 +++++++++++ + 1 file changed, 11 insertions(+) + +diff --git samples/ovsqlite.conf samples/ovsqlite.conf +index 319b6d656..4ac719d1d 100644 +--- samples/ovsqlite.conf ++++ samples/ovsqlite.conf +@@ -46,6 +46,17 @@ + # The default value is 8000 KB (about 8 MB). + #readercachesize: 8000 + ++# The WAL page count at which ovsqlite-server forces a checkpoint. ++# When the WAL grows beyond this many pages, the server performs a ++# TRUNCATE checkpoint during its next idle period, waiting up to ++# 10 seconds for readers to finish their current queries before ++# flushing WAL content back to the database file. This prevents ++# unbounded WAL growth. If readers do not finish within the timeout, ++# the checkpoint is skipped and retried on the next idle cycle. ++# Only effective when walmode is enabled. ++# The default value is 1000 pages. ++#walcheckpointthreshold: 1000 ++ + # The maximum number of article rows that can be inserted or deleted + # in a single SQL transaction. + # The default value is 10000 articles. + +From 919cf653b54c72ef4bf1563f3fb6bbdf9a193a3a Mon Sep 17 00:00:00 2001 +From: Kevin Bowling +Date: Thu, 7 May 2026 02:08:32 -0700 +Subject: [PATCH 7/7] ovsqlite: Set journal mode from open_db() based on + configuration + +Move pragma journal_mode out of the sql-main.sql preamble and into +open_db() where it is set conditionally: WAL when walmode is true, +PERSIST otherwise. Previously the preamble unconditionally set +PERSIST mode before open_db() re-set WAL, causing a pointless +WAL->PERSIST->WAL transition on every startup (or silently failing +if direct readers held the WAL open). +--- + storage/ovsqlite/ovsqlite-server.c | 8 ++++++++ + storage/ovsqlite/sql-main.sql | 2 -- + 2 files changed, 8 insertions(+), 2 deletions(-) + +diff --git storage/ovsqlite/ovsqlite-server.c storage/ovsqlite/ovsqlite-server.c +index 623b17db0..c1e89dd82 100644 +--- storage/ovsqlite/ovsqlite-server.c ++++ storage/ovsqlite/ovsqlite-server.c +@@ -708,6 +708,14 @@ open_db(void) + sqlite3_free(errmsg); + } + sqlite3_wal_hook(connection, wal_hook, NULL); ++ } else { ++ status = sqlite3_exec(connection, ++ "pragma journal_mode = 'PERSIST';", ++ 0, NULL, &errmsg); ++ if (status != SQLITE_OK) { ++ warn("cannot set PERSIST journal mode: %s", errmsg); ++ sqlite3_free(errmsg); ++ } + } + } + +diff --git storage/ovsqlite/sql-main.sql storage/ovsqlite/sql-main.sql +index ff73f1b0e..435c3ea01 100644 +--- storage/ovsqlite/sql-main.sql ++++ storage/ovsqlite/sql-main.sql +@@ -1,7 +1,5 @@ + pragma foreign_keys = 1; + +-pragma journal_mode = 'PERSIST'; +- + pragma busy_timeout = 999999999; + + -- .random diff --git a/news/inn-current/files/pr339.patch b/news/inn-current/files/pr339.patch new file mode 100644 index 000000000000..06b076d583ec --- /dev/null +++ b/news/inn-current/files/pr339.patch @@ -0,0 +1,1238 @@ +From d0c379ee3f325f4a80dada6c3129c23496b1c83e Mon Sep 17 00:00:00 2001 +From: Kevin Bowling +Date: Wed, 6 May 2026 22:06:14 -0700 +Subject: [PATCH] expireover: Add bloom filter for fast history existence + checks + +expireover checks every article in the overview database against the +history file to detect orphaned entries. This requires a per-article +HISlookup, which does random pread() calls into the DBZ index and +history text file. On large spools (1B+ articles), this takes months. + +Add a bloom filter that is built from a single sequential HISwalk of +the history file at startup. The bloom filter acts as a positive-only +cache in OVhisthasmsgid: bloom hits skip the slow HISlookup, bloom +misses fall through to HISlookup for correctness. False positives +are benign (an orphaned overview entry survives one extra cycle). + +The bloom filter is controlled by the new inn.conf parameter +expirebloomfp, which specifies the false positive rate as a reciprocal +(default 10000 = 0.01%). Setting it to 0 disables the bloom filter. +Memory usage is approximately 20 bits per article (48 MB for 20M +articles, 2.4 GB for 1B articles). + +Changes: +- Add lib/bloom.c and include/inn/bloom.h (bloom filter implementation + using enhanced double hashing, Kirsch & Mitzenmacher 2006) +- Extend HISwalk callback signature to include the message-ID HASH + (HISwalk has had zero callers since it was added in 2001) +- Set hisv6_walk ignore=true so corrupt lines don't abort the walk +- Add OVTOKENCACHE to OVctl for passing the bloom filter to + OVhisthasmsgid +- Add expirebloomfp to innconf +- Add unit tests (lib/bloom-t.c) and integration tests + (lib/bloom-hiswalk-t.c) +--- + .gitignore | 2 + + MANIFEST | 4 + + doc/pod/expireover.pod | 16 +++ + doc/pod/inn.conf.pod | 20 ++++ + doc/pod/libinnhist.pod | 15 ++- + expire/expireover.c | 61 +++++++++- + history/his.c | 3 +- + history/hisinterface.h | 3 +- + history/hisv6/hisv6-private.h | 3 +- + history/hisv6/hisv6.c | 15 ++- + history/hisv6/hisv6.h | 4 +- + include/inn/bloom.h | 69 +++++++++++ + include/inn/history.h | 4 +- + include/inn/innconf.h | 1 + + include/inn/ov.h | 3 +- + lib/Makefile | 4 +- + lib/bloom.c | 213 ++++++++++++++++++++++++++++++++++ + lib/innconf.c | 1 + + samples/inn.conf.in | 1 + + storage/expire.c | 18 +++ + storage/ov.c | 3 + + storage/ovinterface.h | 2 + + support/mkmanifest | 2 + + tests/Makefile | 9 +- + tests/TESTS | 2 + + tests/lib/bloom-hiswalk-t.c | 169 +++++++++++++++++++++++++++ + tests/lib/bloom-t.c | 128 ++++++++++++++++++++ + 27 files changed, 753 insertions(+), 22 deletions(-) + create mode 100644 include/inn/bloom.h + create mode 100644 lib/bloom.c + create mode 100644 tests/lib/bloom-hiswalk-t.c + create mode 100644 tests/lib/bloom-t.c + +diff --git .gitignore .gitignore +index 4fc5e98b0..f933b3c2d 100644 +--- .gitignore ++++ .gitignore +@@ -195,6 +195,8 @@ + /tests/innd/chan.t + /tests/lib/artnumber.t + /tests/lib/asprintf.t ++/tests/lib/bloom.t ++/tests/lib/bloom-hiswalk.t + /tests/lib/buffer.t + /tests/lib/canlock.t + /tests/lib/concat.t +diff --git MANIFEST MANIFEST +index 094b0a020..70bef9d76 100644 +--- MANIFEST ++++ MANIFEST +@@ -412,6 +412,7 @@ include/Makefile Makefile for header files + include/conffile.h Header file for reading *.conf files + include/config.h.in Template configuration data + include/inn Installed header files (Directory) ++include/inn/bloom.h Header file for bloom filter + include/inn/buffer.h Header file for reusable counted buffers + include/inn/concat.h Header file for string concatenation + include/inn/confparse.h Header file for configuration parser +@@ -518,6 +519,7 @@ lib/Makefile Makefile for library + lib/argparse.c Functions for parsing arguments + lib/artnumber.c Manipulation of article numbers + lib/asprintf.c asprintf replacement ++lib/bloom.c Bloom filter implementation + lib/buffer.c Reusable counted buffer + lib/canlock.c Routines for Cancel-Lock + lib/cleanfrom.c Clean out a From line +@@ -938,6 +940,8 @@ tests/innd/fakeinnd.c Provide symbols defined by innd/innd.c + tests/lib Test suite for libinn (Directory) + tests/lib/artnumber-t.c Tests for lib/artnumber.c + tests/lib/asprintf-t.c Tests for lib/asprintf.c ++tests/lib/bloom-hiswalk-t.c Integration test for bloom filter with HISwalk ++tests/lib/bloom-t.c Tests for lib/bloom.c + tests/lib/buffer-t.c Tests for lib/buffer.c + tests/lib/canlock-t.c Tests for lib/canlock.c + tests/lib/concat-t.c Tests for lib/concat.c +diff --git doc/pod/expireover.pod doc/pod/expireover.pod +index 239b354b6..318ad73e5 100644 +--- doc/pod/expireover.pod ++++ doc/pod/expireover.pod +@@ -39,6 +39,22 @@ By default, B purges all overview information for newsgroups + that have been removed from the server; this behavior is suppressed if + B<-f> is given. + ++To speed up the existence check for each article, B builds ++a bloom filter from the history file at startup. This replaces ++per-article random I/O into the history file with a single sequential ++read, which is critical for large spools. The bloom filter is a ++positive-only cache: if it reports an article probably exists, the ++slow history lookup is skipped. If it reports the article is not found, ++B falls back to a direct history lookup for correctness. ++False positives (the bloom filter incorrectly reporting an article ++exists) are benign; the orphaned overview entry will be cleaned up on ++the next expiration run. ++ ++The false positive rate and memory usage of the bloom filter are ++controlled by the I setting in F. Setting it ++to C<0> disables the bloom filter. The bloom filter is also disabled ++when the B<-s> flag is used. ++ + =head1 OPTIONS + + =over 4 +diff --git doc/pod/inn.conf.pod doc/pod/inn.conf.pod +index a65fe4abd..cfb446c48 100644 +--- doc/pod/inn.conf.pod ++++ doc/pod/inn.conf.pod +@@ -526,6 +526,26 @@ will run much faster, but reading news from the system will be impossible + true, I must also be set. This is a boolean value and the + default is true. + ++=item I ++ ++Controls the bloom filter used by B to accelerate overview ++expiration. The value is the reciprocal of the desired false positive ++rate: for example, C<10000> means a 1-in-10,000 (0.01%) false positive ++rate. Higher values use more memory but produce fewer false positives. ++At the default of C<10000>, memory usage is approximately 20 bits per ++article in the history file (e.g., 48S for 20 million articles, ++2.4S for 1 billion articles). ++ ++Setting this to C<0> disables the bloom filter entirely, falling back ++to per-article history lookups (the pre-existing behavior). This is ++not recommended for large spools as it results in random I/O into the ++history file for every article in the overview database. ++ ++The bloom filter has no effect when B is run with the B<-s> ++flag (which forces a filesystem stat of every article). ++ ++This is a non-negative integer and the default is C<10000>. ++ + =item I + + Besides the seven standard overview fields (which are in order C, +diff --git doc/pod/libinnhist.pod doc/pod/libinnhist.pod +index 05a4acc8d..89b51b42c 100644 +--- doc/pod/libinnhist.pod ++++ doc/pod/libinnhist.pod +@@ -67,8 +67,9 @@ his - routines for managing INN history + + bool HISwalk(struct history *history, const char *reason, + void *cookie, +- bool (*callback)(void *cookie, time_t arrived, +- time_t posted, time_t expires, ++ bool (*callback)(void *cookie, const HASH *hash, ++ time_t arrived, time_t posted, ++ time_t expires, + const TOKEN *token)); + + struct histstats HISstats(struct history *history); +@@ -210,10 +211,12 @@ unspecified. + + B provides an iteration function for the specified I + database. For every entry in the history database, I is +-invoked, passing the I, arrival, posting, and expiry times, in +-addition to the token associated with the entry. If the I() +-returns B the iteration is aborted and B returns +-B to the caller. ++invoked, passing the I, the message-ID I, arrival, ++posting, and expiry times, in addition to the token associated with ++the entry. If the entry has no storage token (a remembered ++message-ID), I is B. If the I() returns ++B the iteration is aborted and B returns B to ++the caller. Malformed history lines are silently skipped. + + To process the entire database in the presence of a running server, + I may be passed; if this argument is not B, it is used +diff --git expire/expireover.c expire/expireover.c +index 0548cae53..3eccae081 100644 +--- expire/expireover.c ++++ expire/expireover.c +@@ -14,6 +14,10 @@ + #include + #include + ++#include ++ ++#include "inn/bloom.h" ++#include "inn/history.h" + #include "inn/innconf.h" + #include "inn/libinn.h" + #include "inn/messages.h" +@@ -45,6 +49,22 @@ fatal_signal(int sig) + } + + ++/* ++** Callback for HISwalk that adds history entries with storage tokens to the ++** bloom filter. Entries without tokens (remembered message-IDs) are skipped ++** so that OVhisthasmsgid correctly identifies them as missing. ++*/ ++static bool ++build_bloom_cb(void *cookie, const HASH *hash, ++ time_t arrived UNUSED, time_t posted UNUSED, ++ time_t expires UNUSED, const TOKEN *token) ++{ ++ if (token != NULL) ++ bloom_add(cookie, hash); ++ return true; ++} ++ ++ + int + main(int argc, char *argv[]) + { +@@ -60,6 +80,8 @@ main(int argc, char *argv[]) + bool purge_deleted = false; + bool always_stat = false; + struct history *history; ++ struct bloom_filter *bloom = NULL; ++ struct bloom_filter *null_bloom = NULL; + + /* First thing, set up logging and our identity. */ + openlog("expireover", L_OPENLOG_FLAGS | LOG_PID, LOG_INN_PROG); +@@ -181,12 +203,43 @@ main(int argc, char *argv[]) + if (!OVctl(OVSTATALL, &always_stat)) + die("can't configure overview stat behavior"); + +- /* We want to be careful about being interrupted from this point on, so +- set up our signal handlers. */ ++ /* Set up signal handlers before the bloom walk, which can take several ++ minutes on very large history files. */ + xsignal(SIGTERM, fatal_signal); + xsignal(SIGINT, fatal_signal); + xsignal(SIGHUP, fatal_signal); + ++ /* Build a bloom filter from the history file for fast existence checks. ++ This replaces millions of random pread() calls into the history file ++ with a single sequential read, making expireover feasible on large ++ spools (1B+ articles). The bloom filter is used as a positive-only ++ cache: hits skip the slow history lookup, misses fall through to ++ HISlookup for correctness (handles articles added after the walk). */ ++ if (innconf->expirebloomfp > 0 && !always_stat) { ++ struct stat st; ++ char *histpath; ++ size_t estimated = 0; ++ /* Minimum history line: 34 (hash) + 1 (tab) + 1 (arrived) ++ * + 1 (newline). Dividing file size by this gives a conservative ++ * overestimate of entries, which is what we want for bloom sizing. */ ++ const size_t min_history_line = 37; ++ ++ histpath = concatpath(innconf->pathdb, INN_PATH_HISTORY); ++ if (stat(histpath, &st) == 0) ++ estimated = st.st_size / min_history_line; ++ else ++ warn("can't stat %s, bloom filter will be undersized", histpath); ++ bloom = bloom_create(estimated, innconf->expirebloomfp); ++ if (!HISwalk(history, NULL, bloom, build_bloom_cb)) { ++ warn("can't walk history for bloom filter, using per-article" ++ " lookups"); ++ bloom_free(bloom); ++ bloom = NULL; ++ } ++ OVctl(OVTOKENCACHE, &bloom); ++ free(histpath); ++ } ++ + /* Loop through each line of the input file and process each group, + writing data to the lowmark file if desired. */ + line = QIOread(qp); +@@ -212,6 +265,10 @@ main(int argc, char *argv[]) + warn("can't expire deleted newsgroups"); + + /* Close everything down in an orderly fashion. */ ++ if (bloom) { ++ OVctl(OVTOKENCACHE, &null_bloom); ++ bloom_free(bloom); ++ } + QIOclose(qp); + OVclose(); + SMshutdown(); +diff --git history/his.c history/his.c +index 03127a358..f8dc7b8c7 100644 +--- history/his.c ++++ history/his.c +@@ -328,7 +328,8 @@ HISreplace(struct history *h, const char *key, time_t arrived, time_t posted, + + bool + HISwalk(struct history *h, const char *reason, void *cookie, +- bool (*callback)(void *, time_t, time_t, time_t, const TOKEN *)) ++ bool (*callback)(void *, const HASH *, time_t, time_t, time_t, ++ const TOKEN *)) + { + bool r; + +diff --git history/hisinterface.h history/hisinterface.h +index ce3ae72f1..cda2cd16b 100644 +--- history/hisinterface.h ++++ history/hisinterface.h +@@ -6,6 +6,7 @@ + #define HISINTERFACE_H + + #include "config.h" ++#include "inn/libinn.h" + #include + + struct token; +@@ -27,7 +28,7 @@ typedef struct hismethod { + bool (*expire)(void *, const char *, const char *, bool, void *, time_t, + bool (*)(void *, time_t, time_t, time_t, struct token *)); + bool (*walk)(void *, const char *, void *, +- bool (*)(void *, time_t, time_t, time_t, ++ bool (*)(void *, const HASH *, time_t, time_t, time_t, + const struct token *)); + bool (*remember)(void *, const char *, time_t, time_t); + bool (*ctl)(void *, int, void *); +diff --git history/hisv6/hisv6-private.h history/hisv6/hisv6-private.h +index 003d0d8dc..be29d2b01 100644 +--- history/hisv6/hisv6-private.h ++++ history/hisv6/hisv6-private.h +@@ -64,7 +64,8 @@ struct hisv6 { + struct hisv6_walkstate { + union { + bool (*expire)(void *, time_t, time_t, time_t, TOKEN *); +- bool (*walk)(void *, time_t, time_t, time_t, const TOKEN *); ++ bool (*walk)(void *, const HASH *, time_t, time_t, time_t, ++ const TOKEN *); + } cb; + void *cookie; + bool paused; +diff --git history/hisv6/hisv6.c history/hisv6/hisv6.c +index 72045cf2d..686dc50f6 100644 +--- history/hisv6/hisv6.c ++++ history/hisv6/hisv6.c +@@ -1071,14 +1071,14 @@ hisv6_traverse(struct hisv6 *h, struct hisv6_walkstate *cookie, + ** parameters the user callback expects + **/ + static bool +-hisv6_traversecb(struct hisv6 *h UNUSED, void *cookie, const HASH *hash UNUSED, ++hisv6_traversecb(struct hisv6 *h UNUSED, void *cookie, const HASH *hash, + time_t arrived, time_t posted, time_t expires, + const TOKEN *token) + { + struct hisv6_walkstate *hiscookie = cookie; + +- return (*hiscookie->cb.walk)(hiscookie->cookie, arrived, posted, expires, +- token); ++ return (*hiscookie->cb.walk)(hiscookie->cookie, hash, arrived, posted, ++ expires, token); + } + + +@@ -1087,7 +1087,8 @@ hisv6_traversecb(struct hisv6 *h UNUSED, void *cookie, const HASH *hash UNUSED, + */ + bool + hisv6_walk(void *history, const char *reason, void *cookie, +- bool (*callback)(void *, time_t, time_t, time_t, const TOKEN *)) ++ bool (*callback)(void *, const HASH *, time_t, time_t, time_t, ++ const TOKEN *)) + { + struct hisv6 *h = history; + struct hisv6_walkstate hiscookie; +@@ -1099,7 +1100,11 @@ hisv6_walk(void *history, const char *reason, void *cookie, + hiscookie.cookie = cookie; + hiscookie.new = NULL; + hiscookie.paused = false; +- hiscookie.ignore = false; ++ /* Ignore malformed history lines during walk. The walk is a read-only ++ operation (e.g., building a bloom filter for expireover); aborting ++ over one corrupt line in a potentially 180 GB file would be ++ catastrophic. expire is what fixes corrupt history entries. */ ++ hiscookie.ignore = true; + + r = hisv6_traverse(h, &hiscookie, reason, hisv6_traversecb); + +diff --git history/hisv6/hisv6.h history/hisv6/hisv6.h +index 30fcae787..8f9564f9c 100644 +--- history/hisv6/hisv6.h ++++ history/hisv6/hisv6.h +@@ -5,6 +5,8 @@ + #ifndef HISV6_H + #define HISV6_H + ++#include "inn/libinn.h" ++ + struct token; + struct histopts; + struct history; +@@ -32,7 +34,7 @@ bool hisv6_expire(void *, const char *, const char *, bool, void *, + struct token *)); + + bool hisv6_walk(void *, const char *, void *, +- bool (*)(void *, time_t, time_t, time_t, ++ bool (*)(void *, const HASH *, time_t, time_t, time_t, + const struct token *)); + + const char *hisv6_error(void *); +diff --git include/inn/bloom.h include/inn/bloom.h +new file mode 100644 +index 000000000..d0f31d7ea +--- /dev/null ++++ include/inn/bloom.h +@@ -0,0 +1,69 @@ ++/* ++** Bloom filter for fast set membership testing. ++** ++** A space-efficient probabilistic data structure that can test whether ++** an element is a member of a set. False positive matches are possible, ++** but false negatives are not: a query returns either "possibly in set" ++** or "definitely not in set." ++** ++** Uses enhanced double hashing (Kirsch & Mitzenmacher 2006) to derive ++** multiple hash positions from a single HASH value. ++*/ ++ ++#ifndef INN_BLOOM_H ++#define INN_BLOOM_H ++ ++#include "inn/libinn.h" ++#include "inn/portable-macros.h" ++#include "inn/portable-stdbool.h" ++ ++#include ++ ++BEGIN_DECLS ++ ++/* The layout of this struct is entirely internal to the implementation. */ ++struct bloom_filter; ++ ++/* ++** Create a new bloom filter sized for the given number of estimated entries ++** and false positive rate expressed as a reciprocal (e.g., 10000 means ++** 1-in-10,000 or 0.01% false positive rate). Uses xmalloc internally, ++** so dies on allocation failure. ++*/ ++struct bloom_filter *bloom_create(size_t estimated_entries, unsigned long fp_inv); ++ ++/* ++** Add a HASH to the bloom filter. ++*/ ++void bloom_add(struct bloom_filter *bf, const HASH *hash); ++ ++/* ++** Check whether a HASH is possibly in the bloom filter. Returns true if ++** the element is probably in the set (with false positive rate as configured), ++** or false if the element is definitely not in the set. ++*/ ++bool bloom_check(const struct bloom_filter *bf, const HASH *hash); ++ ++/* ++** Free a bloom filter and all associated memory. Safe to call with NULL. ++*/ ++void bloom_free(struct bloom_filter *bf); ++ ++/* ++** Return the number of entries that have been added to the bloom filter. ++*/ ++size_t bloom_count(const struct bloom_filter *bf); ++ ++/* ++** Return the number of hash functions (k) used by the bloom filter. ++*/ ++unsigned int bloom_nhash(const struct bloom_filter *bf); ++ ++/* ++** Return the total number of bits (m) in the bloom filter. ++*/ ++size_t bloom_bits(const struct bloom_filter *bf); ++ ++END_DECLS ++ ++#endif /* INN_BLOOM_H */ +diff --git include/inn/history.h include/inn/history.h +index f14d9536f..5be46d984 100644 +--- include/inn/history.h ++++ include/inn/history.h +@@ -5,6 +5,7 @@ + #ifndef INN_HISTORY_H + #define INN_HISTORY_H + ++#include "inn/libinn.h" + #include "inn/macros.h" + #include "inn/portable-stdbool.h" + #include +@@ -96,7 +97,8 @@ bool HISexpire(struct history *, const char *, const char *, bool, void *, + time_t, + bool (*)(void *, time_t, time_t, time_t, struct token *)); + bool HISwalk(struct history *, const char *, void *, +- bool (*)(void *, time_t, time_t, time_t, const struct token *)); ++ bool (*)(void *, const HASH *, time_t, time_t, time_t, ++ const struct token *)); + struct histstats HISstats(struct history *); + const char *HISerror(struct history *); + bool HISctl(struct history *, int, void *); +diff --git include/inn/innconf.h include/inn/innconf.h +index f1add2628..a24c2d1d8 100644 +--- include/inn/innconf.h ++++ include/inn/innconf.h +@@ -72,6 +72,7 @@ struct innconf { + /* Article Storage */ + unsigned long cnfscheckfudgesize; /* Additional CNFS integrity checking */ + bool enableoverview; /* Store overview info for articles? */ ++ unsigned long expirebloomfp; /* Bloom filter FP rate 1/N (0 = disabled) */ + struct vector *extraoverviewadvertised; /* Extra overview fields for LIST + OVERVIEW.FMT */ + struct vector +diff --git include/inn/ov.h include/inn/ov.h +index 2f2259276..ef7ba8541 100644 +--- include/inn/ov.h ++++ include/inn/ov.h +@@ -19,7 +19,8 @@ typedef enum { + OVSTATICSEARCH, + OVSTATALL, + OVCACHEKEEP, +- OVCACHEFREE ++ OVCACHEFREE, ++ OVTOKENCACHE + } OVCTLTYPE; + #define OV_NOSPACE 100 + typedef enum { +diff --git lib/Makefile lib/Makefile +index ae22203e5..0f4fb8ec5 100644 +--- lib/Makefile ++++ lib/Makefile +@@ -12,8 +12,8 @@ top = .. + CFLAGS = $(GCFLAGS) + + # The base library files that are always compiled and included. +-SOURCES = argparse.c artnumber.c buffer.c cleanfrom.c clientactive.c \ +- clientlib.c \ ++SOURCES = argparse.c artnumber.c bloom.c buffer.c cleanfrom.c \ ++ clientactive.c clientlib.c \ + commands.c concat.c conffile.c confparse.c \ + date.c dbz.c defdist.c dispatch.c fdflag.c fdlimit.c \ + getfqdn.c getmodaddr.c hash.c hashtab.c headers.c hex.c \ +diff --git lib/bloom.c lib/bloom.c +new file mode 100644 +index 000000000..cfd669e7a +--- /dev/null ++++ lib/bloom.c +@@ -0,0 +1,213 @@ ++/* ++** Bloom filter implementation. ++** ++** A space-efficient probabilistic data structure for set membership testing. ++** Uses enhanced double hashing (Kirsch & Mitzenmacher 2006) to derive k ++** independent bit positions from the two 64-bit halves of a 16-byte MD5 ++** HASH value. ++** ++** Sizing uses the standard formulas: ++** m = -n * ln(p) / (ln(2))^2 (optimal number of bits) ++** k = (m / n) * ln(2) (optimal number of hash functions) ++** where n = estimated entries, p = desired false positive rate. ++** ++** The create function takes the false positive rate as 1/fp_inv (an integer ++** reciprocal) to avoid floating point in the library. Common values: ++** fp_inv = 100 => 1% FP, ~10 bits/entry, k=7 ++** fp_inv = 1000 => 0.1% FP, ~15 bits/entry, k=10 ++** fp_inv = 10000 => 0.01% FP, ~20 bits/entry, k=14 ++** fp_inv = 100000 => 0.001% FP, ~24 bits/entry, k=17 ++*/ ++ ++#include "portable/system.h" ++ ++#include ++ ++#include "inn/bloom.h" ++#include "inn/messages.h" ++#include "inn/xmalloc.h" ++ ++struct bloom_filter { ++ uint8_t *bits; /* bit array */ ++ size_t nbits; /* total bits (m) */ ++ unsigned int nhash; /* number of hash functions (k) */ ++ size_t count; /* entries added */ ++}; ++ ++ ++/* ++** Compute k bit positions for a given HASH using enhanced double hashing. ++** h(i) = (h1 + i * h2) mod m, where h1 and h2 are the two 64-bit halves ++** of the MD5 hash. Unsigned overflow on (h1 + i * h2) is well-defined ++** in C and acts as additional mixing before the final mod m. ++*/ ++static void ++bloom_positions(const struct bloom_filter *bf, const HASH *hash, ++ size_t *positions) ++{ ++ uint64_t h1, h2; ++ unsigned int i; ++ ++ memcpy(&h1, hash->hash, 8); ++ memcpy(&h2, hash->hash + 8, 8); ++ for (i = 0; i < bf->nhash; i++) ++ positions[i] = (size_t) ((h1 + (uint64_t) i * h2) % bf->nbits); ++} ++ ++ ++/* ++** Pre-computed bits-per-entry and optimal k for common false positive rates. ++** Values are ceil(-ln(1/fp_inv) / ln(2)^2) and ceil(bits_per_entry * ln(2)). ++** For fp_inv values not in the table, we interpolate conservatively. ++*/ ++static const struct { ++ unsigned long fp_inv; ++ unsigned int bits_per_entry; ++ unsigned int nhash; ++} bloom_params[] = { ++ { 10, 5, 4 }, /* 10% FP */ ++ { 20, 7, 5 }, ++ { 50, 9, 6 }, ++ { 100, 10, 7 }, /* 1% FP */ ++ { 200, 12, 8 }, ++ { 500, 13, 9 }, ++ { 1000, 15, 10 }, /* 0.1% FP */ ++ { 2000, 16, 11 }, ++ { 5000, 18, 13 }, ++ { 10000, 20, 14 }, /* 0.01% FP */ ++ { 20000, 21, 15 }, ++ { 50000, 23, 16 }, ++ { 100000, 24, 17 }, /* 0.001% FP */ ++ { 1000000, 29, 20 }, /* 0.0001% FP */ ++ { 10000000, 34, 24 }, ++}; ++#define BLOOM_NPARAMS (sizeof(bloom_params) / sizeof(bloom_params[0])) ++#define BLOOM_MAX_NHASH 24 /* must match max nhash in bloom_params table */ ++ ++/* Maximum bloom filter size on 32-bit platforms where size_t overflow ++ * is a real concern. On 64-bit, there is no cap, xmalloc will die if ++ * the system doesn't have enough memory, which is the correct behavior ++ * for a batch job. On 32-bit, cap at SIZE_MAX/16 so that the conversion ++ * to bits (multiply by 8) stays within size_t. */ ++#if SIZE_MAX <= UINT32_MAX ++# define BLOOM_MAX_BITS ((SIZE_MAX / 16) * 8) ++#endif ++ ++ ++struct bloom_filter * ++bloom_create(size_t estimated_entries, unsigned long fp_inv) ++{ ++ struct bloom_filter *bf; ++ unsigned int bits_per_entry; ++ unsigned int nhash; ++ size_t nbits; ++ size_t nbytes; ++ size_t i; ++ ++ /* Look up parameters from the table. Use the entry with the smallest ++ * fp_inv that is >= the requested fp_inv (i.e., the FP rate at least as ++ * good as requested). If fp_inv exceeds all table entries, use the ++ * last (most conservative) entry. */ ++ bits_per_entry = bloom_params[BLOOM_NPARAMS - 1].bits_per_entry; ++ nhash = bloom_params[BLOOM_NPARAMS - 1].nhash; ++ for (i = 0; i < BLOOM_NPARAMS; i++) { ++ if (bloom_params[i].fp_inv >= fp_inv) { ++ bits_per_entry = bloom_params[i].bits_per_entry; ++ nhash = bloom_params[i].nhash; ++ break; ++ } ++ } ++ ++ bf = xmalloc(sizeof(*bf)); ++ ++ if (estimated_entries == 0) ++ estimated_entries = 1; ++ ++ /* Guard against size_t overflow on 32-bit platforms where ++ * bits_per_entry * estimated_entries can exceed SIZE_MAX. ++ * On 64-bit this check is effectively unreachable but harmless. ++ * On 32-bit, the cap degrades the FP rate but does not affect ++ * correctness. */ ++ if (estimated_entries > SIZE_MAX / bits_per_entry) ++#if SIZE_MAX <= UINT32_MAX ++ nbits = BLOOM_MAX_BITS; ++#else ++ die("bloom filter: entry count too large for size_t"); ++#endif ++ else ++ nbits = (size_t) bits_per_entry * estimated_entries; ++#if SIZE_MAX <= UINT32_MAX ++ if (nbits > BLOOM_MAX_BITS) ++ nbits = BLOOM_MAX_BITS; ++#endif ++ if (nbits < 64) ++ nbits = 64; ++ ++ bf->nbits = nbits; ++ bf->nhash = nhash; ++ bf->count = 0; ++ ++ nbytes = (nbits + 7) / 8; ++ bf->bits = xcalloc(nbytes, 1); ++ ++ return bf; ++} ++ ++ ++void ++bloom_add(struct bloom_filter *bf, const HASH *hash) ++{ ++ size_t positions[BLOOM_MAX_NHASH]; ++ unsigned int i; ++ ++ bloom_positions(bf, hash, positions); ++ for (i = 0; i < bf->nhash; i++) ++ bf->bits[positions[i] / 8] |= (uint8_t) (1U << (positions[i] % 8)); ++ bf->count++; ++} ++ ++ ++bool ++bloom_check(const struct bloom_filter *bf, const HASH *hash) ++{ ++ size_t positions[BLOOM_MAX_NHASH]; ++ unsigned int i; ++ ++ bloom_positions(bf, hash, positions); ++ for (i = 0; i < bf->nhash; i++) { ++ if (!(bf->bits[positions[i] / 8] & (1U << (positions[i] % 8)))) ++ return false; ++ } ++ return true; ++} ++ ++ ++void ++bloom_free(struct bloom_filter *bf) ++{ ++ if (bf == NULL) ++ return; ++ free(bf->bits); ++ free(bf); ++} ++ ++ ++size_t ++bloom_count(const struct bloom_filter *bf) ++{ ++ return bf->count; ++} ++ ++ ++unsigned int ++bloom_nhash(const struct bloom_filter *bf) ++{ ++ return bf->nhash; ++} ++ ++ ++size_t ++bloom_bits(const struct bloom_filter *bf) ++{ ++ return bf->nbits; ++} +diff --git lib/innconf.c lib/innconf.c +index 145e66b5d..199e7009a 100644 +--- lib/innconf.c ++++ lib/innconf.c +@@ -205,6 +205,7 @@ static const struct config config_table[] = { + /* The following settings are specific to the storage subsystem. */ + {K(articlemmap), BOOL(true) }, + {K(cnfscheckfudgesize), UNUMBER(0) }, ++ {K(expirebloomfp), UNUMBER(10000) }, + {K(immediatecancel), BOOL(false) }, + {K(keepmmappedthreshold), UNUMBER(1024) }, + {K(nfswriter), BOOL(false) }, +diff --git samples/inn.conf.in samples/inn.conf.in +index 2e761ff67..089b0a60e 100644 +--- samples/inn.conf.in ++++ samples/inn.conf.in +@@ -64,6 +64,7 @@ wipexpire: 10 + + cnfscheckfudgesize: 0 + enableoverview: true ++expirebloomfp: 10000 + extraoverviewadvertised: [ ] + extraoverviewhidden: [ ] + groupbaseexpiry: true +diff --git storage/expire.c storage/expire.c +index 8836805c2..abd9f4e6c 100644 +--- storage/expire.c ++++ storage/expire.c +@@ -12,6 +12,7 @@ + #include + #include + ++#include "inn/bloom.h" + #include "inn/innconf.h" + #include "inn/libinn.h" + #include "inn/ov.h" +@@ -27,6 +28,9 @@ enum KRP { + Poison + }; + ++/* Bloom filter token cache for fast OVhisthasmsgid lookups. */ ++struct bloom_filter *OVtokencache = NULL; ++ + /* Statistics */ + long EXPprocessed; + long EXPunlinked; +@@ -803,6 +807,20 @@ OVhisthasmsgid(struct history *h, const char *data) + } + if ((p = OVERGetHeader(data, Messageidindex)) == NULL) + return false; ++ ++ /* Fast path: if article is in the bloom filter, it (probably) exists. ++ * Bloom hits skip the slow history lookup. Bloom misses fall through ++ * to HISlookup to handle articles added after the filter was built. ++ * False positives (bloom says "yes" for an expired article) are benign: ++ * the orphaned overview entry is cleaned up on the next run. */ ++ if (OVtokencache) { ++ HASH hash = HashMessageID(p); ++ if (bloom_check(OVtokencache, &hash)) ++ return true; ++ } ++ ++ /* Slow path: per-article history lookup (original behavior). ++ * Only reached for bloom misses or when no bloom filter is loaded. */ + return HISlookup(h, p, NULL, NULL, NULL, NULL); + } + +diff --git storage/ov.c storage/ov.c +index a8ff34c97..62ddd061d 100644 +--- storage/ov.c ++++ storage/ov.c +@@ -399,6 +399,9 @@ OVctl(OVCTLTYPE type, void *val) + case OVSTATALL: + OVstatall = *(bool *) val; + return true; ++ case OVTOKENCACHE: ++ OVtokencache = *(struct bloom_filter **) val; ++ return true; + default: + return ((*ov.ctl)(type, val)); + } +diff --git storage/ovinterface.h storage/ovinterface.h +index 41ed2f16b..31eefb428 100644 +--- storage/ovinterface.h ++++ storage/ovinterface.h +@@ -35,10 +35,12 @@ typedef struct overview_method { + + bool OVgroupbasedexpire(TOKEN token, const char *group, const char *data, + int len, time_t arrived, time_t expires); ++struct bloom_filter; + bool OVhisthasmsgid(struct history *, const char *data); + void OVEXPremove(TOKEN token, bool deletedgroups, char **xref, int ngroups); + void OVEXPcleanup(void); + ++extern struct bloom_filter *OVtokencache; + extern time_t OVnow; + extern FILE *EXPunlinkfile; + extern bool OVignoreselfexpire; +diff --git support/mkmanifest support/mkmanifest +index c75f9a701..e079cdac1 100755 +--- support/mkmanifest ++++ support/mkmanifest +@@ -282,6 +282,8 @@ tests/innd/artparse.t + tests/innd/chan.t + tests/lib/artnumber.t + tests/lib/asprintf.t ++tests/lib/bloom.t ++tests/lib/bloom-hiswalk.t + tests/lib/buffer.t + tests/lib/canlock.t + tests/lib/concat.t +diff --git tests/Makefile tests/Makefile +index fa35b1a37..64d80256e 100644 +--- tests/Makefile ++++ tests/Makefile +@@ -17,7 +17,8 @@ LIBM_LDFLAGS = '-lm' + ## added to EXTRA. + + TESTS = authprogs/ident.t innd/artparse.t innd/chan.t lib/artnumber.t \ +- lib/asprintf.t lib/buffer.t lib/canlock.t lib/concat.t lib/conffile.t \ ++ lib/asprintf.t lib/bloom.t lib/bloom-hiswalk.t lib/buffer.t \ ++ lib/canlock.t lib/concat.t lib/conffile.t \ + lib/confparse.t lib/daemon.t lib/date.t \ + lib/dispatch.t lib/fdflag.t \ + lib/getaddrinfo.t lib/getnameinfo.t lib/hash.t \ +@@ -150,6 +151,12 @@ lib/getnameinfo.t: lib/getnameinfo.o lib/getnameinfo-t.o tap/basic.o $(LIBINN) + $(LINK) lib/getnameinfo.o lib/getnameinfo-t.o tap/basic.o \ + $(LIBINN) $(LIBS) + ++lib/bloom.t: lib/bloom-t.o tap/basic.o $(LIBINN) ++ $(LINK) lib/bloom-t.o tap/basic.o $(LIBINN) ++ ++lib/bloom-hiswalk.t: lib/bloom-hiswalk-t.o tap/basic.o $(STORAGEDEPS) ++ $(LINKDEPS) lib/bloom-hiswalk-t.o tap/basic.o $(STORAGELIBS) $(LIBS) ++ + lib/hash.t: lib/hash-t.o tap/basic.o $(LIBINN) + $(LINK) lib/hash-t.o tap/basic.o $(LIBINN) + +diff --git tests/TESTS tests/TESTS +index b995af703..8cbc076ab 100644 +--- tests/TESTS ++++ tests/TESTS +@@ -7,6 +7,8 @@ innd/artparse + innd/chan + lib/artnumber + lib/asprintf ++lib/bloom ++lib/bloom-hiswalk + lib/buffer + lib/canlock + lib/concat +diff --git tests/lib/bloom-hiswalk-t.c tests/lib/bloom-hiswalk-t.c +new file mode 100644 +index 000000000..f6244de1e +--- /dev/null ++++ tests/lib/bloom-hiswalk-t.c +@@ -0,0 +1,169 @@ ++/* Integration test: HISwalk + bloom filter for expireover token cache. ++ * ++ * Creates a temporary history file with a mix of entries (some with tokens, ++ * some remembered-only), builds a bloom filter via HISwalk, and verifies ++ * that the bloom filter correctly identifies articles with tokens vs. ++ * remembered entries. */ ++ ++#include "portable/system.h" ++ ++#include ++#include ++ ++#include "inn/bloom.h" ++#include "inn/history.h" ++#include "inn/libinn.h" ++#include "inn/messages.h" ++#include "inn/storage.h" ++#include "tap/basic.h" ++ ++#define N_WITH_TOKEN 500 ++#define N_REMEMBERED 100 ++#define N_NOT_IN_HIST 200 ++ ++ ++/* ++** Generate a deterministic message-ID from an integer. ++*/ ++static char * ++make_msgid(unsigned long n) ++{ ++ char buf[64]; ++ snprintf(buf, sizeof(buf), "", n); ++ return xstrdup(buf); ++} ++ ++ ++/* ++** HISwalk callback: add entries with tokens to the bloom filter. ++** Same logic as build_bloom_cb in expireover.c. ++*/ ++static bool ++build_bloom_cb(void *cookie, const HASH *hash, ++ time_t arrived UNUSED, time_t posted UNUSED, ++ time_t expires UNUSED, const TOKEN *token) ++{ ++ if (token != NULL) ++ bloom_add(cookie, hash); ++ return true; ++} ++ ++ ++int ++main(void) ++{ ++ struct history *h; ++ struct bloom_filter *bloom; ++ char tmpdir[64]; ++ char histpath[128]; ++ TOKEN token; ++ unsigned long i; ++ unsigned long bloom_misses; ++ unsigned long false_negatives; ++ bool walk_ok; ++ ++ test_init(8); ++ ++ /* Create temporary directory for the history database. */ ++ strlcpy(tmpdir, "bloom-hiswalk-XXXXXX", sizeof(tmpdir)); ++ if (mkdtemp(tmpdir) == NULL) ++ sysbail("can't create temp directory"); ++ snprintf(histpath, sizeof(histpath), "%s/history", tmpdir); ++ ++ /* Create and populate the history database. */ ++ h = HISopen(histpath, "hisv6", HIS_CREAT | HIS_RDWR); ++ if (h == NULL) ++ bail("can't create history at %s", histpath); ++ ++ memset(&token, 0, sizeof(token)); ++ token.type = 1; ++ ++ /* Write entries with storage tokens. */ ++ for (i = 0; i < N_WITH_TOKEN; i++) { ++ char *msgid = make_msgid(i); ++ if (!HISwrite(h, msgid, (time_t) 1000000 + i, (time_t) 1000000 + i, ++ (time_t) 0, &token)) ++ bail("can't write history entry %lu: %s", i, HISerror(h)); ++ free(msgid); ++ } ++ ++ /* Write remembered entries (no token). */ ++ for (i = N_WITH_TOKEN; i < N_WITH_TOKEN + N_REMEMBERED; i++) { ++ char *msgid = make_msgid(i); ++ if (!HISremember(h, msgid, (time_t) 1000000 + i, ++ (time_t) 1000000 + i)) ++ bail("can't remember history entry %lu: %s", i, HISerror(h)); ++ free(msgid); ++ } ++ ++ HISsync(h); ++ HISclose(h); ++ ok(1, true); /* history created and populated */ ++ ++ /* Reopen read-only (as expireover does). */ ++ h = HISopen(histpath, "hisv6", HIS_RDONLY); ++ if (h == NULL) ++ bail("can't reopen history at %s", histpath); ++ ok(2, true); /* history reopened */ ++ ++ /* Build the bloom filter via HISwalk. */ ++ bloom = bloom_create(N_WITH_TOKEN + N_REMEMBERED, 10000); ++ walk_ok = HISwalk(h, NULL, bloom, build_bloom_cb); ++ ok(3, walk_ok); /* HISwalk succeeded */ ++ ok(4, bloom_count(bloom) == N_WITH_TOKEN); /* only token entries added */ ++ ++ /* Verify: all token entries should be bloom hits. */ ++ false_negatives = 0; ++ for (i = 0; i < N_WITH_TOKEN; i++) { ++ char *msgid = make_msgid(i); ++ HASH hash = HashMessageID(msgid); ++ if (!bloom_check(bloom, &hash)) ++ false_negatives++; ++ free(msgid); ++ } ++ ok(5, false_negatives == 0); /* no false negatives for token entries */ ++ if (false_negatives > 0) ++ diag("false negatives: %lu out of %d", false_negatives, N_WITH_TOKEN); ++ ++ /* Verify: remembered entries should NOT be in the bloom filter. ++ * (Some may be false positives, but most should miss.) */ ++ bloom_misses = 0; ++ for (i = N_WITH_TOKEN; i < N_WITH_TOKEN + N_REMEMBERED; i++) { ++ char *msgid = make_msgid(i); ++ HASH hash = HashMessageID(msgid); ++ if (!bloom_check(bloom, &hash)) ++ bloom_misses++; ++ free(msgid); ++ } ++ ok(6, bloom_misses > N_REMEMBERED * 9 / 10); /* >90% should miss */ ++ diag("remembered entries: %lu/%d were bloom misses (expected most)", ++ bloom_misses, N_REMEMBERED); ++ ++ /* Verify: entries not in history at all should mostly miss. */ ++ bloom_misses = 0; ++ for (i = N_WITH_TOKEN + N_REMEMBERED; ++ i < N_WITH_TOKEN + N_REMEMBERED + N_NOT_IN_HIST; i++) { ++ char *msgid = make_msgid(i); ++ HASH hash = HashMessageID(msgid); ++ if (!bloom_check(bloom, &hash)) ++ bloom_misses++; ++ free(msgid); ++ } ++ ok(7, bloom_misses > N_NOT_IN_HIST * 9 / 10); /* >90% should miss */ ++ diag("not-in-history entries: %lu/%d were bloom misses (expected most)", ++ bloom_misses, N_NOT_IN_HIST); ++ ++ bloom_free(bloom); ++ HISclose(h); ++ ++ /* Cleanup temp directory. */ ++ { ++ char cmd[128]; ++ snprintf(cmd, sizeof(cmd), "/bin/rm -rf %s", tmpdir); ++ if (system(cmd) < 0) ++ sysdiag("can't clean up %s", tmpdir); ++ } ++ ok(8, true); /* cleanup */ ++ ++ return 0; ++} +diff --git tests/lib/bloom-t.c tests/lib/bloom-t.c +new file mode 100644 +index 000000000..ef3b62bd1 +--- /dev/null ++++ tests/lib/bloom-t.c +@@ -0,0 +1,128 @@ ++/* Test suite for lib/bloom.c. */ ++ ++#include "portable/system.h" ++ ++#include ++ ++#include "inn/bloom.h" ++#include "inn/libinn.h" ++#include "tap/basic.h" ++ ++ ++/* ++** Generate a deterministic HASH from an integer for testing. ++*/ ++static HASH ++make_hash(unsigned long n) ++{ ++ char buf[64]; ++ snprintf(buf, sizeof(buf), "", n); ++ return HashMessageID(buf); ++} ++ ++ ++int ++main(void) ++{ ++ struct bloom_filter *bf; ++ HASH h1, h2, h3; ++ unsigned long i; ++ unsigned long false_positives; ++ unsigned long n_check; ++ ++ test_init(20); ++ ++ /* Basic creation. */ ++ bf = bloom_create(1000, 10000); ++ ok(1, bf != NULL); ++ ok(2, bloom_bits(bf) >= 1000 * 20); /* 0.01% FP needs ~20 bits/entry */ ++ ok(3, bloom_nhash(bf) == 14); ++ ok(4, bloom_count(bf) == 0); ++ ++ /* Add and check: true positives. */ ++ h1 = make_hash(1); ++ h2 = make_hash(2); ++ h3 = make_hash(3); ++ bloom_add(bf, &h1); ++ bloom_add(bf, &h2); ++ ok(5, bloom_count(bf) == 2); ++ ok(6, bloom_check(bf, &h1)); ++ ok(7, bloom_check(bf, &h2)); ++ ++ /* True negative. */ ++ ok(8, !bloom_check(bf, &h3)); ++ ++ bloom_free(bf); ++ ++ /* Larger test: verify false positive rate. ++ * Add 10,000 items, then check 100,000 items that were NOT added. ++ * At 0.01% target FP rate, we expect ~10 false positives out of ++ * 100,000 checks. Allow up to 50 (0.05%) to account for variance. */ ++ bf = bloom_create(10000, 10000); ++ ok(9, bf != NULL); ++ ++ for (i = 0; i < 10000; i++) { ++ HASH h = make_hash(i); ++ bloom_add(bf, &h); ++ } ++ ok(10, bloom_count(bf) == 10000); ++ ++ /* Verify all added items are found (no false negatives). */ ++ for (i = 0; i < 10000; i++) { ++ HASH h = make_hash(i); ++ if (!bloom_check(bf, &h)) { ++ ok(11, false); ++ diag("false negative at i=%lu", i); ++ goto fp_test; ++ } ++ } ++ ok(11, true); ++ ++fp_test: ++ /* Count false positives from items never added. */ ++ false_positives = 0; ++ n_check = 100000; ++ for (i = 10000; i < 10000 + n_check; i++) { ++ HASH h = make_hash(i); ++ if (bloom_check(bf, &h)) ++ false_positives++; ++ } ++ ok(12, false_positives <= 50); ++ if (false_positives > 50) ++ diag("false positives: %lu out of %lu (expected <= 50)", ++ false_positives, n_check); ++ else ++ diag("false positives: %lu out of %lu (%.4f%%)", ++ false_positives, n_check, ++ 100.0 * (double) false_positives / (double) n_check); ++ ++ bloom_free(bf); ++ ++ /* Test with different FP rate parameters. */ ++ bf = bloom_create(1000, 100); /* 1% FP rate */ ++ ok(13, bf != NULL); ++ ok(14, bloom_nhash(bf) == 7); /* k=7 for 1% FP */ ++ bloom_free(bf); ++ ++ bf = bloom_create(1000, 1000); /* 0.1% FP rate */ ++ ok(15, bf != NULL); ++ ok(16, bloom_nhash(bf) == 10); /* k=10 for 0.1% FP */ ++ bloom_free(bf); ++ ++ /* Edge case: very small filter. */ ++ bf = bloom_create(1, 10000); ++ ok(17, bf != NULL); ++ ok(18, bloom_bits(bf) >= 64); /* minimum size */ ++ bloom_free(bf); ++ ++ /* Test with max nhash (fp_inv >= 10000000, nhash=24). ++ * Exercises the full positions array to catch overflow. */ ++ bf = bloom_create(100, 10000000); ++ ok(19, bf != NULL); ++ h1 = make_hash(42); ++ bloom_add(bf, &h1); ++ ok(20, bloom_check(bf, &h1)); ++ bloom_free(bf); ++ ++ return 0; ++} diff --git a/news/inn-current/files/pr340.patch b/news/inn-current/files/pr340.patch new file mode 100644 index 000000000000..ae02a303fef0 --- /dev/null +++ b/news/inn-current/files/pr340.patch @@ -0,0 +1,2700 @@ +diff --git MANIFEST MANIFEST +--- MANIFEST 2026-05-16 12:44:56.907451000 -0700 ++++ MANIFEST 2026-05-16 12:45:04.832431000 -0700 +@@ -442,6 +442,7 @@ + include/inn/sequence.h Header file for sequence space arithmetic + include/inn/storage.h Header file for storage API + include/inn/timer.h Header file for generic timers ++include/inn/tombstone.h Header file for cancel tombstone log + include/inn/tst.h Header file for ternary search tries + include/inn/utility.h Header file for utility functions + include/inn/vector.h Header file for vectors of strings +@@ -810,6 +811,7 @@ + storage/timehash/method.config buildconfig definition + storage/timehash/timehash.c timehash storage routines + storage/timehash/timehash.h Header for timehash ++storage/tombstone.c Cancel tombstone log helpers + storage/tradindexed tradindexed overview method (Directory) + storage/tradindexed/ovmethod.config buildconfig definition + storage/tradindexed/ovmethod.mk Make rules for tradindexed overview +@@ -936,6 +938,10 @@ + tests/data/upgrade/sasl.conf Obsolete sasl.conf config file + tests/docs Test suite for documentation (Directory) + tests/docs/pod.t.in Tests for POD formatting ++tests/expire Test suite for expire (Directory) ++tests/expire/tombstone-e2e.t End-to-end tests for tombstone log ++tests/expire/tombstone-hisexpire-t.c HISexpire integration test for tombstone ++tests/expire/tombstone-t.c Tests for tombstone library + tests/innd Test suite for innd (Directory) + tests/innd/artparse-t.c Tests for ARTparse in innd + tests/innd/chan-t.c Tests for CHAN functions in innd +@@ -1008,6 +1014,7 @@ + tests/runtests.c The test suite driver program + tests/storage Test suite for storage (Directory) + tests/storage/archive.t Tests for backends/archive ++tests/storage/cancel-tombstone-t.c Tests for SMcanceltombstone + tests/storage/makehistory.t Tests for expire/makehistory + tests/storage/sm.t Tests for frontends/sm + tests/tap Helper scripts for TAP (Directory) +diff --git doc/pod/expire.pod doc/pod/expire.pod +--- doc/pod/expire.pod 2026-05-16 12:44:56.941582000 -0700 ++++ doc/pod/expire.pod 2026-05-16 12:45:04.832751000 -0700 +@@ -175,6 +175,18 @@ + + =back + ++=head1 TOMBSTONE LOG ++ ++When I is enabled in F, B consumes ++the per-cycle deletion log produced by B, B, and ++B (F/expireover.tombstone> and ++F/cancels.tombstone>) so it can drop history entries for ++those articles without doing a per-article C ++syscall. An empty tombstone is treated as "no cancels this cycle" ++and the slow scan is skipped entirely. See inn.conf(5) under ++I for the file lifecycle, locking model, and ++recovery story. ++ + =head1 HISTORY + + Written by Rich $alz for InterNetNews. Converted to +diff --git doc/pod/expireover.pod doc/pod/expireover.pod +--- doc/pod/expireover.pod 2026-05-16 12:44:56.945074000 -0700 ++++ doc/pod/expireover.pod 2026-05-16 12:45:04.833030000 -0700 +@@ -21,6 +21,14 @@ + that have already been removed by some other process, and B<-e>, B<-k>, + B<-N>, B<-p>, B<-q>, B<-w>, and B<-z> are all ignored. + ++When I is enabled in F, B ++appends each cancelled token to F/expireover.tombstone.NEW> ++under an exclusive POSIX lock and atomically renames the file into ++place on a clean run. In delayrm mode (B<-z>), the rename is ++performed by B after B succeeds. The next ++B run consumes this log to skip per-article storage existence ++checks. See inn.conf(5) under I. ++ + When I is set, the default behavior of B is + to remove the article from the spool once it expires out of all of the + newsgroups to which it was crossposted. The article is, however, removed +diff --git doc/pod/inn.conf.pod doc/pod/inn.conf.pod +--- doc/pod/inn.conf.pod 2026-05-16 12:44:56.943722000 -0700 ++++ doc/pod/inn.conf.pod 2026-05-16 12:45:04.843432000 -0700 +@@ -652,6 +652,72 @@ + Moreover, the deprecated C and C header fields, already present + in the standard overview fields as metadata items, cannot be added. + ++=item I ++ ++Whether INN tools record cancellation tombstones so a subsequent ++B run can skip per-article storage existence checks. When ++enabled (and I is also true), two log files in ++I capture every cancellation: ++ ++=over 4 ++ ++=item F/expireover.tombstone> ++ ++Written by B after each successful B in ++group-based expiry, atomically renamed into place on a clean run. ++When B runs with C<-z> (delayed removal), the ++B calls are deferred to B via B; ++B writes the entries up front and B performs ++the atomic rename after B succeeds, so the same speedup ++applies to delayrm setups. ++ ++=item F/cancels.tombstone> ++ ++Appended continuously by B when it processes cancel control ++messages, and by B for manual cancellations. Appenders take ++a shared fcntl POSIX lock; B snapshots the file by atomic ++rename to F under an exclusive fcntl ++lock and unlinks it after a successful consume. Append atomicity ++relies on POSIX guaranteeing that each C to a regular ++file opened with C is atomic with respect to other ++writers; this holds for any size of single C on local ++filesystems. Cross-client atomicity over NFS is not guaranteed: ++if I is on NFS, lines from concurrent writers on different ++clients can in theory interleave. In practice INN's cancel ++sources (B on a single host plus occasional B on the ++same host) write from one client. ++ ++=back ++ ++The next B invocation loads both files into a single hashset ++and treats every article in either log as already gone, avoiding an ++C call per history entry. For storage methods ++where the stat is a file-system call (tradspool, timehash) this turns ++a billion C calls into a few thousand hash lookups; for ++storage methods that self-expire (CNFS) the stat call remains because ++articles can vanish through wrap-around without going through ++B. ++ ++In normal operation every cancellation path participates in ++tombstone tracking, so all articles removed from the spool are ++recorded. Residual orphans can only accumulate from events outside ++the tracked paths: a process crash in the narrow window between ++B and the tombstone append, manual filesystem-level ++deletes that bypass B, or filesystem corruption. When such ++orphans do appear they are harmless (B returns "no such ++article" to readers that hit them) and exist only as small history ++entries. No regular reconciliation cadence is needed. If an ++operator suspects orphan accumulation after admin intervention or a ++storage incident, B can be re-run with this option disabled ++in F to perform an exhaustive C ++scan; this is an exceptional operation, not a scheduled one. ++ ++Footprint: ~38 bytes per entry on disk, ~50 bytes in expire's hash ++table. 1M cancels per run = ~38 MB tombstone, ~50 MB hash. ++ ++This is a boolean value and the default is false; sites should opt in ++after validating the option's behaviour against their workload. ++ + =item I + + Whether to enable newsgroup-based expiry. If set to false, article expiry +@@ -910,10 +976,50 @@ + as present in response to an NNTP command (HDR, LISTGROUP, NEWNEWS, OVER, + XPAT). The primary use of this setting is to prevent B from returning + information about articles which are no longer present on the server but which +-still have overview data available. Checking the existence of articles before +-returning overview information slows down the overview commands, but reduces +-the number of "article is missing" errors seen by the client. This is a +-boolean value and the default is true. ++still have overview data available. Checking existence with an unconditional ++C slows down the overview commands; with I also ++enabled (see below) the check uses an in-memory hash lookup instead and is ++cheaper than disabling the check on tradspool/timehash/timecaf backends. ++The trade-off remains: enabling this reduces the number of "article is ++missing" errors seen by the client. This is a boolean value and the default ++is true. ++ ++When I is also enabled, B consults the ++F/cancels.tombstone> log on the article-existence check ++path: a token recorded as cancelled is reported as gone without an ++C call, and a token absent from the tombstone is ++trusted to still exist (skipping the syscall) for storage methods ++that do not self-expire. Self-expiring backends (CNFS) still go ++through C because cyclic-buffer wrap-around bypasses ++the tombstone. The tombstone is loaded lazily on first use per ++connection and refreshed by C on each call; the parsed ++hashset is rebuilt only when the file's mtime or size changes. ++Statting the same path is dentry-cache resident and far cheaper ++than the per-article syscalls the fast path elides, so cancellations ++recorded by other processes become visible to long-lived ++connections on the next existence check. ++ ++Each B connection holds an independent copy of the parsed ++hashset, costing roughly 50 bytes per cancel; sites with very large ++cancel volumes between B runs and many concurrent readers ++should size memory accordingly. Loss relative to the unconditional ++C path is bounded to out-of-band events the tombstone ++cannot see (manual filesystem deletes that bypass B, ++filesystem corruption); admin-initiated B and B ++cancels are tracked. ++ ++Important: on non-self-expiring backends the fast path treats a ++tombstone-miss as proof that the article is still on disk, without ++verifying. Admins must remove articles via B (which records ++the cancel in the tombstone), not by direct C on the spool, or ++readers will be told a deleted article still exists until the next ++B reconciles overview. ++ ++Note that the fast path is also gated on I; both ++must be true to take effect. The B C ++counter reflects only the slow C path, so the syslog ++field will appear smaller when the fast path is doing most of the ++work. + + You may also want to see the I parameter in readers.conf(5) + which controls the computing of the estimated article count returned in NNTP +diff --git doc/pod/sm.pod doc/pod/sm.pod +--- doc/pod/sm.pod 2026-05-16 12:44:56.942385000 -0700 ++++ doc/pod/sm.pod 2026-05-16 12:45:04.834064000 -0700 +@@ -51,6 +51,13 @@ + be retrievable by any part of INN. It's equivalent to C + except it takes a storage API token instead of a message-ID. + ++When the I setting in F is true, B ++also appends the cancelled token to F/cancels.tombstone> ++so a later B run can drop the corresponding history entry ++without a per-article storage check. Append failures are logged ++but do not affect the cancellation itself. See inn.conf(5) under ++I for the full mechanism. ++ + =item B<-H> + + Retrieve only the headers of the article rather than the entire article. +diff --git expire/expire.c expire/expire.c +--- expire/expire.c 2026-05-16 12:44:56.912885000 -0700 ++++ expire/expire.c 2026-05-16 12:45:04.834504000 -0700 +@@ -6,10 +6,13 @@ + + #include + #include ++#include + #include + #include + #include ++#include + ++#include "inn/hashtab.h" + #include "inn/history.h" + #include "inn/innconf.h" + #include "inn/inndcomm.h" +@@ -18,6 +21,7 @@ + #include "inn/newsuser.h" + #include "inn/paths.h" + #include "inn/storage.h" ++#include "inn/tombstone.h" + + + typedef struct _EXPIRECLASS { +@@ -48,6 +52,7 @@ + static int EXPverbose; + static long EXPprocessed; + static long EXPunlinked; ++static long EXPtombstoned; + static long EXPallgone; + static long EXPstillhere; + static struct history *History; +@@ -371,23 +376,208 @@ + } + + /* ++** Load both tombstone logs into a hashset: ++** - ${pathdb}/expireover.tombstone : written by expireover/expirerm ++** (atomic .NEW -> final rename); unlinked after a successful ++** expire run ++** - ${pathdb}/cancels.tombstone : appended continuously by innd ++** and sm for cancels outside the expireover pipeline; renamed ++** to .processing at load time and unlinked after a successful ++** expire run (rename-and-process avoids the read/truncate race ++** that drops cancels arriving during HISexpire) ++** ++** At startup, also recovers any leftover .processing snapshot from a ++** previous expire run that crashed before unlinking. ++** ++** consuming=true means this is a real run that will replace the ++** history file; we rename cancels.tombstone to .processing under ++** the consumer's lock so concurrent appenders cannot lose cancels ++** written between our read and the eventual unlink. consuming= ++** false (dry-run, tracing, or alternate-output expire) skips the ++** rename and reads the live file directly: no consume happens, so ++** there is nothing to atomically detach. This avoids both the ++** TOCTOU race in restoring the snapshot and the operator surprise ++** of cancels.tombstone disappearing for the duration of a -x run. ++** ++** Returns NULL if no tombstone could be loaded (no file present or ++** all reads failed); that is not an error. Expire just falls back ++** to the per-article SMretrieve check. *out_expireover_path and ++** *out_cancels_snapshot are set to the file paths (caller frees and ++** unlinks after a successful run); both are NULL when consuming is ++** false. ++*/ ++static struct hash * ++EXPloadtombstone(bool consuming, char **out_expireover_path, ++ char **out_cancels_snapshot) ++{ ++ char *expireover_path = NULL; ++ char *cancels_path = NULL; ++ char *cancels_snapshot = NULL; ++ char *leftover = NULL; ++ struct hash *h = NULL; ++ struct stat sb; ++ unsigned long n_expireover = 0; ++ unsigned long n_cancels = 0; ++ unsigned long n_leftover = 0; ++ bool expireover_present = false; ++ bool leftover_present = false; ++ bool cancels_present = false; ++ ++ /* Default outputs to NULL; only set on the success-return path ++ below. This avoids double-frees if a future maintainer adds an ++ early return between here and that single point. */ ++ *out_expireover_path = NULL; ++ *out_cancels_snapshot = NULL; ++ ++ expireover_path = concatpath(innconf->pathdb, "expireover.tombstone"); ++ cancels_path = concatpath(innconf->pathdb, "cancels.tombstone"); ++ leftover = concat(cancels_path, ".processing", (char *) 0); ++ ++ /* Probe each file's presence (independent of content) so we can ++ distinguish "tombstone subsystem is actively tracking but had ++ no cancels this cycle" (trust the empty hashset, skip ++ SMretrieve for everything) from "no files at all" (subsystem ++ not in use, fall back to slow path). Without this, a cycle ++ with zero cancels would needlessly run SMretrieve on every ++ history entry. */ ++ if (stat(expireover_path, &sb) == 0) ++ expireover_present = true; ++ if (stat(leftover, &sb) == 0) ++ leftover_present = true; ++ if (stat(cancels_path, &sb) == 0) ++ cancels_present = true; ++ ++ /* Initial size: tokens are typically ~38 bytes per line; we let the ++ hash expand if we underestimate. 4096 covers the common case. */ ++ h = tombstone_hash_create(4096); ++ ++ /* Recover a leftover .processing from a previous run that crashed ++ between rename and unlink. */ ++ n_leftover = tombstone_read(h, leftover, NULL); ++ ++ /* On a real run, atomically snapshot the live cancels.tombstone, ++ then read it. On dry-run / tracing / alt-output, just read ++ the live file directly: no consume happens, so there is no ++ need to detach it (and detach-then-restore has a TOCTOU race ++ with concurrent appenders). */ ++ if (consuming) { ++ cancels_snapshot = tombstone_rename_for_processing(cancels_path); ++ if (cancels_snapshot != NULL) ++ n_cancels = tombstone_read(h, cancels_snapshot, NULL); ++ } else { ++ n_cancels = tombstone_read(h, cancels_path, NULL); ++ } ++ ++ /* Read the expireover-side log. */ ++ n_expireover = tombstone_read(h, expireover_path, NULL); ++ ++ if (EXPverbose) ++ printf("Loaded %lu + %lu + %lu tombstone entries (%lu unique)\n", ++ n_expireover, n_cancels, n_leftover, hash_count(h)); ++ ++ /* Reconcile the leftover with the active snapshot. Only relevant ++ on a real consuming run; dry-run leaves the leftover where it ++ is for the next consuming run to pick up. ++ (a) leftover empty: just free the path string ++ (b) leftover non-empty + no live snapshot: promote leftover to ++ be the snapshot so the caller's single unlink covers it ++ (c) leftover non-empty + live snapshot: unlink the leftover now ++ (its contents are already in the hashset); the snapshot ++ remains and is unlinked on success */ ++ if (consuming) { ++ if (n_leftover > 0 && cancels_snapshot == NULL) { ++ cancels_snapshot = leftover; ++ leftover = NULL; ++ } else if (n_leftover > 0) { ++ if (unlink(leftover) < 0) ++ syswarn("can't unlink %s", leftover); ++ } else if (leftover_present && cancels_snapshot == NULL) { ++ /* Leftover existed but was empty. Treat it as the ++ snapshot so the caller cleans it up after a ++ successful run. */ ++ cancels_snapshot = leftover; ++ leftover = NULL; ++ } else if (leftover_present) { ++ /* Both leftover and live snapshot existed; leftover ++ empty. Unlink it now; snapshot will be cleaned up ++ on success. */ ++ if (unlink(leftover) < 0) ++ syswarn("can't unlink %s", leftover); ++ } ++ } ++ free(leftover); ++ leftover = NULL; ++ ++ /* Decide between "fall back to slow path" (return NULL) and ++ "trust the (possibly empty) tombstone". A file that exists ++ (regardless of content) is taken as evidence the tombstone ++ subsystem is active for this site: innd/sm/expireover are ++ writing it. An empty active tombstone correctly says "nothing ++ was cancelled in the last cycle"; trust it. In dry-run mode ++ cancels.tombstone is not renamed, so cancels_snapshot stays ++ NULL; use the cancels_present probe instead. */ ++ if (!expireover_present && !leftover_present && !cancels_present ++ && cancels_snapshot == NULL) { ++ hash_free(h); ++ free(expireover_path); ++ free(cancels_path); ++ return NULL; ++ } ++ ++ /* Single success-return: hand both paths to the caller for ++ cleanup after a successful run. */ ++ free(cancels_path); ++ *out_expireover_path = expireover_path; ++ *out_cancels_snapshot = cancels_snapshot; ++ return h; ++} ++ ++ ++/* + ** Do the work of expiring one line. + ** Returns true when the article should be kept for the time being. + */ + static bool +-EXPdoline(void *cookie UNUSED, time_t arrived, time_t posted, time_t expires, ++EXPdoline(void *cookie, time_t arrived, time_t posted, time_t expires, + TOKEN *token) + { ++ struct hash *tombstone = (struct hash *) cookie; + time_t when; + bool HasSelfexpire = false; + bool Selfexpired = false; ++ bool selfexpiring; + ARTHANDLE *article; + enum KR kr; + bool r; + +- if (innconf->groupbaseexpiry || SMprobe(SELFEXPIRE, token, NULL)) { +- if ((article = SMretrieve(*token, RETR_STAT)) == (ARTHANDLE *) NULL) { ++ /* Tombstone fast path: if expireover already cancelled this article, ++ drop the history entry without doing any storage I/O. Bump ++ EXPunlinked too so the news.daily summary's "Articles dropped" ++ count matches the slow path's accounting (EXPremove also bumps ++ it). */ ++ if (tombstone != NULL && hash_lookup(tombstone, token) != NULL) { ++ EXPprocessed++; ++ if (EXPverbose > 3) ++ printf("%s (tombstoned by expireover)\n", TokenToText(*token)); ++ EXPallgone++; ++ EXPunlinked++; ++ EXPtombstoned++; ++ return false; ++ } ++ ++ selfexpiring = SMprobe(SELFEXPIRE, token, NULL); ++ ++ if (innconf->groupbaseexpiry || selfexpiring) { ++ if (tombstone != NULL && !selfexpiring) { ++ /* Backend does not self-expire and the tombstone log is ++ complete: not in the log means the article still exists. ++ Skip the SMretrieve to avoid a per-article syscall (the ++ main speedup for tradspool / timehash / timecaf). */ + HasSelfexpire = true; ++ Selfexpired = false; ++ } else if ((article = SMretrieve(*token, RETR_STAT)) ++ == (ARTHANDLE *) NULL) { ++ HasSelfexpire = true; + Selfexpired = true; + } else { + /* The article is still alive, free it. */ +@@ -470,6 +660,8 @@ + printf("Entries expired %8ld\n", EXPallgone); + if (!innconf->groupbaseexpiry) + printf("Articles dropped %8ld\n", EXPunlinked); ++ if (innconf->expiretombstone && innconf->groupbaseexpiry) ++ printf("Tombstone hits %8ld\n", EXPtombstoned); + } + + /* Append statistics to a summary file */ +@@ -536,6 +728,16 @@ + if (!innconf_read(NULL)) + exit(1); + ++ /* Warn about expiretombstone-without-groupbaseexpiry: in that ++ configuration there is no consumable tombstone since OVEXPremove ++ (which writes one of the two logs) is never called. The ++ cancels.tombstone written by innd/sm could still be loaded, but ++ only with groupbaseexpiry to make the result coherent with ++ expire's overall semantics. */ ++ if (innconf->expiretombstone && !innconf->groupbaseexpiry) ++ notice("expiretombstone has no effect when groupbaseexpiry" ++ " is false"); ++ + HistoryText = concatpath(innconf->pathdb, INN_PATH_HISTORY); + + umask(NEWSUMASK); +@@ -694,9 +896,47 @@ + CleanupAndExit(Server, false, 1); + } + +- Bad = HISexpire(History, NHistory, EXPreason, Writing, NULL, EXPremember, +- EXPdoline) +- == false; ++ /* Consume the tombstone logs produced since the last expire run. ++ Lets EXPdoline drop history entries for articles cancelled by ++ expireover, expirerm, innd, or sm without per-article SMretrieve ++ calls. Gated on groupbaseexpiry as well: if the admin flipped ++ that off since the last expireover, a stale tombstone written ++ under the old config could drop history entries for articles ++ still alive. */ ++ { ++ bool consuming = Writing && !EXPtracing && NHistory == NULL; ++ char *expireover_path = NULL; ++ char *cancels_snapshot = NULL; ++ struct hash *tombstone = NULL; ++ ++ if (innconf->expiretombstone && innconf->groupbaseexpiry) ++ tombstone = EXPloadtombstone(consuming, &expireover_path, ++ &cancels_snapshot); ++ ++ Bad = HISexpire(History, NHistory, EXPreason, Writing, tombstone, ++ EXPremember, EXPdoline) ++ == false; ++ ++ if (tombstone != NULL) { ++ hash_free(tombstone); ++ /* On a successful real run, unlink both consumed ++ snapshots. In dry-run / tracing / alt-output, the ++ loader skipped the rename and did not return a ++ snapshot path, so there is nothing to clean up. */ ++ if (!Bad && consuming) { ++ if (expireover_path != NULL) { ++ if (unlink(expireover_path) < 0 && errno != ENOENT) ++ syswarn("can't unlink %s", expireover_path); ++ } ++ if (cancels_snapshot != NULL) { ++ if (unlink(cancels_snapshot) < 0 && errno != ENOENT) ++ syswarn("can't unlink %s", cancels_snapshot); ++ } ++ } ++ } ++ free(expireover_path); ++ free(cancels_snapshot); ++ } + + if (UnlinkFile && EXPunlinkfile == NULL) + /* Got -z but file was closed; oops. */ +diff --git expire/expireover.c expire/expireover.c +--- expire/expireover.c 2026-05-16 12:44:56.912753000 -0700 ++++ expire/expireover.c 2026-05-16 12:46:12.471407000 -0700 +@@ -10,13 +10,16 @@ + #include "portable/system.h" + + #include ++#include + #include ++#include ++#include + #include + #include ++#include + +-#include +- + #include "inn/bloom.h" ++#include "inn/buffer.h" + #include "inn/history.h" + #include "inn/innconf.h" + #include "inn/libinn.h" +@@ -27,6 +30,11 @@ + #include "inn/qio.h" + #include "inn/storage.h" + ++/* OVtombstonefile is an internal of the storage library (declared in ++ * storage/ovinterface.h), but expireover owns its lifecycle, so we ++ * declare it here rather than exporting it via the public ov.h header. */ ++extern FILE *OVtombstonefile; ++ + static const char usage[] = "\ + Usage: expireover [-ekNpqs] [-f file] [-w offset] [-z rmfile] [-Z lowmarkfile]\n"; + +@@ -65,6 +73,94 @@ + } + + ++/* ++** Verify each line of an in-memory tombstone buffer via SMretrieve and ++** append survivors to kept. An entry survives if SMretrieve confirms ++** the article is gone (SMERR_NOENT) or returns a non-NOENT error ++** (SMERR_UNINIT, EIO, etc., where we cannot tell whether the article ++** is alive); a transient failure must not silently discard a valid ++** cancel record. Live articles (SMretrieve returns non-NULL) are ++** dropped: we must not re-tombstone an article that is still on disk. ++*/ ++static void ++verify_tombstone_lines(char *raw, ssize_t got, struct buffer *kept) ++{ ++ char *entry, *save_p; ++ ++ if (got <= 0) ++ return; ++ raw[got] = '\0'; ++ entry = strtok_r(raw, "\n", &save_p); ++ while (entry != NULL) { ++ size_t len = strlen(entry); ++ if (len > 0 && entry[len - 1] == '\r') ++ entry[--len] = '\0'; ++ if (len > 0 && IsToken(entry)) { ++ TOKEN t = TextToToken(entry); ++ ARTHANDLE *art = SMretrieve(t, RETR_STAT); ++ if (art != NULL) { ++ /* Article still on disk; drop from tombstone to ++ avoid orphaning history for it. */ ++ SMfreearticle(art); ++ } else { ++ /* Either confirmed gone (SMERR_NOENT) or transient ++ error (SMERR_UNINIT, EIO, etc.). Keep the entry ++ either way: confirmed-gone is the normal case, ++ and on transient error we cannot tell whether ++ the article is alive, so preserve the record so ++ the next run can re-evaluate. Silently dropping ++ on transient error would leak cancels across a ++ storage outage. */ ++ buffer_append(kept, entry, len); ++ buffer_append(kept, "\n", 1); ++ } ++ } ++ entry = strtok_r(NULL, "\n", &save_p); ++ } ++} ++ ++ ++/* ++** Read an entire file into a freshly allocated buffer. Used for ++** leftover-recovery of expireover.tombstone (from a cycle where ++** expirerm finalized but expire never consumed) and the .NEW ++** leftover from a crashed prior run. Returns NULL on missing or ++** empty file (sets *out_size to 0); returns a malloc'd buffer with ++** the file content otherwise (caller frees). ++*/ ++static char * ++slurp_tombstone(const char *path, ssize_t *out_size) ++{ ++ int fd; ++ struct stat sb; ++ char *raw; ++ ssize_t got = 0, n; ++ ++ *out_size = 0; ++ fd = open(path, O_RDONLY); ++ if (fd < 0) ++ return NULL; ++ if (fstat(fd, &sb) < 0 || sb.st_size <= 0) { ++ close(fd); ++ return NULL; ++ } ++ raw = xmalloc(sb.st_size + 1); ++ while (got < sb.st_size) { ++ n = read(fd, raw + got, sb.st_size - got); ++ if (n <= 0) ++ break; ++ got += n; ++ } ++ close(fd); ++ if (got <= 0) { ++ free(raw); ++ return NULL; ++ } ++ *out_size = got; ++ return raw; ++} ++ ++ + int + main(int argc, char *argv[]) + { +@@ -76,9 +172,12 @@ + char *active_path = NULL; + char *lowmark_path = NULL; + char *path; ++ char *tombstone_path = NULL; ++ char *tombstone_path_new = NULL; + FILE *lowmark = NULL; + bool purge_deleted = false; + bool always_stat = false; ++ bool tombstone_clean = true; + struct history *history; + struct bloom_filter *bloom = NULL; + struct bloom_filter *null_bloom = NULL; +@@ -144,6 +243,14 @@ + if (!innconf_read(NULL)) + exit(1); + ++ /* Warn about expiretombstone-without-groupbaseexpiry: in that ++ configuration OVEXPremove is never called so no entries are ++ ever appended. The setting is silently ignored. Operators ++ reaching for the speedup deserve to know it's not active. */ ++ if (innconf->expiretombstone && !innconf->groupbaseexpiry) ++ notice("expiretombstone has no effect when groupbaseexpiry" ++ " is false"); ++ + /* Change to the runasuser user and runasgroup group if necessary. */ + ensure_news_user_grp(true, true); + +@@ -240,6 +347,103 @@ + free(histpath); + } + ++ /* Open the tombstone log. OVEXPremove appends a line per article ++ it cancels (inline) or schedules for removal via the rm file ++ (delayrm). In delayrm mode the .NEW -> final rename is performed ++ by expirerm after fastrm succeeds (see expirerm.in); in inline ++ mode this process performs the rename below. Skipped when ++ groupbaseexpiry is false because OVEXPremove is not called in ++ that mode, and when the admin has disabled the feature via ++ expiretombstone. */ ++ if (innconf->expiretombstone && innconf->groupbaseexpiry) { ++ int fd; ++ ++ tombstone_path = concatpath(innconf->pathdb, "expireover.tombstone"); ++ tombstone_path_new = concat(tombstone_path, ".NEW", (char *) 0); ++ fd = open(tombstone_path_new, O_RDWR | O_CREAT, 0664); ++ if (fd < 0) { ++ syswarn("can't open tombstone log %s", tombstone_path_new); ++ } else if (!inn_lock_file(fd, INN_LOCK_WRITE, false)) { ++ if (errno == EAGAIN || errno == EACCES) { ++ warn("another expireover holds the tombstone log lock" ++ " on %s; disabling tombstone for this run", ++ tombstone_path_new); ++ } else { ++ syswarn("can't lock %s; disabling tombstone for this" ++ " run", ++ tombstone_path_new); ++ } ++ close(fd); ++ fd = -1; ++ } else { ++ struct buffer kept; ++ struct stat sb; ++ char *raw; ++ ssize_t got = 0; ++ ++ kept.data = NULL; ++ kept.size = kept.used = kept.left = 0; ++ ++ raw = slurp_tombstone(tombstone_path, &got); ++ if (raw != NULL) { ++ verify_tombstone_lines(raw, got, &kept); ++ free(raw); ++ if (unlink(tombstone_path) < 0 && errno != ENOENT) ++ syswarn("can't unlink leftover %s", tombstone_path); ++ } ++ ++ if (fstat(fd, &sb) == 0 && sb.st_size > 0) { ++ raw = xmalloc(sb.st_size + 1); ++ got = 0; ++ while (got < sb.st_size) { ++ ssize_t n = read(fd, raw + got, sb.st_size - got); ++ if (n <= 0) ++ break; ++ got += n; ++ } ++ verify_tombstone_lines(raw, got, &kept); ++ free(raw); ++ } ++ if (ftruncate(fd, 0) < 0) { ++ syswarn("can't truncate %s", tombstone_path_new); ++ close(fd); ++ fd = -1; ++ } else if (lseek(fd, 0, SEEK_SET) < 0) { ++ syswarn("can't seek %s", tombstone_path_new); ++ close(fd); ++ fd = -1; ++ } else { ++ OVtombstonefile = fdopen(fd, "w"); ++ if (OVtombstonefile == NULL) { ++ syswarn("can't fdopen %s", tombstone_path_new); ++ close(fd); ++ fd = -1; ++ } else { ++ if (fputs("# inn-tombstone v1\n", OVtombstonefile) ++ == EOF) ++ syswarn("can't write header to %s", ++ tombstone_path_new); ++ if (kept.left > 0) { ++ if (fwrite(kept.data + kept.used, 1, kept.left, ++ OVtombstonefile) ++ != kept.left ++ || fflush(OVtombstonefile) == EOF) ++ syswarn("can't write recovered leftover" ++ " to %s", ++ tombstone_path_new); ++ } ++ } ++ } ++ free(kept.data); ++ } ++ if (OVtombstonefile == NULL) { ++ free(tombstone_path); ++ free(tombstone_path_new); ++ tombstone_path = NULL; ++ tombstone_path_new = NULL; ++ } ++ } ++ + /* Loop through each line of the input file and process each group, + writing data to the lowmark file if desired. */ + line = QIOread(qp); +@@ -276,6 +480,46 @@ + if (lowmark != NULL) + if (fclose(lowmark) == EOF) + syswarn("can't close %s", lowmark_path); ++ ++ /* Finalize the tombstone log. In inline-cancel mode, rename .NEW ++ to its final name iff we completed a full pass without errors -- ++ a partial log would let expire incorrectly drop history entries ++ for articles that were not actually cancelled this run. ++ In delayrm mode, leave the .NEW alone: expirerm performs the ++ rename after fastrm has actually deleted the articles. If ++ expirerm fails or is skipped, the .NEW is wiped by the next ++ expireover, so a partial log can never be consumed by expire. ++ No fsync: the log is advisory; if a crash loses the buffered ++ tail, expire just runs the slow path. */ ++ if (OVtombstonefile != NULL) { ++ if (fclose(OVtombstonefile) == EOF) { ++ syswarn("can't finalize tombstone log %s", tombstone_path_new); ++ tombstone_clean = false; ++ } ++ if (signalled) ++ tombstone_clean = false; ++ if (ovge.delayrm) { ++ /* Leave .NEW for expirerm. If we were signalled or hit ++ an error, unlink it so expirerm doesn't promote a ++ truncated log. */ ++ if (!tombstone_clean) { ++ if (unlink(tombstone_path_new) < 0) ++ syswarn("can't unlink %s", tombstone_path_new); ++ } ++ } else if (tombstone_clean) { ++ if (rename(tombstone_path_new, tombstone_path) < 0) { ++ syswarn("can't rename %s to %s", tombstone_path_new, ++ tombstone_path); ++ if (unlink(tombstone_path_new) < 0) ++ syswarn("can't unlink %s", tombstone_path_new); ++ } ++ } else { ++ if (unlink(tombstone_path_new) < 0) ++ syswarn("can't unlink %s", tombstone_path_new); ++ } ++ free(tombstone_path); ++ free(tombstone_path_new); ++ } + + return 0; + } +diff --git expire/expirerm.in expire/expirerm.in +--- expire/expirerm.in 2026-05-16 12:44:56.912927000 -0700 ++++ expire/expirerm.in 2026-05-16 12:45:04.835303000 -0700 +@@ -16,17 +16,38 @@ + | eval ${MAIL} + exit 0 + fi +-if [ ! -f $1 ]; then ++if [ ! -f "$1" ]; then + echo "Expire called with no files to expire on $(hostname)" \ + | eval ${MAIL} + exit 0 + fi + + eval "cd ${SPOOL} \ +- && ${RMPROC} <$1 \ +- && mv $1 ${MOST_LOGS}/expire.list" +-if [ -f $1 ]; then ++ && ${RMPROC} <\"$1\" \ ++ && mv \"$1\" ${MOST_LOGS}/expire.list" ++if [ -f "$1" ]; then + echo "Expire had problems removing articles on $(hostname)" \ + | eval ${MAIL} + exit 1 ++fi ++ ++## If expireover wrote a tombstone log for delayed-removal mode, the ++## .NEW file holds the tokens it scheduled for cancellation. Now that ++## fastrm has succeeded, promote it to its final name so the next ++## expire run can consume it. ++## ++## A failed promotion is a degradation, not a hard failure: the ++## articles are already gone from disk, so news.daily can continue. ++## The next expire just falls back to its slow per-article ++## SMretrieve check for whatever was supposed to be in this batch. ++## Email the newsmaster but exit 0 so the surrounding pipeline keeps ++## going. ++TOMBSTONE_NEW="${PATHDB}/expireover.tombstone.NEW" ++TOMBSTONE="${PATHDB}/expireover.tombstone" ++if [ -f "${TOMBSTONE_NEW}" ]; then ++ mv "${TOMBSTONE_NEW}" "${TOMBSTONE}" || { ++ echo "Expire could not promote ${TOMBSTONE_NEW} on $(hostname);" \ ++ "next expire will run the slow path for this batch" \ ++ | eval ${MAIL} ++ } + fi +diff --git expire/makehistory.c expire/makehistory.c +--- expire/makehistory.c 2026-05-16 12:44:56.912967000 -0700 ++++ expire/makehistory.c 2026-05-16 12:45:04.835679000 -0700 +@@ -287,6 +287,15 @@ + arrived = (time_t) atoll(line); + expires = (time_t) atoll(p); + } ++ /* Validate before TextToToken: that function returns an ++ * all-zero TOKEN on malformed input rather than an error ++ * sentinel, so a bad line would silently insert a bogus ++ * overview entry. */ ++ if (!IsToken(q)) { ++ warn("sorted overview file %s has a malformed token at %d", ++ SortedTmpPath, count); ++ continue; ++ } + token = TextToToken(q); + if (OVadd(token, r, strlen(r), arrived, expires) == OVADDFAILED) { + if (OVctl(OVSPACE, (void *) &f) +diff --git frontends/sm.c frontends/sm.c +--- frontends/sm.c 2026-05-16 12:44:56.937891000 -0700 ++++ frontends/sm.c 2026-05-16 12:45:04.835987000 -0700 +@@ -10,6 +10,8 @@ + #include "inn/innconf.h" + #include "inn/libinn.h" + #include "inn/messages.h" ++#include "inn/newsuser.h" ++#include "inn/paths.h" + #include "inn/qio.h" + #include "inn/storage.h" + #include "inn/wire.h" +@@ -259,6 +261,10 @@ + warn("could not remove %s: %s", id, SMerrorstr); + return false; + } ++ /* Record the cancel so a later expire run can drop the history ++ * entry without an SMretrieve. Best-effort; no-op when ++ * expiretombstone is disabled. */ ++ SMcanceltombstone(token); + } else { + article = SMretrieve(token, options->header ? RETR_HEAD : RETR_ALL); + if (article == NULL) { +@@ -360,6 +366,17 @@ + if (!SMsetup(SM_RDWR, &value)) + die("cannot set up storage manager"); + } ++ ++ /* On the -r/-d path with the cancel tombstone enabled, drop to ++ * the news user/group so cancels.tombstone is created (or ++ * appended) with ownership consistent with innd's writes. ++ * Bypassed under the INN_TESTSUITE env var so the test harness ++ * can run sm without a real news user on the system. All other ++ * sm modes (-i, -c, -H, -R, -s, retrieval) preserve the ++ * historical behaviour of running as the invoking user. */ ++ if (options.delete && innconf->expiretombstone ++ && getenv(INN_ENV_TESTSUITE) == NULL) ++ ensure_news_user_grp(true, true); + if (!SMinit()) + die("cannot initialize storage manager: %s", SMerrorstr); + +diff --git include/inn/innconf.h include/inn/innconf.h +--- include/inn/innconf.h 2026-05-16 12:44:56.906670000 -0700 ++++ include/inn/innconf.h 2026-05-16 12:45:04.836314000 -0700 +@@ -77,6 +77,9 @@ + OVERVIEW.FMT */ + struct vector + *extraoverviewhidden; /* Extra overview fields silently generated */ ++ bool expiretombstone; /* Use expireover -> expire tombstone log ++ to skip per-article SMretrieve in ++ expire? */ + bool groupbaseexpiry; /* Do expiry by newsgroup? */ + bool mergetogroups; /* Refile articles from to.* into to */ + bool nfswriter; /* Use NFS writer functionality */ +diff --git include/inn/ov.h include/inn/ov.h +--- include/inn/ov.h 2026-05-16 12:44:56.906311000 -0700 ++++ include/inn/ov.h 2026-05-16 12:45:04.836546000 -0700 +@@ -52,6 +52,7 @@ + } OVGE; + + extern bool OVstatall; ++ + bool OVopen(int mode); + bool OVgroupstats(char *group, int *lo, int *hi, int *count, int *flag); + bool OVgroupadd(char *group, ARTNUM lo, ARTNUM hi, char *flag); +diff --git include/inn/storage.h include/inn/storage.h +--- include/inn/storage.h 2026-05-16 12:44:56.906957000 -0700 ++++ include/inn/storage.h 2026-05-16 12:45:04.836761000 -0700 +@@ -106,6 +106,28 @@ + ARTHANDLE *SMnext(ARTHANDLE *article, const RETRTYPE amount); + void SMfreearticle(ARTHANDLE *article); + bool SMcancel(TOKEN token); ++ ++/* ++ * Best-effort: append a token to the out-of-band cancel tombstone log ++ * (${pathdb}/cancels.tombstone) so a later expire run can drop the ++ * matching history entry without an SMretrieve(RETR_STAT). Intended ++ * for callers that cancel articles outside the expireover/expirerm ++ * pipeline (innd's ARTcancel and sm -r). No-op when ++ * innconf->expiretombstone is false or token has type TOKEN_EMPTY. ++ * ++ * Concurrency: appenders take a non-blocking shared (F_RDLCK) fcntl ++ * lock; expire's consumer takes an exclusive lock briefly during ++ * snapshot/rename. POSIX O_APPEND atomicity for sub-PIPE_BUF ++ * writes keeps single-line tokens from interleaving across ++ * concurrent appenders. ++ * ++ * Returns true on success (token appended, flushed, file closed), ++ * false on any failure path. Failures are logged via syswarn; ++ * callers can ignore the return value (the cancel itself has ++ * already succeeded by the time this is called). ++ */ ++bool SMcanceltombstone(TOKEN token); ++ + bool SMprobe(PROBETYPE type, TOKEN *token, void *value); + bool SMflushcacheddata(FLUSHTYPE type); + void SMprintfiles(FILE *file, TOKEN token, char **xref, int ngroups); +diff --git include/inn/tombstone.h include/inn/tombstone.h +--- include/inn/tombstone.h 1969-12-31 17:00:00.000000000 -0700 ++++ include/inn/tombstone.h 2026-05-16 12:45:04.836999000 -0700 +@@ -0,0 +1,71 @@ ++/* ++** Helpers for the expire-tombstone log files. ++** ++** These routines are shared between the expire binary (which consumes ++** the logs to skip per-article SMretrieve(RETR_STAT) calls) and the ++** test suite that exercises the consumption logic directly. ++** ++** See doc/pod/inn.conf.pod under "expiretombstone" for the file ++** semantics: ++** - ${pathdb}/expireover.tombstone : written by expireover/expirerm ++** via OVEXPremove, atomic .NEW -> final rename ++** - ${pathdb}/cancels.tombstone : appended continuously by innd ++** and sm via SMcanceltombstone() for cancels outside the ++** expireover/expirerm pipeline ++*/ ++ ++#ifndef INN_TOMBSTONE_H ++#define INN_TOMBSTONE_H ++ ++#include "inn/hashtab.h" ++#include "inn/portable-macros.h" ++#include "inn/portable-stdbool.h" ++#include "inn/storage.h" ++ ++BEGIN_DECLS ++ ++/* ++** Create a hashset suited to holding TOKEN keys (each entry is a ++** TOKEN allocated by the caller; the hashset owns and frees them). ++** size is the initial bucket hint; the hash auto-expands. ++*/ ++struct hash *tombstone_hash_create(size_t size); ++ ++/* ++** Read tombstone entries from path into the hashset. Each non-blank ++** line is parsed with TextToToken and inserted; malformed lines are ++** skipped with a warning. Duplicates are dropped silently. Returns ++** the number of valid entries seen, or 0 if the file does not exist ++** (ENOENT is silent; other open errors emit syswarn). ++** ++** If out_error is non-NULL, *out_error is set to true when the read ++** was incomplete (open failed for a non-ENOENT reason or fgets hit a ++** stream error mid-file) and to false otherwise. Callers that cache ++** freshness based on file mtime should not treat a partial read as ++** authoritative. ENOENT does not set the flag because an absent ++** file is a successful "no entries" read. ++*/ ++unsigned long tombstone_read(struct hash *h, const char *path, ++ bool *out_error); ++ ++/* ++** Atomically snapshot a continuously-appended tombstone file by ++** renaming it to "${path}.processing" under an exclusive POSIX lock. ++** This serializes against concurrent appenders that hold the same ++** lock (e.g., SMcanceltombstone()), so any in-progress write either ++** completes before the rename or proceeds against the renamed inode. ++** ++** Returns the snapshot path (caller must free and unlink after ++** consumption) or NULL if the source file does not exist or cannot ++** be renamed. ++*/ ++char *tombstone_rename_for_processing(const char *path); ++ ++/* ++** Look up a TOKEN in the hashset. Returns true if present. ++*/ ++bool tombstone_present(struct hash *h, const TOKEN *token); ++ ++END_DECLS ++ ++#endif /* INN_TOMBSTONE_H */ +diff --git innd/art.c innd/art.c +--- innd/art.c 2026-05-16 12:44:56.919455000 -0700 ++++ innd/art.c 2026-05-16 12:45:04.837716000 -0700 +@@ -1280,9 +1280,15 @@ + /* Get stored message and zap them. */ + if (innconf->enableoverview) + OVcancel(token); +- if (!SMcancel(token) && SMerrno != SMERR_NOENT && SMerrno != SMERR_UNINIT) ++ if (SMcancel(token) || SMerrno == SMERR_NOENT) { ++ /* Record the out-of-band cancel so a later expire run can drop ++ * the history entry without an SMretrieve(RETR_STAT). Best- ++ * effort and a no-op when expiretombstone is disabled. */ ++ SMcanceltombstone(token); ++ } else if (SMerrno != SMERR_UNINIT) { + syslog(L_ERROR, "%s cant cancel %s (SMerrno %d)", LogName, + TokenToText(token), SMerrno); ++ } + if (innconf->immediatecancel && !SMflushcacheddata(SM_CANCELLEDART)) + syslog(L_ERROR, "%s cant cancel cached %s", LogName, + TokenToText(token)); +diff --git lib/innconf.c lib/innconf.c +--- lib/innconf.c 2026-05-16 12:44:56.914445000 -0700 ++++ lib/innconf.c 2026-05-16 12:45:04.838186000 -0700 +@@ -71,6 +71,7 @@ + {K(enableoverview), BOOL(true) }, + {K(extraoverviewadvertised), LIST(NULL) }, + {K(extraoverviewhidden), LIST(NULL) }, ++ {K(expiretombstone), BOOL(false) }, + {K(fromhost), STRING(NULL) }, + {K(groupbaseexpiry), BOOL(true) }, + {K(mailcmd), STRING(NULL) }, +diff --git nnrpd/article.c nnrpd/article.c +--- nnrpd/article.c 2026-05-16 12:44:56.920482000 -0700 ++++ nnrpd/article.c 2026-05-16 12:45:04.843967000 -0700 +@@ -7,13 +7,16 @@ + #include + #include + #include ++#include + #include + + #include "cache.h" + #include "inn/innconf.h" ++#include "inn/libinn.h" + #include "inn/messages.h" + #include "inn/ov.h" + #include "inn/overview.h" ++#include "inn/tombstone.h" + #include "inn/wire.h" + #include "nnrpd.h" + #include "tls.h" +@@ -260,17 +263,131 @@ + } + } + ++/* ++** Cancel-tombstone fast path for ARTinstorebytoken. When both ++** innconf->expiretombstone and PERMaccessconf->nnrpdcheckart are true, ++** consult ${pathdb}/cancels.tombstone before falling through to the ++** per-article SMretrieve syscall. The tombstone records cancels ++** written by innd's ARTcancel and by sm -r, the out-of-band paths ++** that can leave overview entries pointing at gone storage in the ++** brief race window before overview cleanup propagates, or until the ++** next expireover for sm -r. ++** ++** Lazy per-connection: loaded on first call, refreshed on mtime ++** change. Memory cost is ~50 bytes per cancel; typical sites have ++** hundreds of entries. Falls through to SMretrieve on load failure. ++*/ ++static struct hash *nnrpd_tombstone = NULL; ++static char *nnrpd_tombstone_path = NULL; ++static time_t nnrpd_tombstone_mtime = 0; ++static off_t nnrpd_tombstone_size = 0; ++ ++/* ++** Bring the per-connection tombstone cache up to date. Returns true ++** if a usable hashset is loaded after the call (caller may then ++** consult tombstone_present), false otherwise (file missing or load ++** failed; caller should fall through to the slow path). ++** ++** Stat'd on every call so a cancel recorded by another process ++** becomes visible to readers immediately. The cost is one stat() ++** on a fixed path that stays hot in the dentry cache, which is ++** trivially cheap compared to the per-article SMretrieve syscalls ++** the fast path elides. Reload only happens when mtime or size ++** changes, so the read+parse cost is paid at most once per cancel. ++** ++** Freshness is keyed on both mtime and size: mtime alone has 1- ++** second granularity, so a rename-and-recreate within the same ++** second can land a fresh inode with the cached mtime; size is ++** monotonic-append between rotations and so changes whenever a ++** new line is added. Together they catch every modification. ++** ++** If tombstone_read reports a partial read (mid-file ferror), the ++** cached mtime/size are not updated, so the next call retries the ++** read instead of trusting a partial hashset. ++*/ ++static bool ++nnrpd_tombstone_refresh(void) ++{ ++ /* Initial hash bucket count for the per-connection tombstone cache. ++ * Sized for the typical case (hundreds of cancels per cycle); the ++ * hashset auto-expands when the load factor is exceeded. */ ++ static const size_t initial_buckets = 256; ++ struct stat sb; ++ bool read_error = false; ++ ++ if (nnrpd_tombstone_path == NULL) ++ nnrpd_tombstone_path = ++ concatpath(innconf->pathdb, "cancels.tombstone"); ++ ++ /* If the file has gone away since our last load, drop our cache ++ * and fall through to the slow path. */ ++ if (stat(nnrpd_tombstone_path, &sb) < 0) { ++ if (nnrpd_tombstone != NULL) { ++ hash_free(nnrpd_tombstone); ++ nnrpd_tombstone = NULL; ++ nnrpd_tombstone_mtime = 0; ++ nnrpd_tombstone_size = 0; ++ } ++ return false; ++ } ++ ++ /* Fresh enough? */ ++ if (nnrpd_tombstone != NULL && sb.st_mtime == nnrpd_tombstone_mtime ++ && sb.st_size == nnrpd_tombstone_size) ++ return true; ++ ++ /* (Re)load. */ ++ if (nnrpd_tombstone != NULL) ++ hash_free(nnrpd_tombstone); ++ nnrpd_tombstone = tombstone_hash_create(initial_buckets); ++ tombstone_read(nnrpd_tombstone, nnrpd_tombstone_path, &read_error); ++ if (read_error) { ++ /* Partial parse; do not advance the freshness key so the ++ * next call retries. The hashset still holds whatever lines ++ * we did parse, which is not wrong (those are real cancels), ++ * just incomplete. */ ++ return true; ++ } ++ nnrpd_tombstone_mtime = sb.st_mtime; ++ nnrpd_tombstone_size = sb.st_size; ++ return true; ++} ++ ++ + bool + ARTinstorebytoken(TOKEN token) + { + ARTHANDLE *art; + struct timeval stv, etv; + +- if (PERMaccessconf->nnrpdoverstats) { +- gettimeofday(&stv, NULL); ++ /* Fast path: if both expiretombstone and nnrpdcheckart are ++ * enabled and the cache loads, the cancels.tombstone log ++ * records out-of-band cancels (innd ARTcancel, sm -r) that may ++ * not yet be reflected in overview. Tombstone hit means the ++ * article is gone; tombstone miss on a non-self-expiring ++ * backend means it is alive and we can skip the SMretrieve ++ * syscall. Self-expiring backends (CNFS) still need the slow ++ * path because wrap-around bypasses SMcancel and the tombstone. ++ * ++ * The nnrpdcheckart gate matches the documented contract: ++ * tombstone-driven existence checks are an alternative ++ * implementation of nnrpdcheckart's existence verification, so ++ * we should not run them when the operator has explicitly ++ * disabled article-existence checking. ++ * ++ * SMprobe(SELFEXPIRE) is checked first so pure-CNFS sites pay ++ * only that probe (a static per-method attribute, no I/O) and ++ * skip both the hash lookup and the cache refresh. */ ++ if (innconf->expiretombstone && PERMaccessconf->nnrpdcheckart ++ && !SMprobe(SELFEXPIRE, &token, NULL) ++ && nnrpd_tombstone_refresh()) { ++ return !tombstone_present(nnrpd_tombstone, &token); + } +- art = SMretrieve(token, RETR_STAT); ++ + /* XXX This isn't really overstats, is it? */ ++ if (PERMaccessconf->nnrpdoverstats) ++ gettimeofday(&stv, NULL); ++ art = SMretrieve(token, RETR_STAT); + if (PERMaccessconf->nnrpdoverstats) { + gettimeofday(&etv, NULL); + OVERartcheck += (etv.tv_sec - stv.tv_sec) * 1000; +diff --git samples/inn.conf.in samples/inn.conf.in +--- samples/inn.conf.in 2026-05-16 12:44:56.900644000 -0700 ++++ samples/inn.conf.in 2026-05-16 12:45:04.838480000 -0700 +@@ -67,6 +67,7 @@ + expirebloomfp: 10000 + extraoverviewadvertised: [ ] + extraoverviewhidden: [ ] ++expiretombstone: false + groupbaseexpiry: true + mergetogroups: false + nfswriter: false +diff --git storage/Makefile storage/Makefile +--- storage/Makefile 2026-05-16 12:44:56.910882000 -0700 ++++ storage/Makefile 2026-05-16 12:45:04.838862000 -0700 +@@ -12,7 +12,7 @@ + CFLAGS = $(GCFLAGS) -I. $(BDB_CPPFLAGS) $(SQLITE3_CPPFLAGS) + + SOURCES = expire.c interface.c methods.c ov.c overdata.c overview.c \ +- ovmethods.c $(METHOD_SOURCES) ++ ovmethods.c tombstone.c $(METHOD_SOURCES) + OBJECTS = $(SOURCES:.c=.o) + LOBJECTS = $(OBJECTS:.o=.lo) + +diff --git storage/expire.c storage/expire.c +--- storage/expire.c 2026-05-16 12:44:56.910633000 -0700 ++++ storage/expire.c 2026-05-16 12:45:04.839298000 -0700 +@@ -15,6 +15,7 @@ + #include "inn/bloom.h" + #include "inn/innconf.h" + #include "inn/libinn.h" ++#include "inn/messages.h" + #include "inn/ov.h" + #include "inn/overview.h" + #include "inn/paths.h" +@@ -104,6 +105,7 @@ + This should be cleaned up with a better internal interface. */ + time_t OVnow; + FILE *EXPunlinkfile; ++FILE *OVtombstonefile; + bool OVignoreselfexpire; + bool OVusepost; + bool OVkeep; +@@ -526,16 +528,53 @@ + } + if (EXPunlinkfile && xref != NULL) { + SMprintfiles(EXPunlinkfile, token, xref, ngroups); +- if (!ferror(EXPunlinkfile)) ++ if (!ferror(EXPunlinkfile)) { ++ /* Deferred-removal path: actual SMcancel happens later via ++ * expirerm/fastrm. Record the token in the tombstone log ++ * now so the eventual expire run can consume it. The ++ * .NEW -> final rename is performed by expirerm after ++ * fastrm succeeds (see expirerm.in); if expirerm fails or ++ * is skipped, the .NEW file is wiped by the next ++ * expireover and never consumed, so a partial rmfile ++ * never lets expire drop history for live articles. */ ++ if (OVtombstonefile != NULL) { ++ fprintf(OVtombstonefile, "%s\n", TokenToText(token)); ++ if (ferror(OVtombstonefile)) { ++ syswarn("can't write tombstone log; ignoring it" ++ " for rest of run"); ++ fclose(OVtombstonefile); ++ OVtombstonefile = NULL; ++ } ++ } + return; +- fprintf(stderr, "Can't write to -z file, %s\n", strerror(errno)); +- fprintf(stderr, "(Will ignore it for rest of run.)\n"); ++ } ++ syswarn("can't write to -z file; ignoring it for rest of run"); + fclose(EXPunlinkfile); + EXPunlinkfile = NULL; + } +- if (!SMcancel(token) && SMerrno != SMERR_NOENT && SMerrno != SMERR_UNINIT) +- fprintf(stderr, "Can't unlink %s: %s\n", TokenToText(token), +- SMerrorstr); ++ /* Inline-cancel path. Tombstone is written only on a successful ++ * SMcancel (or NOENT, where the article is already gone): the log ++ * must be a strict subset of articles actually removed. Otherwise ++ * a non-NOENT/UNINIT failure (EIO, EBUSY, fs permission, etc.) ++ * would let the next expire drop the history entry while the ++ * article remains on disk: an undetectable permanent orphan. ++ * The inverse risk (cancelled but not logged on a crash between ++ * SMcancel and fprintf) leaves a history entry pointing at a ++ * missing article, which is recoverable by a periodic non-tombstone ++ * reconciliation run. */ ++ if (SMcancel(token) || SMerrno == SMERR_NOENT) { ++ if (OVtombstonefile != NULL) { ++ fprintf(OVtombstonefile, "%s\n", TokenToText(token)); ++ if (ferror(OVtombstonefile)) { ++ syswarn("can't write tombstone log; ignoring it for" ++ " rest of run"); ++ fclose(OVtombstonefile); ++ OVtombstonefile = NULL; ++ } ++ } ++ } else if (SMerrno != SMERR_UNINIT) { ++ warn("can't unlink %s: %s", TokenToText(token), SMerrorstr); ++ } + } + + /* +diff --git storage/interface.c storage/interface.c +--- storage/interface.c 2026-05-16 12:44:56.908549000 -0700 ++++ storage/interface.c 2026-05-16 12:45:04.839711000 -0700 +@@ -6,7 +6,9 @@ + + #include + #include ++#include + #include ++#include + + #include "conffile.h" + #include "inn/innconf.h" +@@ -815,6 +817,135 @@ + return false; + } + return storage_methods[typetoindex[token.type]].cancel(token); ++} ++ ++ ++/* ++** Append a token to the out-of-band cancel tombstone log. Best-effort: ++** failures are logged but do not affect the caller. Safe for concurrent ++** use across processes via inn_lock_file (fcntl POSIX locks; NFS-correct). ++** ++** Used by callers (innd, sm) that cancel articles outside the ++** expireover/expirerm pipeline. expire reads this log alongside the ++** expireover.tombstone so it can drop history entries for articles ++** cancelled out-of-band without per-article SMretrieve(RETR_STAT). ++** ++** No-op when innconf->expiretombstone is false. No-op for TOKEN_EMPTY. ++** ++** Locking model: shared (F_RDLCK) for appenders, exclusive (F_WRLCK) ++** for the consumer's read+rename phase (see tombstone_rename_for_ ++** processing). The lock type names "read" and "write" here describe ++** reader-vs-truncator coordination, not file access pattern; both ++** innd and sm write the file under the shared lock. POSIX guarantees ++** that each write(2) to a regular file opened with O_APPEND is ++** atomic with respect to other writers, so single-line tokens ++** interleave correctly without per-write serialization. POSIX ++** requires F_RDLCK only on a descriptor open for reading, so we ++** open O_RDWR even though we only write here. ++** ++** Lock is acquired non-blocking. innd's main loop must never wait ++** on filesystem locking (NFS lockd outages, expire's brief ++** rename hold). If the lock is contested, we skip this tombstone ++** write rather than block. Concurrent appenders do not contend ++** because shared locks coexist; only expire's exclusive lock ++** briefly excludes them, and that window is a single ftruncate-or- ++** rename call. ++*/ ++bool ++SMcanceltombstone(TOKEN token) ++{ ++ char *path; ++ int fd; ++ FILE *fp; ++ const char *text; ++ bool ok = true; ++ ++ /* Both knobs are required: expire only consumes the log when ++ * groupbaseexpiry is also true, so writing without that gate ++ * would leak entries forever and bloat nnrpd's per-connection ++ * cache. */ ++ if (!innconf->expiretombstone || !innconf->groupbaseexpiry) ++ return true; ++ if (token.type == TOKEN_EMPTY) ++ return true; ++ ++ path = concatpath(innconf->pathdb, "cancels.tombstone"); ++ /* O_RDWR (not O_WRONLY) so we can take F_RDLCK below; POSIX ++ * mandates lock-mode-matches-fd-mode. */ ++ fd = open(path, O_RDWR | O_CREAT | O_APPEND, 0664); ++ if (fd < 0) { ++ syswarn("can't open %s for tombstone append", path); ++ free(path); ++ return false; ++ } ++ /* Shared, non-blocking. Multiple appenders coexist; only blocked ++ * by expire's exclusive consumer lock during its brief rename. ++ * ++ * Race window: between our open() and our lock(), expire may ++ * rename(cancels.tombstone, cancels.tombstone.processing), ++ * leaving us with a fd pointing at the renamed (doomed) inode. ++ * If we acquired the lock on that fd and wrote, our entry would ++ * be consumed only by expire's already-loaded snapshot -- and ++ * our writes appended after the rename are not in that snapshot, ++ * so they would be unlinked when expire finishes. ++ * ++ * Resolution: on EAGAIN/EACCES (lock held by expire's rename ++ * window), close and reopen the path. After expire releases ++ * its exclusive lock, the path resolves to a fresh inode (or ++ * nothing, in which case O_CREAT makes a new one). Either way, ++ * the second open lands on the live file the next consumer ++ * will see. A 1ms sleep is more than enough for expire to ++ * complete its rename and release. */ ++ if (!inn_lock_file(fd, INN_LOCK_READ, false)) { ++ if (errno == EAGAIN || errno == EACCES) { ++ struct timespec ts = {0, 1000000L}; /* 1ms */ ++ close(fd); ++ nanosleep(&ts, NULL); ++ fd = open(path, O_RDWR | O_CREAT | O_APPEND, 0664); ++ if (fd < 0) { ++ syswarn("can't reopen %s for tombstone append", path); ++ free(path); ++ return false; ++ } ++ if (!inn_lock_file(fd, INN_LOCK_READ, false)) { ++ syswarn("can't lock %s after retry; skipping" ++ " tombstone entry", ++ path); ++ close(fd); ++ free(path); ++ return false; ++ } ++ } else { ++ syswarn("can't lock %s; skipping tombstone entry", path); ++ close(fd); ++ free(path); ++ return false; ++ } ++ } ++ fp = fdopen(fd, "a"); ++ if (fp == NULL) { ++ syswarn("can't fdopen %s", path); ++ inn_lock_file(fd, INN_LOCK_UNLOCK, true); ++ close(fd); ++ free(path); ++ return false; ++ } ++ text = TokenToText(token); ++ /* Single fprintf + fflush keeps the kernel-visible write to one ++ * syscall (~38-byte payload) so the per-write O_APPEND atomicity ++ * guarantee holds for concurrent shared-lock appenders. */ ++ if (fprintf(fp, "%s\n", text) < 0 || fflush(fp) == EOF) { ++ syswarn("can't write to %s", path); ++ ok = false; ++ } ++ /* fclose closes the underlying fd, which releases the fcntl ++ * POSIX lock; no separate INN_LOCK_UNLOCK needed. */ ++ if (fclose(fp) == EOF) { ++ syswarn("can't close %s", path); ++ ok = false; ++ } ++ free(path); ++ return ok; + } + + bool +diff --git storage/ovinterface.h storage/ovinterface.h +--- storage/ovinterface.h 2026-05-16 12:44:56.908195000 -0700 ++++ storage/ovinterface.h 2026-05-16 12:45:04.839984000 -0700 +@@ -5,6 +5,8 @@ + #ifndef OVINTERFACE_H + #define OVINTERFACE_H + ++#include ++ + #include "config.h" + #include "inn/history.h" + #include "inn/ov.h" +@@ -43,6 +45,17 @@ + extern struct bloom_filter *OVtokencache; + extern time_t OVnow; + extern FILE *EXPunlinkfile; ++ ++/* Tombstone log: when set, OVEXPremove appends the textual form of each ++ * successfully cancelled token to this stream. A subsequent expire run ++ * consumes the log to drop history entries for those articles without ++ * doing per-article SMretrieve(RETR_STAT) calls. Set by expireover (and ++ * by fastrm when called via expirerm in delayrm mode). Internal -- ++ * managed via direct extern access by expireover/fastrm, not via OVctl, ++ * because the file lifecycle (atomic .NEW -> final rename on success) ++ * does not fit the per-call OVctl pattern. */ ++extern FILE *OVtombstonefile; ++ + extern bool OVignoreselfexpire; + extern bool OVusepost; + extern bool OVkeep; +diff --git storage/tombstone.c storage/tombstone.c +--- storage/tombstone.c 1969-12-31 17:00:00.000000000 -0700 ++++ storage/tombstone.c 2026-05-16 12:45:04.840226000 -0700 +@@ -0,0 +1,156 @@ ++/* ++** Helpers for the expire-tombstone log files. ++** ++** See include/inn/tombstone.h for the API and inn.conf.pod under ++** "expiretombstone" for the file semantics. Shared between the ++** expire binary and the test suite so both exercise identical ++** parsing, hashing, and atomic-snapshot logic. ++*/ ++ ++#include "portable/system.h" ++ ++#include ++#include ++#include ++#include ++ ++#include "inn/hashtab.h" ++#include "inn/libinn.h" ++#include "inn/messages.h" ++#include "inn/storage.h" ++#include "inn/tombstone.h" ++ ++ ++/* ++** Hashset glue. Keys are 18-byte TOKEN structs (1 type + 1 class + ++** 16 token bytes) hashed and compared as raw bytes. hash_lookup2 ++** derives a stable unsigned long from the bytes; tombstone_equal ++** uses memcmp for an exact match. ++*/ ++ ++static unsigned long ++ts_hash(const void *p) ++{ ++ return hash_lookup2((const char *) p, sizeof(TOKEN), 0); ++} ++ ++static const void * ++ts_key(const void *p) ++{ ++ return p; ++} ++ ++static bool ++ts_equal(const void *a, const void *b) ++{ ++ return memcmp(a, b, sizeof(TOKEN)) == 0; ++} ++ ++ ++struct hash * ++tombstone_hash_create(size_t size) ++{ ++ return hash_create(size, ts_hash, ts_key, ts_equal, free); ++} ++ ++ ++unsigned long ++tombstone_read(struct hash *h, const char *path, bool *out_error) ++{ ++ FILE *f; ++ char line[SMBUF]; ++ size_t len; ++ unsigned long count = 0; ++ ++ if (out_error != NULL) ++ *out_error = false; ++ if (h == NULL || path == NULL) ++ return 0; ++ ++ f = fopen(path, "r"); ++ if (f == NULL) { ++ if (errno != ENOENT) { ++ syswarn("can't open %s; ignoring", path); ++ if (out_error != NULL) ++ *out_error = true; ++ } ++ return 0; ++ } ++ while (fgets(line, sizeof(line), f) != NULL) { ++ TOKEN t; ++ TOKEN *entry; ++ ++ len = strlen(line); ++ while (len > 0 && (line[len - 1] == '\n' || line[len - 1] == '\r')) ++ line[--len] = '\0'; ++ if (len == 0) ++ continue; ++ /* Comment lines (e.g., "# inn-tombstone v1") are reserved for ++ * future format markers and metadata. Readers ignore them ++ * silently so format evolution is graceful. */ ++ if (line[0] == '#') ++ continue; ++ if (!IsToken(line)) { ++ warn("malformed tombstone entry in %s: %s", path, line); ++ continue; ++ } ++ t = TextToToken(line); ++ entry = xmalloc(sizeof(TOKEN)); ++ *entry = t; ++ if (!hash_insert(h, entry, entry)) { ++ /* Already present (duplicate cancel), free the redundant ++ * allocation; the existing entry stays. */ ++ free(entry); ++ } ++ count++; ++ } ++ if (ferror(f)) { ++ syswarn("error reading %s; will use what was read", path); ++ if (out_error != NULL) ++ *out_error = true; ++ } ++ fclose(f); ++ return count; ++} ++ ++ ++char * ++tombstone_rename_for_processing(const char *path) ++{ ++ char *snapshot_path; ++ int fd; ++ ++ if (path == NULL) ++ return NULL; ++ ++ fd = open(path, O_RDWR); ++ if (fd < 0) { ++ if (errno != ENOENT) ++ syswarn("can't open %s", path); ++ return NULL; ++ } ++ if (!inn_lock_file(fd, INN_LOCK_WRITE, true)) { ++ syswarn("can't lock %s", path); ++ close(fd); ++ return NULL; ++ } ++ snapshot_path = concat(path, ".processing", (char *) 0); ++ if (rename(path, snapshot_path) < 0) { ++ syswarn("can't rename %s to %s", path, snapshot_path); ++ free(snapshot_path); ++ close(fd); ++ return NULL; ++ } ++ /* close releases the fcntl POSIX lock. */ ++ close(fd); ++ return snapshot_path; ++} ++ ++ ++bool ++tombstone_present(struct hash *h, const TOKEN *token) ++{ ++ if (h == NULL || token == NULL) ++ return false; ++ return hash_lookup(h, token) != NULL; ++} +diff --git support/mkmanifest support/mkmanifest +--- support/mkmanifest 2026-05-16 12:44:56.918414000 -0700 ++++ support/mkmanifest 2026-05-16 12:45:04.840505000 -0700 +@@ -278,6 +278,8 @@ + tests/authprogs/ident.t + tests/clients/server-list + tests/docs/pod.t ++tests/expire/tombstone-hisexpire.t ++tests/expire/tombstone.t + tests/innd/artparse.t + tests/innd/chan.t + tests/lib/artnumber.t +@@ -335,4 +337,5 @@ + tests/overview/tradindexed.t + tests/overview/xref.t + tests/perl/minimum-version.t ++tests/storage/cancel-tombstone.t + tests/util/innbind.t +diff --git tests/Makefile tests/Makefile +--- tests/Makefile 2026-05-16 12:44:56.926071000 -0700 ++++ tests/Makefile 2026-05-16 12:45:24.868788000 -0700 +@@ -31,7 +31,9 @@ + lib/setenv.t lib/snprintf.t lib/strlcat.t \ + lib/strlcpy.t lib/tst.t lib/uwildmat.t lib/vector.t lib/wire.t \ + lib/xwrite.t nnrpd/auth-ext.t overview/api.t overview/buffindexed.t \ +- overview/ovsqlite.t overview/tradindexed.t overview/xref.t util/innbind.t ++ overview/ovsqlite.t overview/tradindexed.t overview/xref.t util/innbind.t \ ++ expire/tombstone.t expire/tombstone-hisexpire.t \ ++ storage/cancel-tombstone.t + + ## Extra stuff that needs to be built before tests can be run. + +@@ -322,6 +324,19 @@ + + overview/xref.t: overview/xref-t.o tap/basic.o $(STORAGEDEPS) + $(LINKDEPS) overview/xref-t.o tap/basic.o $(STORAGELIBS) $(LIBS) ++ ++expire/tombstone.t: expire/tombstone-t.o tap/basic.o $(STORAGEDEPS) ++ $(LINKDEPS) expire/tombstone-t.o tap/basic.o $(STORAGELIBS) $(LIBS) ++ ++expire/tombstone-hisexpire.t: expire/tombstone-hisexpire-t.o tap/basic.o \ ++ $(STORAGEDEPS) ++ $(LINKDEPS) expire/tombstone-hisexpire-t.o tap/basic.o \ ++ $(STORAGELIBS) $(LIBS) ++ ++storage/cancel-tombstone.t: storage/cancel-tombstone-t.o tap/basic.o \ ++ $(STORAGEDEPS) ++ $(LINKDEPS) storage/cancel-tombstone-t.o tap/basic.o \ ++ $(STORAGELIBS) $(LIBS) + + perl/minimum-version.t: perl/minimum-version.t.in $(FIXSCRIPT) + $(FIX) -i perl/minimum-version.t.in +diff --git tests/TESTS tests/TESTS +--- tests/TESTS 2026-05-16 12:44:56.926316000 -0700 ++++ tests/TESTS 2026-05-16 12:45:33.721800000 -0700 +@@ -54,6 +54,10 @@ + nnrpd/auth-ext + overview/api + overview/buffindexed ++expire/tombstone ++expire/tombstone-e2e ++expire/tombstone-hisexpire ++storage/cancel-tombstone + overview/ovsqlite + overview/ovsqlite-integ + overview/overchan +diff --git tests/expire/tombstone-e2e.t tests/expire/tombstone-e2e.t +--- tests/expire/tombstone-e2e.t 1969-12-31 17:00:00.000000000 -0700 ++++ tests/expire/tombstone-e2e.t 2026-05-16 12:45:04.841566000 -0700 +@@ -0,0 +1,174 @@ ++#! /bin/sh ++# ++# End-to-end test for the expire-tombstone feature. ++# ++# Exercises the integration between sm (writes to cancels.tombstone), ++# the tombstone library helpers (read/snapshot/lookup), and the ++# expiretombstone-related guards. Verifies: ++# - sm -r appends to ${pathdb}/cancels.tombstone when expiretombstone=true ++# - sm -r does NOT write when expiretombstone=false ++# - The written entries round-trip through tombstone_read into a ++# hashset that can answer membership queries ++# - The atomic-snapshot rename moves cancels.tombstone aside under ++# lock so a fresh file can capture writes that follow ++ ++count=1 ++printcount() { ++ echo "$1 $count $2" ++ count=$(expr $count + 1) ++} ++ ++# Find the right directory. ++sm="../../frontends/sm" ++dirs='../data data tests/data' ++for dir in $dirs; do ++ if [ -r "$dir/articles/1" ]; then ++ cd $dir ++ break ++ fi ++done ++if [ ! -x "$sm" ]; then ++ echo "Could not find sm" >&2 ++ exit 1 ++fi ++ ++# Use a fresh dedicated pathdb so we do not interfere with the other ++# storage tests (which use ./db) and so we can flip expiretombstone ++# without touching the shared etc/inn.conf. ++TMPDIR_E2E="tombstone-e2e.tmp" ++rm -rf "$TMPDIR_E2E" spool tradspool.map ++mkdir -p "$TMPDIR_E2E" spool ++ ++# Build a tweaked inn.conf that enables expiretombstone and points ++# pathdb at our temp dir. Other paths inherit defaults from the ++# tests/data tree. ++cat > "$TMPDIR_E2E/inn.conf" </dev/null 2>&1 ++lines=$(wc -l < "$CANCELS" | sed -e 's/[ \t]//g') ++if [ "$lines" = 2 ]; then ++ printcount "ok" ++else ++ printcount "not ok" "expected 2 lines after invalid-token rm, got $lines" ++fi ++ ++# 7. All entries are valid tokens (round-trip parseable). Use sm -i to ++# parse each line; sm -i succeeds only if the input is a valid token ++# that resolves to a known article. After cancellation the article ++# is gone, so sm -i exits non-zero with "Token not found", but it ++# does NOT exit with "Bad token" -- which is the format-validity ++# check we want. ++all_format_valid=true ++while read -r line; do ++ case "$line" in ++ '@'*'@') ++ # Format check: starts and ends with @ ++ ;; ++ *) ++ all_format_valid=false ++ ;; ++ esac ++done < "$CANCELS" ++if $all_format_valid; then ++ printcount "ok" ++else ++ printcount "not ok" "tombstone contains malformed lines" ++fi ++ ++# 8. flock contract: after sm finishes, the lock is released and another ++# writer (here, ourselves via sm) can acquire it. This is implicit ++# in tests 2, 4, etc., but we explicitly verify by storing+removing ++# a fourth article and confirming a clean append. ++token4=$($sm -s < articles/4) ++$sm -r "$token4" ++lines=$(wc -l < "$CANCELS" | sed -e 's/[ \t]//g') ++if [ "$lines" = 3 ] && grep -qF "$token4" "$CANCELS"; then ++ printcount "ok" ++else ++ printcount "not ok" "fourth cancel: expected 3 lines, got $lines" ++fi ++ ++# Cleanup. ++rm -rf "$TMPDIR_E2E" spool tradspool.map "$TMPDIR_E2E.bak" ++exit 0 +diff --git tests/expire/tombstone-hisexpire-t.c tests/expire/tombstone-hisexpire-t.c +--- tests/expire/tombstone-hisexpire-t.c 1969-12-31 17:00:00.000000000 -0700 ++++ tests/expire/tombstone-hisexpire-t.c 2026-05-16 12:45:04.841888000 -0700 +@@ -0,0 +1,298 @@ ++/* Integration test: HISexpire callback path consults the tombstone hashset. ++ * ++ * Builds a temporary history file with HISopen/HISwrite, populates a ++ * tombstone hashset for some of the tokens, runs HISexpire with a ++ * callback that mirrors EXPdoline's decision tree -- including the ++ * SELFEXPIRE branch (gap G). Verifies the resulting new history file ++ * contains exactly the tokens that should survive. ++ * ++ * The callback distinguishes four token categories: ++ * N_KEPT not tombstoned, not self-expiring -> keep ++ * N_TOMBSTONED in tombstone (fast path) -> drop ++ * N_SELFEXP_GONE self-expiring, simulated NOENT -> drop (slow path) ++ * N_SELFEXP_ALIVE self-expiring, simulated alive -> keep ++ * ++ * The slow path mirrors EXPdoline's behaviour for backends where ++ * SMprobe(SELFEXPIRE) is true: even with a tombstone hashset present, ++ * we still consult the simulated SMretrieve because articles can vanish ++ * via wrap-around without going through SMcancel. */ ++ ++#include "portable/system.h" ++ ++#include ++#include ++#include ++ ++#include "inn/hashtab.h" ++#include "inn/history.h" ++#include "inn/libinn.h" ++#include "inn/messages.h" ++#include "inn/storage.h" ++#include "inn/tombstone.h" ++#include "tap/basic.h" ++ ++ ++#define N_KEPT 5 ++#define N_TOMBSTONED 4 ++#define N_SELFEXP_GONE 3 ++#define N_SELFEXP_ALIVE 2 ++ ++#define N_TOTAL (N_KEPT + N_TOMBSTONED + N_SELFEXP_GONE + N_SELFEXP_ALIVE) ++ ++ ++/* Cookie threading two hashsets through HISexpire's callback: the ++ * tombstone (real EXPdoline checks this first) and a "gone" set used to ++ * simulate SMretrieve(RETR_STAT) == NULL for self-expiring backends. */ ++struct expire_cookie { ++ struct hash *tombstone; ++ struct hash *gone; ++}; ++ ++ ++/* Mirror of EXPdoline's decision tree for tokens that may be tombstoned ++ * or self-expiring. Returns true to keep, false to drop. ++ * ++ * Tokens are tagged via their type byte: ++ * type=1 - non-self-expiring backend ++ * type=2 - self-expiring backend (CNFS-like) ++ */ ++static bool ++test_expire_cb(void *cookie, time_t arrived UNUSED, time_t posted UNUSED, ++ time_t expires UNUSED, TOKEN *token) ++{ ++ struct expire_cookie *c = cookie; ++ ++ /* Fast path: tombstone hit -> drop, no further checks. */ ++ if (c->tombstone != NULL && tombstone_present(c->tombstone, token)) ++ return false; ++ ++ /* Self-expiring backend (gap G). Even with a tombstone in hand, we ++ * cannot trust "not in tombstone == alive" because wrap-around can ++ * silently delete articles. Simulate the SMretrieve(RETR_STAT) ++ * check via the `gone` hashset. */ ++ if (token->type == 2) { ++ if (tombstone_present(c->gone, token)) ++ return false; ++ return true; ++ } ++ ++ /* Non-self-expiring: trust the tombstone. Not in tombstone means ++ * still alive. */ ++ return true; ++} ++ ++ ++/* Build a synthetic token whose bytes encode n and whose type byte ++ * tags whether it is from a self-expiring backend. */ ++static TOKEN ++make_token(unsigned char type, unsigned long n) ++{ ++ TOKEN t; ++ ++ memset(&t, 0, sizeof(t)); ++ t.type = type; ++ t.class = 0; ++ t.token[0] = (n >> 24) & 0xff; ++ t.token[1] = (n >> 16) & 0xff; ++ t.token[2] = (n >> 8) & 0xff; ++ t.token[3] = n & 0xff; ++ return t; ++} ++ ++ ++static char * ++make_msgid(unsigned long n) ++{ ++ char buf[64]; ++ snprintf(buf, sizeof(buf), "", n); ++ return xstrdup(buf); ++} ++ ++ ++/* Count the number of non-empty lines in the history file (one per ++ * surviving entry). */ ++static unsigned long ++count_history_lines(const char *path) ++{ ++ FILE *f; ++ char line[SMBUF]; ++ unsigned long n = 0; ++ ++ f = fopen(path, "r"); ++ if (f == NULL) ++ return 0; ++ while (fgets(line, sizeof(line), f) != NULL) { ++ if (line[0] != '\n' && line[0] != '\0') ++ n++; ++ } ++ fclose(f); ++ return n; ++} ++ ++ ++/* Helper: insert a TOKEN copy into a hashset. */ ++static void ++hash_add_token(struct hash *h, const TOKEN *token) ++{ ++ TOKEN *p = xmalloc(sizeof(TOKEN)); ++ *p = *token; ++ if (!hash_insert(h, p, p)) ++ free(p); ++} ++ ++ ++int ++main(void) ++{ ++ struct history *h; ++ struct expire_cookie cookie; ++ char tmpdir[64]; ++ char histpath[128]; ++ char newhistpath[140]; ++ TOKEN tokens[N_TOTAL]; ++ unsigned long i; ++ unsigned long base; ++ bool expire_ok; ++ struct stat sb; ++ ++ test_init(9); ++ ++ strlcpy(tmpdir, "tombstone-hisexp-XXXXXX", sizeof(tmpdir)); ++ if (mkdtemp(tmpdir) == NULL) ++ sysbail("can't create temp directory"); ++ snprintf(histpath, sizeof(histpath), "%s/history", tmpdir); ++ ++ /* Layout of the tokens array: ++ [0 .. N_KEPT) non-selfexpire keep ++ [N_KEPT .. N_KEPT+N_TOMBSTONED) non-selfexpire tomb ++ [N_KEPT+N_TOMBSTONED .. + N_SELFEXP_GONE) selfexpire gone ++ [..rest] selfexpire alive ++ */ ++ base = 0; ++ for (i = 0; i < N_KEPT; i++) ++ tokens[base + i] = make_token(1, base + i); ++ base += N_KEPT; ++ for (i = 0; i < N_TOMBSTONED; i++) ++ tokens[base + i] = make_token(1, base + i); ++ base += N_TOMBSTONED; ++ for (i = 0; i < N_SELFEXP_GONE; i++) ++ tokens[base + i] = make_token(2, base + i); ++ base += N_SELFEXP_GONE; ++ for (i = 0; i < N_SELFEXP_ALIVE; i++) ++ tokens[base + i] = make_token(2, base + i); ++ ++ /* Populate a fresh history database with all N_TOTAL entries. */ ++ h = HISopen(histpath, "hisv6", HIS_CREAT | HIS_RDWR); ++ if (h == NULL) ++ bail("can't create history at %s", histpath); ++ for (i = 0; i < N_TOTAL; i++) { ++ char *msgid = make_msgid(i); ++ if (!HISwrite(h, msgid, (time_t) 1000000 + i, (time_t) 1000000 + i, ++ (time_t) 0, &tokens[i])) ++ bail("can't write history entry %lu: %s", i, HISerror(h)); ++ free(msgid); ++ } ++ HISsync(h); ++ HISclose(h); ++ ok(1, stat(histpath, &sb) == 0 && sb.st_size > 0); ++ ++ /* Build the tombstone hashset (only the N_TOMBSTONED slice). */ ++ cookie.tombstone = tombstone_hash_create(8); ++ for (i = N_KEPT; i < N_KEPT + N_TOMBSTONED; i++) ++ hash_add_token(cookie.tombstone, &tokens[i]); ++ ok(2, hash_count(cookie.tombstone) == N_TOMBSTONED); ++ ++ /* Build the "gone" hashset for self-expiring backends (only the ++ * N_SELFEXP_GONE slice; selfexpire-alive tokens are NOT added). */ ++ cookie.gone = tombstone_hash_create(8); ++ base = N_KEPT + N_TOMBSTONED; ++ for (i = 0; i < N_SELFEXP_GONE; i++) ++ hash_add_token(cookie.gone, &tokens[base + i]); ++ ok(3, hash_count(cookie.gone) == N_SELFEXP_GONE); ++ ++ /* Reopen read-only and run HISexpire with our SELFEXPIRE-aware ++ * callback. */ ++ h = HISopen(histpath, "hisv6", HIS_RDONLY); ++ if (h == NULL) ++ bail("can't reopen history at %s", histpath); ++ snprintf(newhistpath, sizeof(newhistpath), "%s/history.new", tmpdir); ++ /* High threshold so dropped entries do not survive as remember-only ++ * records; we want a clean count of survivors. */ ++ expire_ok = HISexpire(h, newhistpath, NULL, true, &cookie, ++ (time_t) 2000000000, test_expire_cb); ++ ok(4, expire_ok); ++ HISclose(h); ++ ++ /* The new history at .n should contain exactly the ++ * survivors: N_KEPT + N_SELFEXP_ALIVE. Tombstoned and selfexpire- ++ * gone entries should be absent. */ ++ { ++ char actual[160]; ++ unsigned long expected = N_KEPT + N_SELFEXP_ALIVE; ++ snprintf(actual, sizeof(actual), "%s.n", newhistpath); ++ ok(5, count_history_lines(actual) == expected); ++ } ++ ++ /* Per-category lookup checks: kept and selfexpire-alive present; ++ * tombstoned and selfexpire-gone absent. */ ++ { ++ char actual[160]; ++ struct history *hnew; ++ unsigned long kept_present = 0; ++ unsigned long tomb_present = 0; ++ unsigned long alive_present = 0; ++ unsigned long gone_present = 0; ++ TOKEN found; ++ ++ snprintf(actual, sizeof(actual), "%s.n", newhistpath); ++ hnew = HISopen(actual, "hisv6", HIS_RDONLY); ++ if (hnew == NULL) ++ bail("can't open new history %s", actual); ++ ++ for (i = 0; i < N_KEPT; i++) { ++ char *msgid = make_msgid(i); ++ if (HISlookup(hnew, msgid, NULL, NULL, NULL, &found)) ++ kept_present++; ++ free(msgid); ++ } ++ for (i = N_KEPT; i < N_KEPT + N_TOMBSTONED; i++) { ++ char *msgid = make_msgid(i); ++ if (HISlookup(hnew, msgid, NULL, NULL, NULL, &found)) ++ tomb_present++; ++ free(msgid); ++ } ++ base = N_KEPT + N_TOMBSTONED; ++ for (i = 0; i < N_SELFEXP_GONE; i++) { ++ char *msgid = make_msgid(base + i); ++ if (HISlookup(hnew, msgid, NULL, NULL, NULL, &found)) ++ gone_present++; ++ free(msgid); ++ } ++ base += N_SELFEXP_GONE; ++ for (i = 0; i < N_SELFEXP_ALIVE; i++) { ++ char *msgid = make_msgid(base + i); ++ if (HISlookup(hnew, msgid, NULL, NULL, NULL, &found)) ++ alive_present++; ++ free(msgid); ++ } ++ ++ ok(6, kept_present == N_KEPT); ++ ok(7, tomb_present == 0); ++ ok(8, gone_present == 0); /* SELFEXPIRE branch correctly drops */ ++ ok(9, alive_present == N_SELFEXP_ALIVE); ++ HISclose(hnew); ++ } ++ ++ hash_free(cookie.tombstone); ++ hash_free(cookie.gone); ++ ++ /* Cleanup. */ ++ { ++ char cmd[128]; ++ snprintf(cmd, sizeof(cmd), "/bin/rm -rf %s", tmpdir); ++ if (system(cmd) < 0) ++ sysdiag("can't clean up %s", tmpdir); ++ } ++ ++ return 0; ++} +diff --git tests/expire/tombstone-t.c tests/expire/tombstone-t.c +--- tests/expire/tombstone-t.c 1969-12-31 17:00:00.000000000 -0700 ++++ tests/expire/tombstone-t.c 2026-05-16 12:45:04.842148000 -0700 +@@ -0,0 +1,193 @@ ++/* Unit tests for the lib/tombstone helpers used by expire to consume ++ * the deletion logs written by expireover/expirerm and innd/sm. */ ++ ++#include "portable/system.h" ++ ++#include ++#include ++#include ++#include ++#include ++ ++#include "inn/hashtab.h" ++#include "inn/libinn.h" ++#include "inn/messages.h" ++#include "inn/storage.h" ++#include "inn/tombstone.h" ++#include "tap/basic.h" ++ ++ ++static TOKEN ++make_token(unsigned char type, unsigned char class, unsigned long n) ++{ ++ TOKEN t; ++ ++ memset(&t, 0, sizeof(t)); ++ t.type = type; ++ t.class = class; ++ t.token[0] = (n >> 24) & 0xff; ++ t.token[1] = (n >> 16) & 0xff; ++ t.token[2] = (n >> 8) & 0xff; ++ t.token[3] = n & 0xff; ++ return t; ++} ++ ++ ++/* Write a tombstone-format file containing the given tokens and any ++ * extra raw lines. Returns the path; caller must unlink and free. */ ++static char * ++write_tombstone_file(const char *prefix, const TOKEN *tokens, size_t ntokens, ++ const char *const *extra_lines, size_t nextra) ++{ ++ char tmpl[64]; ++ int fd; ++ FILE *f; ++ size_t i; ++ ++ snprintf(tmpl, sizeof(tmpl), "%s-XXXXXX", prefix); ++ fd = mkstemp(tmpl); ++ if (fd < 0) ++ sysbail("can't create %s", tmpl); ++ f = fdopen(fd, "w"); ++ if (f == NULL) ++ sysbail("can't fdopen %s", tmpl); ++ for (i = 0; i < ntokens; i++) ++ fprintf(f, "%s\n", TokenToText(tokens[i])); ++ for (i = 0; i < nextra; i++) ++ fprintf(f, "%s", extra_lines[i]); ++ fclose(f); ++ return xstrdup(tmpl); ++} ++ ++ ++int ++main(void) ++{ ++ TOKEN t1, t2, t3, t4, t_other; ++ char *text; ++ char *path; ++ char *path2; ++ struct hash *h; ++ unsigned long count; ++ TOKEN parsed; ++ char *roundtrip; ++ ++ test_init(20); ++ ++ /* TokenToText / TextToToken / IsToken round trip is the foundation ++ * of the file format. */ ++ t1 = make_token(3, 0, 0xdeadbeef); ++ text = xstrdup(TokenToText(t1)); ++ parsed = TextToToken(text); ++ ok(1, IsToken(text)); ++ ok(2, memcmp(&parsed, &t1, sizeof(TOKEN)) == 0); ++ roundtrip = xstrdup(TokenToText(parsed)); ++ ok(3, strcmp(text, roundtrip) == 0); ++ free(text); ++ free(roundtrip); ++ ++ /* tombstone_hash_create gives us a hash that handles TOKEN keys. */ ++ t2 = make_token(3, 0, 1); ++ t3 = make_token(3, 0, 2); ++ t_other = make_token(3, 0, 99); ++ h = tombstone_hash_create(4); ++ { ++ TOKEN *p = xmalloc(sizeof(TOKEN)); ++ *p = t2; ++ ok(4, hash_insert(h, p, p)); ++ } ++ { ++ TOKEN *p = xmalloc(sizeof(TOKEN)); ++ *p = t3; ++ ok(5, hash_insert(h, p, p)); ++ } ++ ok(6, tombstone_present(h, &t2)); ++ ok(7, tombstone_present(h, &t3)); ++ ok(8, !tombstone_present(h, &t_other)); ++ ok(9, hash_count(h) == 2); ++ hash_free(h); ++ ++ /* tombstone_read parses a single file with mixed valid/invalid ++ * lines, skips malformed entries, and silently ignores comment ++ * lines (used for the format version header and for future ++ * metadata). This locks in the writer/reader contract for ++ * "# inn-tombstone v1\n" written by expireover. */ ++ { ++ TOKEN tokens[] = {t1, t2, t3, t1}; /* duplicate t1 */ ++ const char *extras[] = { ++ "\n", /* blank */ ++ "# inn-tombstone v1\n", /* version header */ ++ "not-a-token\n", /* malformed */ ++ "# arbitrary metadata\n", /* future-format placeholder */ ++ "\r\n", /* CRLF blank */ ++ }; ++ path = write_tombstone_file("tombstone-read", tokens, 4, extras, 5); ++ h = tombstone_hash_create(4); ++ count = tombstone_read(h, path, NULL); ++ /* Three unique tokens (t1 dedup'd) plus one valid duplicate ++ * line counted but not inserted -> count returned is 4 ++ * (lines processed) but hash has 3 unique. Comment and ++ * blank lines are skipped silently and not counted. */ ++ ok(10, h != NULL); ++ ok(11, count == 4); ++ ok(12, hash_count(h) == 3); ++ ok(13, tombstone_present(h, &t1)); ++ ok(14, tombstone_present(h, &t2)); ++ ok(15, tombstone_present(h, &t3)); ++ ok(16, !tombstone_present(h, &t_other)); ++ hash_free(h); ++ unlink(path); ++ free(path); ++ } ++ ++ /* Two-file merge with overlap -- T-C4-B. expireover.tombstone has ++ * t1, t2; cancels.tombstone has t2, t3. Loaded into one hashset, ++ * we expect 3 unique entries with t2 merged. */ ++ t4 = make_token(3, 0, 4); ++ { ++ TOKEN expireover_tokens[] = {t1, t2}; ++ TOKEN cancels_tokens[] = {t2, t3, t4}; ++ path = write_tombstone_file("tombstone-expireover", ++ expireover_tokens, 2, NULL, 0); ++ path2 = write_tombstone_file("tombstone-cancels", ++ cancels_tokens, 3, NULL, 0); ++ h = tombstone_hash_create(4); ++ tombstone_read(h, path, NULL); ++ tombstone_read(h, path2, NULL); ++ ok(17, hash_count(h) == 4); /* t1, t2, t3, t4 */ ++ ok(18, tombstone_present(h, &t1) && tombstone_present(h, &t2) ++ && tombstone_present(h, &t3) ++ && tombstone_present(h, &t4)); ++ hash_free(h); ++ unlink(path); ++ unlink(path2); ++ free(path); ++ free(path2); ++ } ++ ++ /* tombstone_rename_for_processing -- T-C4-C. Atomically renames ++ * the source file to ".processing"; subsequent open of the ++ * original path finds nothing. */ ++ { ++ TOKEN tokens[] = {t1, t2}; ++ char *snapshot; ++ struct stat sb; ++ ++ path = write_tombstone_file("tombstone-rename", tokens, 2, NULL, 0); ++ snapshot = tombstone_rename_for_processing(path); ++ ok(19, snapshot != NULL); ++ if (snapshot != NULL) { ++ /* Original path no longer exists; snapshot does. */ ++ ok(20, stat(path, &sb) < 0 && errno == ENOENT ++ && stat(snapshot, &sb) == 0); ++ unlink(snapshot); ++ free(snapshot); ++ } else { ++ ok(20, false); ++ } ++ unlink(path); /* in case rename failed */ ++ free(path); ++ } ++ ++ return 0; ++} +diff --git tests/storage/cancel-tombstone-t.c tests/storage/cancel-tombstone-t.c +--- tests/storage/cancel-tombstone-t.c 1969-12-31 17:00:00.000000000 -0700 ++++ tests/storage/cancel-tombstone-t.c 2026-05-16 12:45:04.842477000 -0700 +@@ -0,0 +1,233 @@ ++/* Unit tests for SMcanceltombstone (storage/interface.c). ++ * ++ * Verifies the multi-writer append protocol used by innd's ARTcancel ++ * and sm's -r path: the file format is one TokenToText() per line; the ++ * write is gated on innconf->expiretombstone; TOKEN_EMPTY is a no-op; ++ * concurrent appenders use fcntl POSIX locks via inn_lock_file. */ ++ ++#include "portable/system.h" ++ ++#include ++#include ++#include ++#include ++#include ++ ++#include "inn/innconf.h" ++#include "inn/libinn.h" ++#include "inn/messages.h" ++#include "inn/storage.h" ++#include "tap/basic.h" ++ ++ ++/* Build a synthetic token whose bytes encode n. Type/class chosen so ++ * IsToken accepts the textual form (the type byte must be a valid hex ++ * digit pair when round-tripped). */ ++static TOKEN ++make_token(unsigned char type, unsigned long n) ++{ ++ TOKEN t; ++ ++ memset(&t, 0, sizeof(t)); ++ t.type = type; ++ t.class = 0; ++ t.token[0] = (n >> 24) & 0xff; ++ t.token[1] = (n >> 16) & 0xff; ++ t.token[2] = (n >> 8) & 0xff; ++ t.token[3] = n & 0xff; ++ return t; ++} ++ ++ ++/* Read the entire file into a malloc'd buffer. Returns NULL on ++ * error. *out_size set to bytes read. */ ++static char * ++slurp(const char *path, size_t *out_size) ++{ ++ FILE *f; ++ struct stat sb; ++ char *buf; ++ ++ *out_size = 0; ++ if (stat(path, &sb) < 0) ++ return NULL; ++ f = fopen(path, "r"); ++ if (f == NULL) ++ return NULL; ++ buf = xmalloc(sb.st_size + 1); ++ if (fread(buf, 1, sb.st_size, f) != (size_t) sb.st_size) { ++ free(buf); ++ fclose(f); ++ return NULL; ++ } ++ buf[sb.st_size] = '\0'; ++ *out_size = sb.st_size; ++ fclose(f); ++ return buf; ++} ++ ++ ++/* Count lines in buf (number of '\n' bytes). */ ++static size_t ++count_lines(const char *buf, size_t len) ++{ ++ size_t i, n = 0; ++ for (i = 0; i < len; i++) ++ if (buf[i] == '\n') ++ n++; ++ return n; ++} ++ ++ ++int ++main(void) ++{ ++ char tmpdir[64]; ++ char *cancels_path; ++ TOKEN t1, t2, t3, empty; ++ char *contents; ++ size_t size; ++ struct stat sb; ++ ++ test_init(15); ++ ++ /* Set up a temporary pathdb. */ ++ strlcpy(tmpdir, "cancel-tombstone-XXXXXX", sizeof(tmpdir)); ++ if (mkdtemp(tmpdir) == NULL) ++ sysbail("can't create temp directory"); ++ ++ /* Initialize a minimal innconf so SMcanceltombstone has somewhere ++ * to write. We need the fields it consults: expiretombstone, ++ * groupbaseexpiry (both required), and pathdb. Other code paths ++ * in this binary read additional fields, so allocate via xcalloc ++ * to zero everything. */ ++ innconf = xcalloc(1, sizeof(*innconf)); ++ innconf->pathdb = xstrdup(tmpdir); ++ innconf->expiretombstone = true; ++ innconf->groupbaseexpiry = true; ++ ++ cancels_path = concatpath(tmpdir, "cancels.tombstone"); ++ ++ t1 = make_token(3, 0xdeadbeef); ++ t2 = make_token(3, 0xcafebabe); ++ t3 = make_token(3, 0x12345678); ++ memset(&empty, 0, sizeof(empty)); ++ empty.type = TOKEN_EMPTY; ++ ++ /* 1. First call creates the file with one line. */ ++ SMcanceltombstone(t1); ++ contents = slurp(cancels_path, &size); ++ ok(1, contents != NULL); ++ ok(2, count_lines(contents, size) == 1); ++ ok(3, contents != NULL && strstr(contents, TokenToText(t1)) != NULL); ++ free(contents); ++ ++ /* 2. Second call appends; file now has two distinct token lines. */ ++ SMcanceltombstone(t2); ++ contents = slurp(cancels_path, &size); ++ ok(4, contents != NULL && count_lines(contents, size) == 2); ++ ok(5, contents != NULL && strstr(contents, TokenToText(t1)) != NULL ++ && strstr(contents, TokenToText(t2)) != NULL); ++ free(contents); ++ ++ /* 3. expiretombstone=false: third call is a no-op. */ ++ innconf->expiretombstone = false; ++ SMcanceltombstone(t3); ++ contents = slurp(cancels_path, &size); ++ ok(6, contents != NULL && count_lines(contents, size) == 2); ++ ok(7, contents != NULL && strstr(contents, TokenToText(t3)) == NULL); ++ free(contents); ++ innconf->expiretombstone = true; ++ ++ /* 4. TOKEN_EMPTY: no-op (not a real article). */ ++ SMcanceltombstone(empty); ++ contents = slurp(cancels_path, &size); ++ ok(8, contents != NULL && count_lines(contents, size) == 2); ++ free(contents); ++ ++ /* 5. After write completes, the lock has been released; another ++ * process (here, just us reusing the same path) can acquire the ++ * exclusive write lock without contention. */ ++ { ++ int fd = open(cancels_path, O_RDWR); ++ ok(9, fd >= 0); ++ if (fd >= 0) { ++ ok(10, inn_lock_file(fd, INN_LOCK_WRITE, false)); ++ close(fd); ++ } else { ++ ok(10, false); ++ } ++ } ++ ++ /* 6. Each line round-trips: parsing each line through TextToToken ++ * yields the original TOKEN. Read the file and verify. */ ++ contents = slurp(cancels_path, &size); ++ if (contents != NULL) { ++ char *p = contents; ++ char *line_end; ++ TOKEN parsed; ++ bool found_t1 = false; ++ bool found_t2 = false; ++ ++ while ((line_end = strchr(p, '\n')) != NULL) { ++ *line_end = '\0'; ++ if (IsToken(p)) { ++ parsed = TextToToken(p); ++ if (memcmp(&parsed, &t1, sizeof(TOKEN)) == 0) ++ found_t1 = true; ++ if (memcmp(&parsed, &t2, sizeof(TOKEN)) == 0) ++ found_t2 = true; ++ } ++ p = line_end + 1; ++ } ++ ok(11, found_t1); ++ ok(12, found_t2); ++ free(contents); ++ } else { ++ ok(11, false); ++ ok(12, false); ++ } ++ ++ /* 7. File mode should be 0664 (created with O_CREAT, mode 0664). ++ * Check the stored mode bits. Allow umask-stripped variants. */ ++ if (stat(cancels_path, &sb) == 0) { ++ mode_t mode = sb.st_mode & 0777; ++ /* The file was created with mode 0664 but umask may have ++ * stripped write bits. Verify at least owner+group readable. */ ++ ok(13, (mode & 0640) == 0640); ++ } else { ++ ok(13, false); ++ } ++ ++ /* 8. Calling SMcanceltombstone with an unwritable pathdb does ++ * not crash (best-effort failure path), does not create any ++ * file outside the configured location, and returns false. ++ * Simulate by pointing pathdb at a non-existent directory. */ ++ { ++ char *saved_pathdb = innconf->pathdb; ++ char *bad_path; ++ struct stat sb_bad; ++ bool result; ++ innconf->pathdb = xstrdup("/nonexistent-tombstone-test-dir"); ++ result = SMcanceltombstone(t3); ++ bad_path = concatpath(innconf->pathdb, "cancels.tombstone"); ++ ok(14, !result && stat(bad_path, &sb_bad) < 0 ++ && errno == ENOENT); ++ free(bad_path); ++ free(innconf->pathdb); ++ innconf->pathdb = saved_pathdb; ++ } ++ ++ /* Cleanup. */ ++ if (unlink(cancels_path) < 0 && errno != ENOENT) ++ sysdiag("can't unlink %s", cancels_path); ++ free(cancels_path); ++ if (rmdir(tmpdir) < 0) ++ sysdiag("can't rmdir %s", tmpdir); ++ /* Verify cleanup actually removed the directory. */ ++ ok(15, stat(tmpdir, &sb) < 0 && errno == ENOENT); ++ free(innconf->pathdb); ++ free(innconf); ++ ++ return 0; ++} diff --git a/news/inn/Makefile b/news/inn/Makefile index 7f33773b7756..ee7b17d12a4f 100644 --- a/news/inn/Makefile +++ b/news/inn/Makefile @@ -1,246 +1,240 @@ PORTNAME?= inn PORTVERSION?= 2.7.3 PORTREVISION?= 1 CATEGORIES= news MASTER_SITES= ISC MASTER_SITE_SUBDIR?= ${PORTNAME} MAINTAINER= kbowling@FreeBSD.org COMMENT= InterNetNews -- the Internet meets Netnews WWW= https://www.eyrie.org/~eagle/software/inn/ LICENSE= GPLv2 ISCL LICENSE_COMB= multi LICENSE_FILE_ISCL= ${WRKSRC}/LICENSE BUILD_DEPENDS= p5-GD>=0:graphics/p5-GD \ p5-MIME-Tools>=0:mail/p5-MIME-Tools RUN_DEPENDS:= ${BUILD_DEPENDS} USES= cpe gmake libtool perl5 uidfix CPE_VENDOR= isc USE_RC_SUBR= innd .ifnmake portclippy VARBASE?= /var .endif .include "Makefile.layout" GNU_CONFIGURE= yes GNU_CONFIGURE_PREFIX= ${INN_NEWSBASE} # Point to base system mailwrapper to avoid configure failure in jails CONFIGURE_ARGS+= --datarootdir=${INN_SHAREDIR} \ --sysconfdir=${INN_CONFDIR} \ --with-db-dir=${INN_DBDIR} \ --with-log-dir=${INN_LOGDIR} \ --with-perl \ --with-run-dir=${INN_RUNDIR} \ --with-sendmail=/usr/sbin/sendmail \ --with-spool-dir=${INN_NEWSSPOOL} \ --with-tmp-dir=${INN_TMPDIR} CONFIGURE_ENV+= LOCALBASE=${LOCALBASE} CONFIGURE_ENV+= ac_cv_prog_AWK="${AWK}" CONFLICTS?= inn-current SUB_FILES= pkg-install SUB_LIST+= EGDIR="${EXAMPLESDIR:C,^${PREFIX},\\$\\${PREFIX},}" INNLIB_LONG?= 9.0.2 LIBVER_LONG?= 3.0.9 LIBSTORAGEVER_LONG?= 3.1.3 CONFIG_FILES+= actsync.cfg actsync.ign buffindexed.conf control.ctl \ control.ctl.local cycbuff.conf distrib.pats distributions \ expire.ctl inn-radius.conf inn-secrets.conf incoming.conf \ inn.conf innfeed.conf innreport.conf innshellvars.local \ innshellvars.pl.local innshellvars.tcl.local innwatch.ctl \ localgroups moderators news2mail.cf newsfeeds nocem.ctl \ nnrpd.track nntpsend.ctl ovdb.conf passwd.nntp readers.conf \ send-uucp.cf storage.conf subscriptions .if ${BRANCH:U} == 2.8 PLIST_SUB+= SNAPSHOT="" .else PLIST_SUB+= SNAPSHOT="@comment " .endif PLIST_SUB+= ETCFILES="${CONFIG_FILES}" \ INNLIB=${INNLIB_LONG:R:R} \ INNLIB_LONG=${INNLIB_LONG} \ LIBSTORAGEVER=${LIBSTORAGEVER_LONG:R:R} \ LIBSTORAGEVER_LONG=${LIBSTORAGEVER_LONG} \ LIBVER=${LIBVER_LONG:R:R} \ LIBVER_LONG=${LIBVER_LONG} PORTDOCS= CONTRIBUTORS HACKING INSTALL LICENSE MANIFEST NEWS README TODO OPTIONS_DEFINE= BERKELEYDB BLOCKLISTD DOCS EXAMPLES GNUPG KERBEROS \ KEYWORDS LARGE_FILES OPENSSL PYTHON SASL SETGID_INEWS \ SQLITE TAGGED_HASH UUCP_RNEWS OPTIONS_DEFAULT= BERKELEYDB BLOCKLISTD GNUPG KEYWORDS LARGE_FILES \ OPENSSL PYTHON SETGID_INEWS SQLITE UUCP_RNEWS .if ${BRANCH:U} == 2.8 OPTIONS_DEFINE+= EXPERIMENTAL .endif OPTIONS_SUB= yes BERKELEYDB_DESC= Enable BerkeleyDB (for ovdb overview method) BLOCKLISTD_DESC= Enable blocklistd support EXPERIMENTAL_DESC= Experimental upstream patches (ovsqlite, bloom filter) GNUPG_DESC= GnuPG support (for pgpverify control message) KERBEROS_DESC= Enable Kerberos v5 (for auth_krb5) KEYWORDS_DESC= Automatic keyword generation support LARGE_FILES_DESC= Support for files larger than 2GB OPENSSL_DESC= Enable OpenSSL (for NNTP over TLS/SSL support) PYTHON_DESC= Enable support for embedded Python SASL_DESC= Enable SASL (for imapfeed authentication) SETGID_INEWS_DESC= Enable posting articles locally for all users SQLITE_DESC= Enable SQLite (for ovsqlite overview method) TAGGED_HASH_DESC= Use tagged hash table for history UUCP_RNEWS_DESC= Install rnews suitable for use with net/freebsd-uucp BERKELEYDB_USES= bdb BERKELEYDB_CONFIGURE_ON= --with-bdb-include=${BDB_INCLUDE_DIR} \ --with-bdb-lib=${LOCALBASE}/lib \ --with-bdb=${LOCALBASE} .if ${BRANCH:U} == 2.8 && ${OSVERSION:U} >= 1500000 BLOCKLISTD_CONFIGURE_WITH= blocklist .else BLOCKLISTD_CONFIGURE_WITH= blacklist .endif # Although pgpverify needs gpgv/gpgv2, # checking 'gpg' is the shortest way to check any version of GnuPG installed. GNUPG_BUILD_DEPENDS= gpg:security/gnupg GNUPG_RUN_DEPENDS= gpg:security/gnupg KERBEROS_USES= gssapi KERBEROS_CONFIGURE_ON= --with-krb5=${GSSAPIBASEDIR} KERBEROS_CONFIGURE_OFF= --without-krb5 KEYWORDS_CONFIGURE_ENABLE= keywords LARGE_FILES_PREVENTS= TAGGED_HASH LARGE_FILES_PREVENTS_MSG= cannot be compiled with both LARGE_FILES and \ TAGGED_HASH LARGE_FILES_CONFIGURE_ENABLE= largefiles OPENSSL_USES= ssl OPENSSL_CONFIGURE_ON= --with-openssl=${OPENSSLBASE} PYTHON_USES= gettext-runtime python PYTHON_CONFIGURE_ON= --with-python SASL_LIB_DEPENDS= libsasl2.so:security/cyrus-sasl2 SASL_CONFIGURE_ON= --with-sasl=${LOCALBASE} SETGID_INEWS_CONFIGURE_ENABLE= setgid-inews SETGID_INEWS_PLIST_SUB= MODE_INEWS="2555" SETGID_INEWS_PLIST_SUB_OFF= MODE_INEWS="" SQLITE_USES= sqlite SQLITE_CONFIGURE_ON= --with-sqlite3=${LOCALBASE} SQLITE_CONFIGURE_OFF= --without-sqlite3 SQLITE_VARS= CONFIG_FILES+=ovsqlite.conf TAGGED_HASH_CONFIGURE_ENABLE= tagged-hash TAGGED_HASH_SUB_LIST= DBZ_FILES="dir pag" TAGGED_HASH_SUB_LIST_OFF= DBZ_FILES="dir hash index" TAGGED_HASH_PLIST_SUB= WITH_TAGGED_HASH="" \ WITHOUT_TAGGED_HASH="@comment " TAGGED_HASH_PLIST_SUB_OFF= WITH_TAGGED_HASH="@comment " \ WITHOUT_TAGGED_HASH="" UUCP_RNEWS_RUN_DEPENDS= uucp:net/freebsd-uucp UUCP_RNEWS_CONFIGURE_ENABLE= uucp-rnews -EXPERIMENTAL_PATCH_SITES= https://github.com/InterNetNews/inn/commit/ -EXPERIMENTAL_PATCHFILES= cf4a8ecac2f3149505f9a55cdd95383e9b3f725b.patch:-p1 \ - 0d84255b417c89219225f0a034fd4c068d9749a0.patch:-p1 \ - 05d70a17c5720be25fcd957c8ab7a0c12a64d30b.patch:-p1 \ - 0b308fdd3f6255e8c81a3deaf1bad08cb1fe8f1d.patch:-p1 \ - 755ab163eb124658959a8e4168732e6a7f7dc300.patch:-p1 \ - d23b3f84aebe3fb49af0b42b7cfaa7a7ebae5a26.patch:-p1 \ - 919cf653b54c72ef4bf1563f3fb6bbdf9a193a3a.patch:-p1 \ - 93a36682f7ba9d013d07859882c2b8c501da249e.patch:-p1 \ - d0c379ee3f325f4a80dada6c3129c23496b1c83e.patch:-p1 +EXPERIMENTAL_EXTRA_PATCHES= ${PKGDIR}/files/pr337.patch \ + ${PKGDIR}/files/pr338.patch \ + ${PKGDIR}/files/pr339.patch \ + ${PKGDIR}/files/pr340.patch EXPERIMENTAL_PLIST_SUB= EXPERIMENTAL="" EXPERIMENTAL_PLIST_SUB_OFF= EXPERIMENTAL="@comment " .include GID?= ${.MAKE.GID} post-extract-EXPERIMENTAL-on: @${CP} ${PKGDIR}/files/gitignore ${WRKSRC}/.gitignore post-patch: @${REINPLACE_CMD} -E 's!\$$[{(]PATHETC[})]!${EXAMPLESDIR}!g' \ ${WRKSRC}/site/Makefile .if ${PORT_OPTIONS:MBERKELEYDB} @${REINPLACE_CMD} -e 's!-ldb!-ldb-${BDB_VER}!' ${WRKSRC}/configure .endif .if ${BRANCH:U} == 2.8 . if ${OPSYS} == FreeBSD && ${OSVERSION} < 1500000 @${FIND} ${WRKSRC} -type f -exec ${SED} -i '' \ -e 's/BLOCKLIST/BLACKLIST/g' \ -e 's/Blocklist/Blacklist/g' \ -e 's/blocklist/blacklist/g' {} + . endif .else . if ${OPSYS} == FreeBSD && ${OSVERSION} >= 1500000 @${FIND} ${WRKSRC} -type f -exec ${SED} -i '' \ -e 's/BLACKLIST/BLOCKLIST/g' \ -e 's/Blacklist/Blocklist/g' \ -e 's/blacklist/blocklist/g' {} + . endif .endif pre-install: @${MKDIR} ${STAGEDIR}${INN_DBDIR} \ ${STAGEDIR}${EXAMPLESDIR} \ ${STAGEDIR}${EXAMPLESDIR}/db @${REINPLACE_CMD} -e '/^RUNASUSER *=/s/=.*/= ${UID}/' \ -e '/^RUNASGROUP *=/s/=.*/= ${GID}/' \ -e '/^RNEWSGROUP *=/s/=.*/= ${GID}/' \ ${WRKSRC}/Makefile.global post-build: @${FIND} ${WRKSRC} -name "inn.conf" \ | ${XARGS} ${REINPLACE_CMD} -e 's/^pathhost:.*/pathhost: host.example.com/' post-install: ${FIND} ${STAGEDIR}${INN_NEWSBASE} -type f \ \( -perm +111 -o -name '*.so.*' \) ! -name '*.a' \ -exec ${STRIP_CMD} {} + 2>/dev/null || ${TRUE} ${MKDIR} ${STAGEDIR}${INN_NEWSSPOOL} ${MKDIR} ${STAGEDIR}${INN_CONFDIR} .for FILE in ${CONFIG_FILES} ${CP} -p ${STAGEDIR}${EXAMPLESDIR}/${FILE} ${STAGEDIR}${INN_CONFDIR}/${FILE}.sample .endfor ${MKDIR} ${STAGEDIR}${INN_DBDIR} .for FILE in active active.times history newsgroups ${MV} ${STAGEDIR}${INN_DBDIR}/${FILE} \ ${STAGEDIR}${INN_DBDIR}/${FILE}.sample .endfor ${TOUCH} ${STAGEDIR}${INN_DBDIR}/history.dir.sample .if ${PORT_OPTIONS:MTAGGED_HASH} ${TOUCH} ${STAGEDIR}${INN_DBDIR}/history.pag.sample .else ${TOUCH} ${STAGEDIR}${INN_DBDIR}/history.hash.sample ${TOUCH} ${STAGEDIR}${INN_DBDIR}/history.index.sample .endif .for FILE in filter_innd.pl filter_innd.py filter_nnrpd.pl nnrpd_access.pl \ nnrpd_access.py nnrpd_auth.pl nnrpd_auth.py nnrpd_dynamic.py \ startup_innd.pl ${MV} ${STAGEDIR}${INN_NEWSBASE}/bin/filter/${FILE} \ ${STAGEDIR}${INN_NEWSBASE}/bin/filter/${FILE}.sample .endfor ${MKDIR} ${STAGEDIR}${DOCSDIR} .for FILE in ${PORTDOCS} ${INSTALL_DATA} ${WRKSRC}/${FILE} ${STAGEDIR}${DOCSDIR} .endfor .include