Merge remote-tracking branch 'upstream/master'

.github/PULL_REQUEST_TEMPLATE.md

@@ -1,10 +1,10 @@
 > [!IMPORTANT]
-> Open for merging new PRs until the next PR freeze date is confirmed!
+> 24.11 FREEZE NOVEMBER 7TH: Non-bugfix PRs not ready by this date will wait for 25.02.
 
 ## Checklist
 Before submitting the PR, ensure the following tasks are completed. If an item is not applicable to your PR, please mark it as checked:
 
 - [ ] The changelog has been updated in the relevant commit(s) according to the [guidelines](https://docs.corelightning.org/docs/coding-style-guidelines#changelog-entries-in-commit-messages).
 - [ ] Tests have been added or modified to reflect the changes.
 - [ ] Documentation has been reviewed and updated as needed.
-- [ ] Related issues have been listed and linked, including any that this PR closes.
+- [ ] Related issues have been listed and linked, including any that this PR closes.

.github/workflows/repro.yml

@@ -0,0 +1,78 @@
+---
+# https://docs.corelightning.org/docs/repro
+name: Repro Build Nightly
+on:
+  # 05:00 Berlin, 03:00 UTC, 23:00 New York, 20:00 Los Angeles
+  schedule:
+    - cron: "0 3 * * *"
+  workflow_dispatch:
+
+jobs:
+  ubuntu:
+    name: "Ubuntu repro build: ${{ matrix.version }}"
+    runs-on: ubuntu-22.04
+    strategy:
+      fail-fast: false    # Let each build finish.
+      matrix:
+        version: ['focal', 'jammy', 'noble']
+    steps:
+      - name: Git checkout
+        uses: actions/checkout@v4
+
+      - name: Build environment setup
+        run: |
+          echo "Building base image for ${{ matrix.version }}"
+          sudo docker run --rm -v $(pwd):/build ubuntu:${{ matrix.version }} bash -c "\
+              apt-get update && \
+              apt-get install -y debootstrap && \
+              debootstrap ${{ matrix.version }} /build/${{ matrix.version }}"
+          sudo tar -C ${{ matrix.version }} -c . | docker import - ${{ matrix.version }}
+
+      - name: Builder image setup
+        run: docker build -t cl-repro-${{ matrix.version }} - < contrib/reprobuild/Dockerfile.${{ matrix.version }}
+
+      - name: Build using the builder image and store Git state
+        run: |
+          # Create release directory.
+          mkdir $GITHUB_WORKSPACE/release
+
+          # Perform the repro build.
+          docker run --name cl-build -v $GITHUB_WORKSPACE:/repo -e FORCE_MTIME=$(date +%F) -t cl-repro-${{ matrix.version }}
+
+          # Commit the image in order to inspect the build later.
+          docker commit cl-build cl-release
+
+          # Inspect the version.
+          docker run --rm -v $GITHUB_WORKSPACE:/repo -t cl-release bash -c "make version > /repo/release/version.txt"
+
+          # Inspect the Git tree state.
+          docker run --rm -v $GITHUB_WORKSPACE:/repo -t cl-release bash -c "\
+              git --no-pager status > /repo/release/git.log && \
+              git --no-pager diff >> /repo/release/git.log"
+
+          # Change permissions on the release files for access by the runner environment.
+          sudo chown -R runner $GITHUB_WORKSPACE/release
+
+      - name: Assert clean version and release
+        run: |
+          echo 'Version:'
+          cat release/version.txt
+          echo -e
+
+          releasefile=$(ls release/clightning-*)
+          echo 'Release file:'
+          ls -al release/clightning-*
+          echo -e
+
+          if [ -n "$(cat release/version.txt | sed -n '/-modded/p')" ] || \
+             [ -n "$(echo $releasefile | sed -n '/-modded/p')" ]
+          then
+            echo "Git Status and Diff:"
+            cat release/git.log
+            echo -e
+
+            echo 'Error: release modded / dirty tree.'
+            exit 1
+          else
+            echo 'Success! Clean release.'
+          fi

Makefile

@@ -840,6 +840,7 @@ install: install-program install-data
 TESTBINS = \
 	$(CLN_PLUGIN_EXAMPLES) \
 	tests/plugins/test_libplugin \
+	tests/plugins/channeld_fakenet \
 	tests/plugins/test_selfdisable_after_getmanifest \
 	tools/hsmtool
 
@@ -903,8 +904,11 @@ installcheck: all-programs
 	fi
 	@rm -rf testinstall || true
 
+version:
+	@echo ${VERSION}
+
 .PHONY: installdirs install-program install-data install uninstall \
-	installcheck ncc bin-tarball show-flags
+	installcheck ncc bin-tarball show-flags version
 
 # Make a tarball of opt/clightning/, optionally with label for distribution.
 ifneq ($(VERSION),)

channeld/channeld.c

@@ -5760,6 +5760,7 @@ static void req_in(struct peer *peer, const u8 *msg)
 	case WIRE_CHANNELD_SPLICE_STATE_ERROR:
 	case WIRE_CHANNELD_LOCAL_ANCHOR_INFO:
 	case WIRE_CHANNELD_REESTABLISHED:
+	case WIRE_CHANNELD_DEV_PEER_SHACHAIN:
 		break;
 	}
 	master_badmsg(-1, msg);

channeld/channeld_wire.csv

@@ -347,3 +347,7 @@ msgdata,channeld_upgraded,new_type,channel_type,
 # Tell peer about our latest and greatest blockheight.
 msgtype,channeld_blockheight,1012
 msgdata,channeld_blockheight,blockheight,u32,
+
+# Tell channeld about peer's shachain seed.
+msgtype,channeld_dev_peer_shachain,1013
+msgdata,channeld_dev_peer_shachain,seed,sha256,

channeld/test/run-commit_tx.c

@@ -13,6 +13,7 @@ static bool print_superverbose;
 #include <ccan/err/err.h>
 #include <ccan/str/hex/hex.h>
 #include <common/channel_id.h>
+#include <common/daemon.h>
 #include <common/key_derive.h>
 #include <common/setup.h>
 #include <common/status.h>
@@ -34,6 +35,9 @@ void fromwire_node_id(const u8 **cursor UNNEEDED, size_t *max UNNEEDED, struct n
 /* Generated stub for pubkey_from_node_id */
 bool pubkey_from_node_id(struct pubkey *key UNNEEDED, const struct node_id *id UNNEEDED)
 { fprintf(stderr, "pubkey_from_node_id called!\n"); abort(); }
+/* Generated stub for send_backtrace */
+void send_backtrace(const char *why UNNEEDED)
+{ fprintf(stderr, "send_backtrace called!\n"); abort(); }
 /* Generated stub for status_fmt */
 void status_fmt(enum log_level level UNNEEDED,
 		const struct node_id *peer UNNEEDED,

channeld/test/run-full_channel.c

@@ -13,6 +13,7 @@
 #include <bitcoin/pubkey.h>
 #include <bitcoin/tx.h>
 #include <ccan/err/err.h>
+#include <common/daemon.h>
 #include <common/setup.h>
 
 /* AUTOGENERATED MOCKS START */
@@ -31,6 +32,9 @@ void memleak_scan_htable(struct htable *memtable UNNEEDED, const struct htable *
 /* Generated stub for pubkey_from_node_id */
 bool pubkey_from_node_id(struct pubkey *key UNNEEDED, const struct node_id *id UNNEEDED)
 { fprintf(stderr, "pubkey_from_node_id called!\n"); abort(); }
+/* Generated stub for send_backtrace */
+void send_backtrace(const char *why UNNEEDED)
+{ fprintf(stderr, "send_backtrace called!\n"); abort(); }
 /* Generated stub for status_failed */
 void status_failed(enum status_failreason code UNNEEDED,
 		   const char *fmt UNNEEDED, ...)

cli/test/run-human-mode.c

@@ -4,6 +4,7 @@
 #include <common/bigsize.h>
 #include <common/channel_id.h>
 #include <common/configvar.h>
+#include <common/daemon.h>
 #include <common/json_stream.h>
 #include <common/node_id.h>
 #include <common/setup.h>
@@ -92,6 +93,9 @@ bool log_level_parse(const char *levelstr UNNEEDED, size_t len UNNEEDED,
 /* Generated stub for pubkey_from_node_id */
 bool pubkey_from_node_id(struct pubkey *key UNNEEDED, const struct node_id *id UNNEEDED)
 { fprintf(stderr, "pubkey_from_node_id called!\n"); abort(); }
+/* Generated stub for send_backtrace */
+void send_backtrace(const char *why UNNEEDED)
+{ fprintf(stderr, "send_backtrace called!\n"); abort(); }
 /* Generated stub for towire_amount_msat */
 void towire_amount_msat(u8 **pptr UNNEEDED, const struct amount_msat msat UNNEEDED)
 { fprintf(stderr, "towire_amount_msat called!\n"); abort(); }

cli/test/run-large-input.c

@@ -4,6 +4,7 @@
 #include <common/bigsize.h>
 #include <common/channel_id.h>
 #include <common/configvar.h>
+#include <common/daemon.h>
 #include <common/json_stream.h>
 #include <common/node_id.h>
 #include <common/setup.h>
@@ -92,6 +93,9 @@ bool log_level_parse(const char *levelstr UNNEEDED, size_t len UNNEEDED,
 /* Generated stub for pubkey_from_node_id */
 bool pubkey_from_node_id(struct pubkey *key UNNEEDED, const struct node_id *id UNNEEDED)
 { fprintf(stderr, "pubkey_from_node_id called!\n"); abort(); }
+/* Generated stub for send_backtrace */
+void send_backtrace(const char *why UNNEEDED)
+{ fprintf(stderr, "send_backtrace called!\n"); abort(); }
 /* Generated stub for towire_amount_msat */
 void towire_amount_msat(u8 **pptr UNNEEDED, const struct amount_msat msat UNNEEDED)
 { fprintf(stderr, "towire_amount_msat called!\n"); abort(); }

cli/test/run-remove-hint.c

@@ -4,6 +4,7 @@
 #include <common/bigsize.h>
 #include <common/channel_id.h>
 #include <common/configvar.h>
+#include <common/daemon.h>
 #include <common/json_stream.h>
 #include <common/node_id.h>
 #include <common/setup.h>
@@ -95,6 +96,9 @@ bool log_level_parse(const char *levelstr UNNEEDED, size_t len UNNEEDED,
 /* Generated stub for pubkey_from_node_id */
 bool pubkey_from_node_id(struct pubkey *key UNNEEDED, const struct node_id *id UNNEEDED)
 { fprintf(stderr, "pubkey_from_node_id called!\n"); abort(); }
+/* Generated stub for send_backtrace */
+void send_backtrace(const char *why UNNEEDED)
+{ fprintf(stderr, "send_backtrace called!\n"); abort(); }
 /* Generated stub for towire_amount_msat */
 void towire_amount_msat(u8 **pptr UNNEEDED, const struct amount_msat msat UNNEEDED)
 { fprintf(stderr, "towire_amount_msat called!\n"); abort(); }

common/msg_queue.c

@@ -1,13 +1,17 @@
 #include "config.h"
 #include <assert.h>
 #include <ccan/cast/cast.h>
+#include <ccan/membuf/membuf.h>
+#include <common/daemon.h>
 #include <common/msg_queue.h>
 #include <common/utils.h>
 #include <wire/wire.h>
 
+static bool warned_once;
+
 struct msg_queue {
 	bool fd_passing;
-	const u8 **q;
+	MEMBUF(const u8 *) mb;
 };
 
 static int extract_fd(const u8 *msg)
@@ -24,18 +28,29 @@ static int extract_fd(const u8 *msg)
 /* Close any fds left in queue! */
 static void destroy_msg_queue(struct msg_queue *q)
 {
-	for (size_t i = 0; i < tal_count(q->q); i++) {
-		int fd = extract_fd(q->q[i]);
+	const u8 **elems = membuf_elems(&q->mb);
+	for (size_t i = 0; i < membuf_num_elems(&q->mb); i++) {
+		int fd = extract_fd(elems[i]);
 		if (fd != -1)
 			close(fd);
 	}
 }
 
+/* Realloc helper for tal membufs */
+static void *membuf_tal_realloc(struct membuf *mb, void *rawelems,
+				size_t newsize)
+{
+	char *p = rawelems;
+
+	tal_resize(&p, newsize);
+	return p;
+}
+
 struct msg_queue *msg_queue_new(const tal_t *ctx, bool fd_passing)
 {
 	struct msg_queue *q = tal(ctx, struct msg_queue);
 	q->fd_passing = fd_passing;
-	q->q = tal_arr(q, const u8 *, 0);
+	membuf_init(&q->mb, tal_arr(q, const u8 *, 0), 0, membuf_tal_realloc);
 
 	if (q->fd_passing)
 		tal_add_destructor(q, destroy_msg_queue);
@@ -44,15 +59,23 @@ struct msg_queue *msg_queue_new(const tal_t *ctx, bool fd_passing)
 
 static void do_enqueue(struct msg_queue *q, const u8 *add TAKES)
 {
-	tal_arr_expand(&q->q, tal_dup_talarr(q, u8, add));
+	const u8 **msg = membuf_add(&q->mb, 1);
+
+	*msg = tal_dup_talarr(q, u8, add);
+
+	if (!warned_once && msg_queue_length(q) > 250000) {
+		/* Can cause re-entry, so set flag first! */
+		warned_once = true;
+		send_backtrace("excessive queue length");
+	}
 
 	/* In case someone is waiting */
 	io_wake(q);
 }
 
 size_t msg_queue_length(const struct msg_queue *q)
 {
-	return tal_count(q->q);
+	return membuf_num_elems(&q->mb);
 }
 
 void msg_enqueue(struct msg_queue *q, const u8 *add)
@@ -73,16 +96,12 @@ void msg_enqueue_fd(struct msg_queue *q, int fd)
 
 const u8 *msg_dequeue(struct msg_queue *q)
 {
-	size_t n = tal_count(q->q);
-	const u8 *msg;
+	size_t n = msg_queue_length(q);
 
 	if (!n)
 		return NULL;
 
-	msg = q->q[0];
-	memmove(q->q, q->q + 1, sizeof(*q->q) * (n-1));
-	tal_resize(&q->q, n-1);
-	return msg;
+	return membuf_consume(&q->mb, 1)[0];
 }
 
 int msg_extract_fd(const struct msg_queue *q, const u8 *msg)

common/status.c

@@ -151,15 +151,15 @@ void status_vfmt(enum log_level level,
 
 	/* We only suppress async debug msgs.  IO messages are even spammier
 	 * but they only occur when explicitly asked for */
-	if (level == LOG_DBG && status_conn) {
+	if ((level == LOG_DBG || level == LOG_TRACE) && status_conn) {
 		size_t qlen = daemon_conn_queue_length(status_conn);
 
 		/* Once suppressing, we keep suppressing until we're empty */
 		if (traces_suppressed && qlen == 0) {
 			size_t n = traces_suppressed;
 			traces_suppressed = 0;
 			/* Careful: recursion! */
-			status_debug("...[%zu debug messages suppressed]...", n);
+			status_debug("...[%zu debug/trace messages suppressed]...", n);
 		} else if (traces_suppressed || qlen > TRACE_QUEUE_LIMIT) {
 			traces_suppressed++;
 			return;

contrib/giantnode.py

@@ -41,13 +41,9 @@
 
 args = parser.parse_args()
 
-nodes = (LightningRpc('/tmp/l1-regtest/regtest/lightning-rpc'),
-         LightningRpc('/tmp/l2-regtest/regtest/lightning-rpc'),
-         LightningRpc('/tmp/l3-regtest/regtest/lightning-rpc'))
-
-nodes = (LightningRpc('/tmp/l1-regtest/regtest/lightning-rpc'),
-         LightningRpc('/tmp/l2-regtest/regtest/lightning-rpc'),
-         LightningRpc('/tmp/l3-regtest/regtest/lightning-rpc'))
+nodes = (LightningRpc('/tmp/l1/regtest/lightning-rpc'),
+         LightningRpc('/tmp/l2/regtest/lightning-rpc'),
+         LightningRpc('/tmp/l3/regtest/lightning-rpc'))
 
 # Convenient aliases
 l1 = nodes[0]
@@ -59,9 +55,9 @@
         print("""
 Bookkeeper is running on l1, will slow things down!  Run this:
 
-echo 'disable-plugin=bookkeeper' >> /tmp/l1-regtest/regtest/config
-echo 'disable-plugin=bookkeeper' >> /tmp/l2-regtest/regtest/config
-echo 'disable-plugin=bookkeeper' >> /tmp/l3-regtest/regtest/config
+echo 'disable-plugin=bookkeeper' >> /tmp/l1/regtest/config
+echo 'disable-plugin=bookkeeper' >> /tmp/l2/regtest/config
+echo 'disable-plugin=bookkeeper' >> /tmp/l3/regtest/config
 stop_ln
 start_ln 3
 """)

contrib/reprobuild/Dockerfile.focal

@@ -45,12 +45,12 @@ RUN git clone https://github.com/pyenv/pyenv.git /root/.pyenv && \
     libsqlite3-dev \
     libssl-dev \
     zlib1g-dev && \
-    pyenv install 3.8.0 && \
-    pyenv global 3.8.0
+    pyenv install 3.10.0 && \
+    pyenv global 3.10.0
 
 RUN wget https://bootstrap.pypa.io/get-pip.py -O /tmp/get-pip.py && python3 /tmp/get-pip.py \
     && rm /tmp/get-pip.py \
-    && pip install poetry mako grpcio-tools
+    && pip install poetry mako grpcio-tools==1.62.2
 
 RUN wget https://sh.rustup.rs -O rustup-install.sh && \
     bash rustup-install.sh --default-toolchain none --quiet -y && \

devtools/gossmap-compress.c

@@ -278,7 +278,7 @@ static u64 get_delay(struct gossmap *gossmap,
 	return chan->half[dir].delay;
 }
 
-static void pubkey_for_node(size_t nodeidx, struct pubkey *key,
+static void pubkey_for_node(u64 nodeidx, struct pubkey *key,
 			    const struct pubkey **node_ids)
 {
 	struct secret seckey;