From 5a639a40ca3cacfa968ae525dc962bcb14fa77e1 Mon Sep 17 00:00:00 2001 From: Joseph Goulden Date: Mon, 16 Sep 2024 09:18:06 +0100 Subject: [PATCH 1/6] build: Add simple flake Changelog-Added: Nix users can now install CLN from the new flake. --- .github/workflows/ci.yaml | 20 +++++++++ .gitignore | 4 ++ .version | 1 + flake.lock | 58 ++++++++++++++++++++++++ flake.nix | 51 +++++++++++++++++++++ nix/pkgs/default.nix | 94 +++++++++++++++++++++++++++++++++++++++ 6 files changed, 228 insertions(+) create mode 100644 .version create mode 100644 flake.lock create mode 100644 flake.nix create mode 100644 nix/pkgs/default.nix diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 90e12817325b..85e325373f9a 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -528,6 +528,25 @@ jobs: cat config.vars VALGRIND=0 poetry run pytest tests/ -vvv -n ${PYTEST_PAR} ${PYTEST_OPTS} + check-flake: + name: Check Nix Flake + runs-on: ubuntu-20.04 + strategy: + fail-fast: true + steps: + - name: Checkout + uses: actions/checkout@v4 + with: + submodules: 'recursive' + - name: Check Nix flake inputs + uses: DeterminateSystems/flake-checker-action@v8 + - name: Install Nix + uses: cachix/install-nix-action@V27 + with: + nix_path: nixpkgs=channel:nixos-24.05 + - name: Check flake + run: nix flake check .?submodules=1# + gather: # A dummy task that depends on the full matrix of tests, and # signals successful completion. Used for the PR status to pass @@ -540,6 +559,7 @@ jobs: - integration-valgrind - integration-sanitizers - min-btc-support + - check-flake steps: - name: Complete run: | diff --git a/.gitignore b/.gitignore index 602f9710da06..b7f2de9530f4 100644 --- a/.gitignore +++ b/.gitignore @@ -93,3 +93,7 @@ release/ SHA256SUMS-* doc/.doc_version autogenerate-examples-status.log + +# Ignore nix outputs +result +result-[0-9]* diff --git a/.version b/.version new file mode 100644 index 000000000000..3dce2e921cd2 --- /dev/null +++ b/.version @@ -0,0 +1 @@ +24.08 diff --git a/flake.lock b/flake.lock new file mode 100644 index 000000000000..b62d8464241a --- /dev/null +++ b/flake.lock @@ -0,0 +1,58 @@ +{ + "nodes": { + "flake-parts": { + "inputs": { + "nixpkgs-lib": "nixpkgs-lib" + }, + "locked": { + "lastModified": 1726153070, + "narHash": "sha256-HO4zgY0ekfwO5bX0QH/3kJ/h4KvUDFZg8YpkNwIbg1U=", + "owner": "hercules-ci", + "repo": "flake-parts", + "rev": "bcef6817a8b2aa20a5a6dbb19b43e63c5bf8619a", + "type": "github" + }, + "original": { + "owner": "hercules-ci", + "repo": "flake-parts", + "type": "github" + } + }, + "nixpkgs": { + "locked": { + "lastModified": 1725693463, + "narHash": "sha256-ZPzhebbWBOr0zRWW10FfqfbJlan3G96/h3uqhiFqmwg=", + "owner": "NixOS", + "repo": "nixpkgs", + "rev": "68e7dce0a6532e876980764167ad158174402c6f", + "type": "github" + }, + "original": { + "owner": "NixOS", + "ref": "nixos-24.05", + "repo": "nixpkgs", + "type": "github" + } + }, + "nixpkgs-lib": { + "locked": { + "lastModified": 1725233747, + "narHash": "sha256-Ss8QWLXdr2JCBPcYChJhz4xJm+h/xjl4G0c0XlP6a74=", + "type": "tarball", + "url": "https://github.com/NixOS/nixpkgs/archive/356624c12086a18f2ea2825fed34523d60ccc4e3.tar.gz" + }, + "original": { + "type": "tarball", + "url": "https://github.com/NixOS/nixpkgs/archive/356624c12086a18f2ea2825fed34523d60ccc4e3.tar.gz" + } + }, + "root": { + "inputs": { + "flake-parts": "flake-parts", + "nixpkgs": "nixpkgs" + } + } + }, + "root": "root", + "version": 7 +} diff --git a/flake.nix b/flake.nix new file mode 100644 index 000000000000..06d411916708 --- /dev/null +++ b/flake.nix @@ -0,0 +1,51 @@ +{ + description = "Core Lightning (CLN): A specification compliant Lightning Network implementation in C"; + + inputs = { + nixpkgs.url = "github:NixOS/nixpkgs/nixos-24.05"; + flake-parts.url = "github:hercules-ci/flake-parts"; + }; + + outputs = + inputs@{ + self, + nixpkgs, + flake-parts, + ... + }: + flake-parts.lib.mkFlake { inherit inputs; } { + systems = nixpkgs.lib.systems.flakeExposed; + perSystem = + { + config, + pkgs, + self', + ... + }: + { + packages = rec { + # This package depends on git submodules so use a shell command like 'nix build .?submodules=1'. + cln = pkgs.callPackage nix/pkgs/default.nix { inherit self pkgs; }; + default = cln; + }; + apps = { + lightningd = { + program = "${self'.packages.cln}/bin/lightningd"; + }; + lightning-cli = { + program = "${self'.packages.cln}/bin/lightning-cli"; + }; + lightning-hsmtool = { + program = "${self'.packages.cln}/bin/lightning-hsmtool"; + }; + reckless = { + program = "${self'.packages.cln}/bin/reckless"; + }; + }; + checks = { + cln = self'.packages.cln; + }; + formatter = pkgs.nixfmt-rfc-style; + }; + }; +} diff --git a/nix/pkgs/default.nix b/nix/pkgs/default.nix new file mode 100644 index 000000000000..51f7f79207be --- /dev/null +++ b/nix/pkgs/default.nix @@ -0,0 +1,94 @@ +{ + self, + lib, + pkgs, +}: +with pkgs; +let + version = builtins.readFile ../../.version; + py3 = python3.withPackages (p: [ + p.mako + p.grpcio-tools + ]); +in +stdenv.mkDerivation { + name = "cln"; + src = ../../.; + inherit version; + makeFlags = [ + "VERSION=${version}" + "MTIME=${self.lastModifiedDate}" + ]; + + # when building on darwin we need cctools to provide the correct libtool + # as libwally-core detects the host as darwin and tries to add the -static + # option to libtool, also we have to add the modified gsed package. + nativeBuildInputs = + [ + autoconf + autogen + automake + cargo + gettext + gitMinimal + libtool + lowdown + protobuf + py3 + unzip + which + ] + ++ lib.optionals stdenv.isDarwin [ + cctools + darwin.autoSignDarwinBinariesHook + ]; + + buildInputs = [ + gmp + jq + libsodium + sqlite + zlib + ]; + + # this causes some python trouble on a darwin host so we skip this step. + # also we have to tell libwally-core to use sed instead of gsed. + postPatch = + if !stdenv.isDarwin then + '' + patchShebangs \ + tools/generate-wire.py \ + tools/update-mocks.sh \ + tools/mockup.sh \ + tools/fromschema.py \ + devtools/sql-rewrite.py + '' + else + '' + substituteInPlace external/libwally-core/tools/autogen.sh --replace gsed sed && \ + substituteInPlace external/libwally-core/configure.ac --replace gsed sed + ''; + + configureFlags = [ "--disable-valgrind" ]; + + enableParallelBuilding = true; + + # workaround for build issue, happens only x86_64-darwin, not aarch64-darwin + # ccan/ccan/fdpass/fdpass.c:16:8: error: variable length array folded to constant array as an extension [-Werror,-Wgnu-folding-constant] + # char buf[CMSG_SPACE(sizeof(fd))]; + env.NIX_CFLAGS_COMPILE = lib.optionalString ( + stdenv.isDarwin && stdenv.isx86_64 + ) "-Wno-error=gnu-folding-constant"; + + # The `clnrest` plugin requires a Python environment to run + postInstall = '' + rm -r $out/libexec/c-lightning/plugins/clnrest + ''; + + meta = with lib; { + description = "Core Lightning (CLN): A specification compliant Lightning Network implementation in C"; + homepage = "https://github.com/ElementsProject/lightning"; + license = licenses.mit; + platforms = platforms.linux ++ platforms.darwin; + }; +} From 7779834c5c68edc347496fa8d02ea84a2121ba00 Mon Sep 17 00:00:00 2001 From: Joseph Goulden Date: Sun, 29 Sep 2024 15:01:08 +0100 Subject: [PATCH 2/6] build: Add nix derivation for building cargo workspace --- flake.lock | 76 +++++++++++++++++++++++++++++++++++-- flake.nix | 29 ++++++++++---- nix/checks/flake-module.nix | 19 ++++++++++ nix/pkgs/default.nix | 4 +- nix/pkgs/flake-module.nix | 13 +++++++ nix/pkgs/rust.nix | 19 ++++++++++ nix/shells.nix | 19 ++++++++++ nix/treefmt.nix | 11 ++++++ 8 files changed, 176 insertions(+), 14 deletions(-) create mode 100644 nix/checks/flake-module.nix create mode 100644 nix/pkgs/flake-module.nix create mode 100644 nix/pkgs/rust.nix create mode 100644 nix/shells.nix create mode 100644 nix/treefmt.nix diff --git a/flake.lock b/flake.lock index b62d8464241a..2b4169eb0ce8 100644 --- a/flake.lock +++ b/flake.lock @@ -1,5 +1,36 @@ { "nodes": { + "advisory-db": { + "flake": false, + "locked": { + "lastModified": 1727353582, + "narHash": "sha256-2csMEEOZhvowVKZNBHk1kMJqk72ZMrPj9LQYCzP6EKs=", + "owner": "rustsec", + "repo": "advisory-db", + "rev": "cb905e6e405834bdff1eb1e20c9b10edb5403889", + "type": "github" + }, + "original": { + "owner": "rustsec", + "repo": "advisory-db", + "type": "github" + } + }, + "crane": { + "locked": { + "lastModified": 1727316705, + "narHash": "sha256-/mumx8AQ5xFuCJqxCIOFCHTVlxHkMT21idpbgbm/TIE=", + "owner": "ipetkov", + "repo": "crane", + "rev": "5b03654ce046b5167e7b0bccbd8244cb56c16f0e", + "type": "github" + }, + "original": { + "owner": "ipetkov", + "repo": "crane", + "type": "github" + } + }, "flake-parts": { "inputs": { "nixpkgs-lib": "nixpkgs-lib" @@ -20,11 +51,11 @@ }, "nixpkgs": { "locked": { - "lastModified": 1725693463, - "narHash": "sha256-ZPzhebbWBOr0zRWW10FfqfbJlan3G96/h3uqhiFqmwg=", + "lastModified": 1727540905, + "narHash": "sha256-40J9tW7Y794J7Uw4GwcAKlMxlX2xISBl6IBigo83ih8=", "owner": "NixOS", "repo": "nixpkgs", - "rev": "68e7dce0a6532e876980764167ad158174402c6f", + "rev": "fbca5e745367ae7632731639de5c21f29c8744ed", "type": "github" }, "original": { @@ -46,10 +77,47 @@ "url": "https://github.com/NixOS/nixpkgs/archive/356624c12086a18f2ea2825fed34523d60ccc4e3.tar.gz" } }, + "nixpkgs_2": { + "locked": { + "lastModified": 1726871744, + "narHash": "sha256-V5LpfdHyQkUF7RfOaDPrZDP+oqz88lTJrMT1+stXNwo=", + "owner": "nixos", + "repo": "nixpkgs", + "rev": "a1d92660c6b3b7c26fb883500a80ea9d33321be2", + "type": "github" + }, + "original": { + "owner": "nixos", + "ref": "nixpkgs-unstable", + "repo": "nixpkgs", + "type": "github" + } + }, "root": { "inputs": { + "advisory-db": "advisory-db", + "crane": "crane", "flake-parts": "flake-parts", - "nixpkgs": "nixpkgs" + "nixpkgs": "nixpkgs", + "treefmt-nix": "treefmt-nix" + } + }, + "treefmt-nix": { + "inputs": { + "nixpkgs": "nixpkgs_2" + }, + "locked": { + "lastModified": 1727431250, + "narHash": "sha256-uGRlRT47ecicF9iLD1G3g43jn2e+b5KaMptb59LHnvM=", + "owner": "numtide", + "repo": "treefmt-nix", + "rev": "879b29ae9a0378904fbbefe0dadaed43c8905754", + "type": "github" + }, + "original": { + "owner": "numtide", + "repo": "treefmt-nix", + "type": "github" } } }, diff --git a/flake.nix b/flake.nix index 06d411916708..7c44aafe616b 100644 --- a/flake.nix +++ b/flake.nix @@ -3,7 +3,17 @@ inputs = { nixpkgs.url = "github:NixOS/nixpkgs/nixos-24.05"; + flake-parts.url = "github:hercules-ci/flake-parts"; + + crane.url = "github:ipetkov/crane"; + + treefmt-nix.url = "github:numtide/treefmt-nix"; + + advisory-db = { + url = "github:rustsec/advisory-db"; + flake = false; + }; }; outputs = @@ -15,18 +25,25 @@ }: flake-parts.lib.mkFlake { inherit inputs; } { systems = nixpkgs.lib.systems.flakeExposed; + imports = [ + inputs.treefmt-nix.flakeModule + ./nix/pkgs/flake-module.nix + ./nix/checks/flake-module.nix + ./nix/shells.nix + ./nix/treefmt.nix + ]; perSystem = { config, pkgs, self', + system, ... }: { - packages = rec { - # This package depends on git submodules so use a shell command like 'nix build .?submodules=1'. - cln = pkgs.callPackage nix/pkgs/default.nix { inherit self pkgs; }; - default = cln; + _module.args.pkgs = import inputs.nixpkgs { + inherit system; + overlays = [ (final: prev: { craneLib = (inputs.crane.mkLib pkgs); }) ]; }; apps = { lightningd = { @@ -42,10 +59,6 @@ program = "${self'.packages.cln}/bin/reckless"; }; }; - checks = { - cln = self'.packages.cln; - }; - formatter = pkgs.nixfmt-rfc-style; }; }; } diff --git a/nix/checks/flake-module.nix b/nix/checks/flake-module.nix new file mode 100644 index 000000000000..131f5fffca7c --- /dev/null +++ b/nix/checks/flake-module.nix @@ -0,0 +1,19 @@ +{ inputs, self, ... }: +{ + perSystem = + { pkgs, config, ... }: + let + advisory-db = inputs.advisory-db; + in + { + checks = { + cln = config.packages.cln; + rust = config.packages.rust; + cargo-audit = pkgs.craneLib.cargoAudit { + src = ../../.; + inherit advisory-db; + }; + formatting = config.treefmt.build.check self; + }; + }; +} diff --git a/nix/pkgs/default.nix b/nix/pkgs/default.nix index 51f7f79207be..423794eeeeb1 100644 --- a/nix/pkgs/default.nix +++ b/nix/pkgs/default.nix @@ -2,6 +2,7 @@ self, lib, pkgs, + config, }: with pkgs; let @@ -28,12 +29,10 @@ stdenv.mkDerivation { autoconf autogen automake - cargo gettext gitMinimal libtool lowdown - protobuf py3 unzip which @@ -83,6 +82,7 @@ stdenv.mkDerivation { # The `clnrest` plugin requires a Python environment to run postInstall = '' rm -r $out/libexec/c-lightning/plugins/clnrest + cp ${config.packages.rust}/bin/cln-grpc $out/libexec/c-lightning/plugins ''; meta = with lib; { diff --git a/nix/pkgs/flake-module.nix b/nix/pkgs/flake-module.nix new file mode 100644 index 000000000000..5132f17c7da4 --- /dev/null +++ b/nix/pkgs/flake-module.nix @@ -0,0 +1,13 @@ +{ self, ... }: +{ + perSystem = + { pkgs, config, ... }: + { + packages = rec { + # This package depends on git submodules so use a shell command like 'nix build .?submodules=1'. + cln = pkgs.callPackage ./default.nix { inherit self pkgs config; }; + rust = pkgs.callPackage ./rust.nix { craneLib = pkgs.craneLib; }; + default = cln; + }; + }; +} diff --git a/nix/pkgs/rust.nix b/nix/pkgs/rust.nix new file mode 100644 index 000000000000..653ad680fa75 --- /dev/null +++ b/nix/pkgs/rust.nix @@ -0,0 +1,19 @@ +{ + pkgs, + lib, + craneLib, + ... +}: +let + version = builtins.readFile ../../.version; + src = lib.cleanSourceWith { + src = ../../.; + filter = path: type: (lib.hasSuffix "\.proto" path) || (craneLib.filterCargoSources path type); + }; +in +craneLib.buildPackage { + pname = "rust"; + inherit src version; + strictDeps = true; + nativeBuildInputs = with pkgs; [ protobuf ]; +} diff --git a/nix/shells.nix b/nix/shells.nix new file mode 100644 index 000000000000..4f7bb90bcd75 --- /dev/null +++ b/nix/shells.nix @@ -0,0 +1,19 @@ +{ self, ... }: +{ + perSystem = + { + config, + pkgs, + system, + ... + }: + { + devShells = { + rust = pkgs.craneLib.devShell { + checks = { + inherit (self.checks.${system}) rust; + }; + }; + }; + }; +} diff --git a/nix/treefmt.nix b/nix/treefmt.nix new file mode 100644 index 000000000000..ef6e99a34052 --- /dev/null +++ b/nix/treefmt.nix @@ -0,0 +1,11 @@ +{ ... }: +{ + perSystem = + { pkgs, lib, ... }: + { + treefmt = { + projectRootFile = "flake.nix"; + programs.nixfmt.enable = true; + }; + }; +} From 7c1a1ac6d0460d1d82d347533befcb12463127a0 Mon Sep 17 00:00:00 2001 From: Joseph Goulden Date: Mon, 30 Sep 2024 16:33:53 +0100 Subject: [PATCH 3/6] build: Exclude python plugins from flake build --- .version | 2 +- nix/pkgs/default.nix | 6 +++--- nix/shells.nix | 1 + plugins/Makefile | 4 +++- 4 files changed, 8 insertions(+), 5 deletions(-) diff --git a/.version b/.version index 3dce2e921cd2..d1c32a23d3a7 100644 --- a/.version +++ b/.version @@ -1 +1 @@ -24.08 +24.08.1 diff --git a/nix/pkgs/default.nix b/nix/pkgs/default.nix index 423794eeeeb1..28b5e522f631 100644 --- a/nix/pkgs/default.nix +++ b/nix/pkgs/default.nix @@ -8,8 +8,8 @@ with pkgs; let version = builtins.readFile ../../.version; py3 = python3.withPackages (p: [ - p.mako p.grpcio-tools + p.mako ]); in stdenv.mkDerivation { @@ -19,6 +19,7 @@ stdenv.mkDerivation { makeFlags = [ "VERSION=${version}" "MTIME=${self.lastModifiedDate}" + "NO_PYTHON=1" ]; # when building on darwin we need cctools to provide the correct libtool @@ -33,6 +34,7 @@ stdenv.mkDerivation { gitMinimal libtool lowdown + pkgconf py3 unzip which @@ -79,9 +81,7 @@ stdenv.mkDerivation { stdenv.isDarwin && stdenv.isx86_64 ) "-Wno-error=gnu-folding-constant"; - # The `clnrest` plugin requires a Python environment to run postInstall = '' - rm -r $out/libexec/c-lightning/plugins/clnrest cp ${config.packages.rust}/bin/cln-grpc $out/libexec/c-lightning/plugins ''; diff --git a/nix/shells.nix b/nix/shells.nix index 4f7bb90bcd75..e2c4f23f53d7 100644 --- a/nix/shells.nix +++ b/nix/shells.nix @@ -9,6 +9,7 @@ }: { devShells = { + default = pkgs.mkShell { inputsFrom = [ config.packages.default ]; }; rust = pkgs.craneLib.devShell { checks = { inherit (self.checks.${system}) rust; diff --git a/plugins/Makefile b/plugins/Makefile index 23c8d1492b72..320aefa0c00b 100644 --- a/plugins/Makefile +++ b/plugins/Makefile @@ -114,9 +114,11 @@ C_PLUGINS := \ plugins/spenderp \ plugins/cln-askrene +ifneq ($(NO_PYTHON), 1) PY_PLUGINS := \ plugins/clnrest/clnrest \ plugins/wss-proxy/wss-proxy +endif ifeq ($(HAVE_SQLITE3),1) C_PLUGINS += plugins/sql @@ -216,7 +218,7 @@ plugins/txprepare: $(PLUGIN_TXPREPARE_OBJS) $(PLUGIN_LIB_OBJS) $(PLUGIN_COMMON_O plugins/bcli: $(PLUGIN_BCLI_OBJS) $(PLUGIN_LIB_OBJS) $(PLUGIN_COMMON_OBJS) $(JSMN_OBJS) -plugins/keysend: wire/tlvstream.o wire/onion_wiregen.o $(PLUGIN_KEYSEND_OBJS) $(PLUGIN_LIB_OBJS) $(PLUGIN_PAY_LIB_OBJS) $(PLUGIN_COMMON_OBJS) $(JSMN_OBJS) common/gossmap.o common/fp16.o common/route.o common/dijkstra.o common/blindedpay.o common/blindedpath.o common/hmac.o common/blinding.o common/onion_encode.o common/gossmods_listpeerchannels.o common/sciddir_or_pubkey.o +plugins/keysend: wire/tlvstream.o wire/onion_wiregen.o $(PLUGIN_KEYSEND_OBJS) $(PLUGIN_LIB_OBJS) $(PLUGIN_PAY_LIB_OBJS) $(PLUGIN_COMMON_OBJS) $(JSMN_OBJS) common/gossmap.o common/fp16.o common/route.o common/dijkstra.o common/blindedpay.o common/blindedpath.o common/hmac.o common/blinding.o common/onion_encode.o common/gossmods_listpeerchannels.o common/sciddir_or_pubkey.o $(PLUGIN_KEYSEND_OBJS): $(PLUGIN_PAY_LIB_HEADER) plugins/spenderp: bitcoin/block.o bitcoin/preimage.o bitcoin/psbt.o common/psbt_open.o common/json_channel_type.o common/channel_type.o common/features.o wire/peer_wiregen.o $(PLUGIN_SPENDER_OBJS) $(PLUGIN_LIB_OBJS) $(PLUGIN_COMMON_OBJS) $(JSMN_OBJS) From 26cf1468a61a164ba34cd709593fd9448c40d047 Mon Sep 17 00:00:00 2001 From: Joseph Goulden Date: Mon, 30 Sep 2024 21:54:00 +0100 Subject: [PATCH 4/6] build: Update .version as part of the release --- Makefile | 6 +++++- .../release-checklist.md | 12 ++++++------ 2 files changed, 11 insertions(+), 7 deletions(-) diff --git a/Makefile b/Makefile index 02a3c8e35df3..d0436ebc82fc 100644 --- a/Makefile +++ b/Makefile @@ -726,7 +726,7 @@ clean: obsclean PYLNS=client proto testing # See doc/contribute-to-core-lightning/contributor-workflow.md -update-py-versions: update-pyln-versions update-clnrest-version update-wss-proxy-version update-poetry-lock +update-versions: update-pyln-versions update-clnrest-version update-wss-proxy-version update-poetry-lock update-dot-version update-pyln-versions: $(PYLNS:%=update-pyln-version-%) @@ -754,6 +754,10 @@ update-reckless-version: @if [ -z "$(NEW_VERSION)" ]; then echo "Set NEW_VERSION!" >&2; exit 1; fi @sed -i "s/__VERSION__ = '\([.-z]*\)'/__VERSION__ = '$(NEW_VERSION)'/" tools/reckless +update-dot-version: + @if [ -z "$(NEW_VERSION)" ]; then echo "Set NEW_VERSION!" >&2; exit 1; fi + echo $(NEW_VERSION) > .version + update-mocks: $(ALL_TEST_PROGRAMS:%=update-mocks/%.c) $(ALL_TEST_PROGRAMS:%=update-mocks/%.c): $(ALL_GEN_HEADERS) $(EXTERNAL_LIBS) libccan.a ccan/ccan/cdump/tools/cdump-enumstr config.vars diff --git a/doc/contribute-to-core-lightning/release-checklist.md b/doc/contribute-to-core-lightning/release-checklist.md index 59fb0376a328..1a048fbaa6c6 100644 --- a/doc/contribute-to-core-lightning/release-checklist.md +++ b/doc/contribute-to-core-lightning/release-checklist.md @@ -15,16 +15,16 @@ Here's a checklist for the release process. 2. Look through outstanding issues, to identify any problems that might be necessary to fixup before the release. Good candidates are reports of the project not building on different architectures or crashes. 3. Identify a good lead for each outstanding issue, and ask them about a fix timeline. 4. Create a milestone for the _next_ release on Github, and go though open issues and PRs and mark accordingly. -5. Ask (via email) the most significant contributor who has not already named a release to name the release (use +5. Ask (via email) the most significant contributor who has not already named a release to name the release (use `devtools/credit --verbose v` to find this contributor). CC previous namers and team. ## Preparing for -rc1 1. Check that `CHANGELOG.md` is well formatted, ordered in areas, covers all signficant changes, and sub-ordered approximately by user impact & coolness. -2. Use `devtools/changelog.py` to collect the changelog entries from pull request commit messages and merge them into the manually maintained `CHANGELOG.md`. This does API queries to GitHub, which are severely +2. Use `devtools/changelog.py` to collect the changelog entries from pull request commit messages and merge them into the manually maintained `CHANGELOG.md`. This does API queries to GitHub, which are severely ratelimited unless you use an API token: set the `GH_TOKEN` environment variable to a Personal Access Token from 3. Create a new CHANGELOG.md heading to `vrc1`, and create a link at the bottom. Note that you should exactly copy the date and name format from a previous release, as the `build-release.sh` script relies on this. -4. Update the contrib/pyln package versions: `make update-py-versions NEW_VERSION=` +4. Update the package versions: `make update-versions NEW_VERSION=` 5. Create a PR with the above. ## Releasing -rc1 @@ -47,7 +47,7 @@ Here's a checklist for the release process. ## Releasing -rc2, ..., -rcN 1. Update CHANGELOG.md by changing rc(N-1) to rcN. Update the changelog list with information from newly merged PRs also. -2. Update the contrib/pyln package versions: `make update-py-versions NEW_VERSION=` +2. Update the package versions: `make update-versions NEW_VERSION=` 3. Add a PR with the rcN. 4. Tag it `git pull && git tag -s vrcN && git push --tags` 5. Draft a new `vrcN` pre-release on Github, upload reproducible builds, SHA256SUMS and SHA256SUMS.asc. @@ -71,9 +71,9 @@ Here's a checklist for the release process. - Build reproducible Ubuntu-v18.04, Ubuntu-v20.04, Ubuntu-v22.04 images. Follow [link](https://docs.corelightning.org/docs/repro#building-using-the-builder-image) for manually Building Ubuntu Images. - Build Docker images for amd64 and arm64v8. Follow [link](https://docs.corelightning.org/docs/docker-images) for more details on Docker publishing. - Create and sign checksums. Follow [link](https://docs.corelightning.org/docs/repro#co-signing-the-release-manifest) for manually signing the release. -6. The tarballs may be owned by root, so revert ownership if necessary: +6. The tarballs may be owned by root, so revert ownership if necessary: `sudo chown ${USER}:${USER} *${VERSION}*` -7. Upload the resulting files to github and save as a draft. +7. Upload the resulting files to github and save as a draft. () 8. Send `SHA256SUMS` & `SHA256SUMS.asc` files to the rest of the team to check and sign the release. 9. Team members can verify the release with the help of `build-release.sh`: From 2ed2182bc7bc6f7983d6eebdeabb29f607b47f4c Mon Sep 17 00:00:00 2001 From: Joseph Goulden Date: Mon, 30 Sep 2024 23:16:07 +0100 Subject: [PATCH 5/6] build: Add cln-postgres package to nix flake --- nix/checks/flake-module.nix | 1 + nix/pkgs/default.nix | 3 +++ nix/pkgs/flake-module.nix | 4 ++++ nix/shells.nix | 1 + 4 files changed, 9 insertions(+) diff --git a/nix/checks/flake-module.nix b/nix/checks/flake-module.nix index 131f5fffca7c..2af5bedd81d9 100644 --- a/nix/checks/flake-module.nix +++ b/nix/checks/flake-module.nix @@ -8,6 +8,7 @@ { checks = { cln = config.packages.cln; + cln-postgres = config.packages.cln-postgres; rust = config.packages.rust; cargo-audit = pkgs.craneLib.cargoAudit { src = ../../.; diff --git a/nix/pkgs/default.nix b/nix/pkgs/default.nix index 28b5e522f631..268bdb92935d 100644 --- a/nix/pkgs/default.nix +++ b/nix/pkgs/default.nix @@ -3,6 +3,7 @@ lib, pkgs, config, + postgresSupport ? false, }: with pkgs; let @@ -32,6 +33,7 @@ stdenv.mkDerivation { automake gettext gitMinimal + postgresql libtool lowdown pkgconf @@ -39,6 +41,7 @@ stdenv.mkDerivation { unzip which ] + ++ lib.optionals postgresSupport [ postgresql ] ++ lib.optionals stdenv.isDarwin [ cctools darwin.autoSignDarwinBinariesHook diff --git a/nix/pkgs/flake-module.nix b/nix/pkgs/flake-module.nix index 5132f17c7da4..31e2353b363d 100644 --- a/nix/pkgs/flake-module.nix +++ b/nix/pkgs/flake-module.nix @@ -6,6 +6,10 @@ packages = rec { # This package depends on git submodules so use a shell command like 'nix build .?submodules=1'. cln = pkgs.callPackage ./default.nix { inherit self pkgs config; }; + cln-postgres = pkgs.callPackage ./default.nix { + inherit self pkgs config; + postgresSupport = true; + }; rust = pkgs.callPackage ./rust.nix { craneLib = pkgs.craneLib; }; default = cln; }; diff --git a/nix/shells.nix b/nix/shells.nix index e2c4f23f53d7..bb3588c28b3a 100644 --- a/nix/shells.nix +++ b/nix/shells.nix @@ -10,6 +10,7 @@ { devShells = { default = pkgs.mkShell { inputsFrom = [ config.packages.default ]; }; + postgres = pkgs.mkShell { inputsFrom = [ config.packages.cln-postgres ]; }; rust = pkgs.craneLib.devShell { checks = { inherit (self.checks.${system}) rust; From fa2a8a294da5768a5fbeb1ad2a206b50f06888cf Mon Sep 17 00:00:00 2001 From: Joseph Goulden Date: Mon, 7 Oct 2024 21:41:57 +0100 Subject: [PATCH 6/6] doc: Add nix flake documentation to developers guide --- doc/developers-guide/developers-guide.md | 20 ++++++++++++++++++-- 1 file changed, 18 insertions(+), 2 deletions(-) diff --git a/doc/developers-guide/developers-guide.md b/doc/developers-guide/developers-guide.md index 17bcfde589ae..2d42e06276cd 100644 --- a/doc/developers-guide/developers-guide.md +++ b/doc/developers-guide/developers-guide.md @@ -46,8 +46,24 @@ Clean up the lightning directories: destroy_ln ``` - - ## Using Polar [Polar](https://lightningpolar.com/) offers a one-click setup of Lightning Network for local app development & testing. + +## Using Nix + +Install [Nix](https://nixos.org/download/) + +Update git submodules `git submodule update --init --recursive`. + +The entry point is `flake.nix` in the root of the project, where the inputs and outputs are defined. + +`nix develop` will create the default shell env with the build and runtime dependencies of the cln package. Then you can call `make` manually and the project will compile as usual. + +`nix develop .#rust` will create a shell env for developing rust. + +`nix build .?submodules=1` will build the default package (cln). + +`nix flake check .?submodules=1` will build the cln and rust packages. Rust tests are run during the build. There are also checks to run cargo audit and nixfmt. + +If you have nix installed you can use `nix run "git+https://github.com/hashrelay/lightning?ref=flake&submodules=1#lightningd"` to run lightningd without having to manually clone the repo. This make use of the flake output apps.