diff --git a/.github/scripts/install-bitcoind.sh b/.github/scripts/install-bitcoind.sh index 842c6ccd92a3..edf73474ebc6 100755 --- a/.github/scripts/install-bitcoind.sh +++ b/.github/scripts/install-bitcoind.sh @@ -1,4 +1,6 @@ #!/bin/sh +# If an argument is specified, that dir is checked before downloading, +# and updated after successful install. set -e @@ -18,13 +20,23 @@ cd /tmp/ # testing against `bitcoind` but still believe that we ran against # `elementsd`. if [ "$TEST_NETWORK" = "liquid-regtest" ]; then - wget "https://github.com/ElementsProject/elements/releases/download/elements-${ELEMENTS_VERSION}/${EFILENAME}" + if [ -f "$1/${EFILENAME}" ]; then + cp "$1/${EFILENAME}" . + else + wget "https://github.com/ElementsProject/elements/releases/download/elements-${ELEMENTS_VERSION}/${EFILENAME}" + fi tar -xf "${EFILENAME}" + [ "$1" = "" ] || cp "${EFILENAME}" "$1"/ sudo mv "${EDIRNAME}"/bin/* "/usr/local/bin" rm -rf "${EFILENAME}" "${EDIRNAME}" else - wget "https://bitcoincore.org/bin/bitcoin-core-${BITCOIN_VERSION}/${FILENAME}" + if [ -f "$1/${FILENAME}" ]; then + cp "$1/${FILENAME}" . + else + wget "https://bitcoincore.org/bin/bitcoin-core-${BITCOIN_VERSION}/${FILENAME}" + fi tar -xf "${FILENAME}" + [ "$1" = "" ] || cp "${FILENAME}" "$1"/ sudo mv "${DIRNAME}"/bin/* "/usr/local/bin" rm -rf "${FILENAME}" "${DIRNAME}" fi diff --git a/.github/scripts/setup.sh b/.github/scripts/setup.sh index a719c52086c4..291c4c074c0f 100755 --- a/.github/scripts/setup.sh +++ b/.github/scripts/setup.sh @@ -3,10 +3,19 @@ set -e export DEBIAN_FRONTEND=noninteractive export RUST_VERSION=stable -sudo useradd -ms /bin/bash tester -sudo apt-get update -qq +sudo mkdir -p /var/cache/apt/archives +mkdir -p ~/ci-cache/apt/ +sudo cp -a ~/ci-cache/apt/. /var/cache/apt/archives/ 2>/dev/null || true -sudo apt-get -qq install --no-install-recommends --allow-unauthenticated -yy \ +sudo apt-get update + +# Install eatmydata, then use it for the rest. +sudo apt-get install --no-install-recommends --allow-unauthenticated -yy \ + -o APT::Keep-Downloaded-Packages=true \ + eatmydata + +sudo eatmydata apt-get install --no-install-recommends --allow-unauthenticated -yy \ + -o APT::Keep-Downloaded-Packages=true \ autoconf \ automake \ binfmt-support \ @@ -14,7 +23,6 @@ sudo apt-get -qq install --no-install-recommends --allow-unauthenticated -yy \ clang \ cppcheck \ docbook-xml \ - eatmydata \ gcc-aarch64-linux-gnu \ gcc-arm-linux-gnueabihf \ gcc-arm-none-eabi \ @@ -64,7 +72,7 @@ sudo apt-get -qq install --no-install-recommends --allow-unauthenticated -yy \ echo "tester ALL=(root) NOPASSWD:ALL" | sudo tee /etc/sudoers.d/tester sudo chmod 0440 /etc/sudoers.d/tester -"$(dirname "$0")"/install-bitcoind.sh +"$(dirname "$0")"/install-bitcoind.sh ~/ci-cache/ curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- \ -y --default-toolchain ${RUST_VERSION} @@ -92,8 +100,14 @@ uv tool install poetry PROTOC_VERSION=29.4 PB_REL="https://github.com/protocolbuffers/protobuf/releases" -curl -LO $PB_REL/download/v${PROTOC_VERSION}/protoc-${PROTOC_VERSION}-linux-x86_64.zip -sudo unzip protoc-${PROTOC_VERSION}-linux-x86_64.zip -d /usr/local/ +PROTOC_ZIP=protoc-${PROTOC_VERSION}-linux-x86_64.zip +if [ ! -f ~/ci-cache/$PROTOC_ZIP ]; then + curl -LO $PB_REL/download/v${PROTOC_VERSION}/$PROTOC_ZIP + # Check it before we commit it to the cache! + unzip -t $PROTOC_ZIP + cp $PROTOC_ZIP ~/ci-cache/ +fi +sudo unzip ~/ci-cache/$PROTOC_ZIP -d /usr/local/ sudo chmod a+x /usr/local/bin/protoc export PROTOC=/usr/local/bin/protoc export PATH=$PATH:/usr/local/bin @@ -110,3 +124,5 @@ sudo setcap cap_net_raw,cap_net_admin=eip /usr/bin/dumpcap # Add ourselves to the wireshark group (still need "sg wireshark..." for it to take effect) sudo usermod -aG wireshark "$(id -nu)" +# Copy archives back for caching +cp /var/cache/apt/archives/*.deb ~/ci-cache/apt/ || true diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 72a9fdee61f6..9e610d9161cf 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -14,7 +14,8 @@ env: # Reduce size, helps upload-artifact work more reliably RUST_PROFILE: small SLOW_MACHINE: 1 - CI_SERVER_URL: "http://35.239.136.52:3170" + # This is for the pytest-trackflaky: + # CI_SERVER_URL: "http://35.239.136.52:3170" PYTEST_OPTS_BASE: "-vvv --junit-xml=report.xml --timeout=1800 --durations=10" TEST_LOG_IGNORE_ERRORS: "1" SCCACHE_GHA_ENABLED: "true" @@ -81,6 +82,11 @@ jobs: - name: Install uv uses: astral-sh/setup-uv@v5 + - uses: actions/cache@v4 + with: + path: ~/ci-cache + key: apt-${{ runner.os }} + - name: Install dependencies env: TEST_NETWORK: ${{ matrix.TEST_NETWORK }} @@ -105,7 +111,7 @@ jobs: if-no-files-found: ignore compile: - name: Compile CLN ${{ matrix.cfg }} + name: Build ${{ matrix.cfg }} runs-on: ubuntu-24.04 timeout-minutes: 30 needs: @@ -132,6 +138,7 @@ jobs: - CFG: compile-clang VALGRIND: 1 COMPILER: clang + COPTFLAGS_VAR: COPTFLAGS="-O3" DEBUG_BUILD: --enable-debugbuild - CFG: compile-clang-sanitizers COMPILER: clang @@ -152,6 +159,11 @@ jobs: - name: Install uv uses: astral-sh/setup-uv@v5 + - uses: actions/cache@v4 + with: + path: ~/ci-cache + key: apt-${{ runner.os }} + - name: Install dependencies env: TEST_NETWORK: ${{ matrix.TEST_NETWORK }} @@ -202,6 +214,11 @@ jobs: - name: Install uv uses: astral-sh/setup-uv@v5 + - uses: actions/cache@v4 + with: + path: ~/ci-cache + key: apt-${{ runner.os }} + - name: Install dependencies env: TEST_NETWORK: ${{ matrix.TEST_NETWORK }} @@ -262,6 +279,11 @@ jobs: - name: Install uv uses: astral-sh/setup-uv@v5 + - uses: actions/cache@v4 + with: + path: ~/ci-cache + key: apt-${{ runner.os }} + - name: Install dependencies env: TEST_NETWORK: ${{ matrix.TEST_NETWORK }} @@ -307,6 +329,11 @@ jobs: - name: Install uv uses: astral-sh/setup-uv@v5 + - uses: actions/cache@v4 + with: + path: ~/ci-cache + key: apt-${{ runner.os }} + - name: Install dependencies env: TEST_NETWORK: ${{ matrix.TEST_NETWORK }} @@ -355,6 +382,11 @@ jobs: - name: Install uv uses: astral-sh/setup-uv@v5 + - uses: actions/cache@v4 + with: + path: ~/ci-cache + key: apt-${{ runner.os }} + - name: Install dependencies env: TEST_NETWORK: ${{ matrix.TEST_NETWORK }} @@ -441,6 +473,11 @@ jobs: - name: Install uv uses: astral-sh/setup-uv@v5 + - uses: actions/cache@v4 + with: + path: ~/ci-cache + key: apt-${{ runner.os }} + - name: Install dependencies env: TEST_NETWORK: ${{ matrix.TEST_NETWORK }} @@ -535,6 +572,11 @@ jobs: - name: Install uv uses: astral-sh/setup-uv@v5 + - uses: actions/cache@v4 + with: + path: ~/ci-cache + key: apt-${{ runner.os }} + - name: Install dependencies env: TEST_NETWORK: ${{ matrix.TEST_NETWORK }} @@ -610,13 +652,15 @@ jobs: - name: Install uv uses: astral-sh/setup-uv@v5 + - uses: actions/cache@v4 + with: + path: ~/ci-cache + key: apt-${{ runner.os }} + - name: Install dependencies run: | bash -x .github/scripts/setup.sh - - name: Install bitcoind - run: .github/scripts/install-bitcoind.sh - - name: Download build uses: actions/download-artifact@v4 with: @@ -672,6 +716,11 @@ jobs: - name: Install uv uses: astral-sh/setup-uv@v5 + - uses: actions/cache@v4 + with: + path: ~/ci-cache + key: apt-${{ runner.os }} + - name: Install dependencies env: TEST_NETWORK: ${{ matrix.TEST_NETWORK }} @@ -729,13 +778,15 @@ jobs: - name: Install uv uses: astral-sh/setup-uv@v5 + - uses: actions/cache@v4 + with: + path: ~/ci-cache + key: apt-${{ runner.os }} + - name: Install dependencies run: | bash -x .github/scripts/setup.sh - - name: Install bitcoind - env: - TEST_NETWORK: regtest - run: .github/scripts/install-bitcoind.sh + - name: Download build uses: actions/download-artifact@v4 with: @@ -790,6 +841,11 @@ jobs: - name: Install uv uses: astral-sh/setup-uv@v5 + - uses: actions/cache@v4 + with: + path: ~/ci-cache + key: apt-${{ runner.os }} + - name: Install dependencies env: TEST_NETWORK: ${{ matrix.TEST_NETWORK }} diff --git a/tests/fixtures.py b/tests/fixtures.py index 7ff7e26d4408..186d8723758b 100644 --- a/tests/fixtures.py +++ b/tests/fixtures.py @@ -119,12 +119,6 @@ def have_binary(name): return False -@pytest.fixture(scope="session") -def have_pcap_tools(): - if not dumpcap_usable(): - pytest.skip("dumpcap/tshark not available or insufficient privileges") - - class TcpCapture: def __init__(self, tmpdir): self.tmpdir = Path(tmpdir) @@ -179,7 +173,7 @@ def assert_constant_payload(self): @pytest.fixture -def tcp_capture(have_pcap_tools, tmp_path): +def tcp_capture(tmp_path): # You will need permissions. Most distributions have a group which has # permissions to use dumpcap: # $ ls -l /usr/bin/dumpcap @@ -187,6 +181,9 @@ def tcp_capture(have_pcap_tools, tmp_path): # $ getcap /usr/bin/dumpcap # /usr/bin/dumpcap cap_net_admin,cap_net_raw=eip # So you just need to be in the wireshark group. + if not dumpcap_usable(): + pytest.skip("dumpcap/tshark not available or insufficient privileges") + cap = TcpCapture(tmp_path) yield cap cap.stop()