Fix ubuntu build (EspressoSystems#1214)

* Fix ubuntu build

- Install postgres client, just
- Make it easier to test locally with docker
- Run with code in repo
- Fix contracts script name for latest foundry
- Skip SQL tests
- Less fuzz runs

.github/workflows/ubuntu-install-without-nix.yml

@@ -9,19 +9,16 @@ concurrency:
   group: ${{ github.workflow }}-${{ github.ref }}
   cancel-in-progress: true
 
-
 jobs:
-  build:
+  ubuntu:
     runs-on: ubuntu-latest
-    container:
-      image: ubuntu:latest
     steps:
       - uses: actions/checkout@v4
+        with:
+          submodules: recursive
+
+      - name: Enable Rust Caching
+        uses: Swatinem/rust-cache@v2
+
       - name: Install and test
-        # Execute all lines that start with four spaces, and remove leading 'sudo '
-        # because we're in a container and already root.
-        run: |
-          grep '^    ' doc/ubuntu.md \
-            | sed 's/^    //' \
-            | sed 's/^sudo //' \
-            | bash -exo pipefail
+        run: scripts/ubuntu-install-test-no-nix

README.md

@@ -4,6 +4,7 @@
 [![Contracts](https://github.com/EspressoSystems/espresso-sequencer/actions/workflows/contracts.yml/badge.svg)](https://github.com/EspressoSystems/espresso-sequencer/actions/workflows/contracts.yml)
 [![Lint](https://github.com/EspressoSystems/espresso-sequencer/actions/workflows/lint.yml/badge.svg)](https://github.com/EspressoSystems/espresso-sequencer/actions/workflows/lint.yml)
 [![Audit](https://github.com/EspressoSystems/espresso-sequencer/actions/workflows/audit.yml/badge.svg)](https://github.com/EspressoSystems/espresso-sequencer/actions/workflows/audit.yml)
+[![Ubuntu](https://github.com/EspressoSystems/espresso-sequencer/actions/workflows/ubuntu-install-without-nix.yml/badge.svg)](https://github.com/EspressoSystems/espresso-sequencer/actions/workflows/ubuntu-install-without-nix.yml)
 
 The Espresso Sequencer offers rollups credible neutrality and enhanced interoperability, without compromising on scale.
 Consisting of a data availability solution and a decentralized network of nodes that sequences transactions, layer-2

doc/ubuntu.md

@@ -5,7 +5,16 @@
 ## Install system dependencies
 
     sudo apt-get update
-    sudo apt-get install -y curl cmake pkg-config libssl-dev protobuf-compiler git
+    sudo apt-get install -y curl cmake pkg-config libssl-dev protobuf-compiler git postgresql-client lsb-release gpg
+
+## Install just
+
+Just is not available in the official ubuntu repos.
+
+    curl --proto '=https' --tlsv1.2 -sSf 'https://proget.makedeb.org/debian-feeds/prebuilt-mpr.pub' | gpg --dearmor | sudo tee /usr/share/keyrings/prebuilt-mpr-archive-keyring.gpg 1> /dev/null
+    echo "deb [arch=all,$(dpkg --print-architecture) signed-by=/usr/share/keyrings/prebuilt-mpr-archive-keyring.gpg] https://proget.makedeb.org prebuilt-mpr $(lsb_release -cs)" | sudo tee /etc/apt/sources.list.d/prebuilt-mpr.list
+    sudo apt-get update
+    sudo apt-get install -y just
 
 ## Install rustup
 
@@ -15,21 +24,28 @@
 ## Install foundry
 
     curl --proto '=https' --tlsv1.2 -sSf -L https://foundry.paradigm.xyz | bash
-    source $HOME/.bashrc
+    export "PATH=$HOME/.foundry/bin:$PATH"
     foundryup
 
 ## Clone the repository
 
-    git clone https://github.com/EspressoSystems/espresso-sequencer/
+    git clone --recursive https://github.com/espressosystems/espresso-sequencer
     cd espresso-sequencer
 
+## Build the contracts
+
+    forge build
+
 ## Run the rust tests
+To run the SQL tests docker needs to be installed and running.
 
     export RUSTFLAGS='--cfg async_executor_impl="async-std" --cfg async_channel_impl="async-std"'
-    cargo test --release
+    export "PATH=$PWD/target/release:$PATH"
+    cargo build --release --bin diff-test
+    cargo test --release --all-features -- --skip sql
 
 ## Run the foundry tests
 
-    cargo build --release --bin diff-test
-    export PATH=$PWD/target/release:$PATH
-    forge test -v
+Here a single fuzz run is used just to check that things are working.
+
+    env FOUNDRY_FUZZ_RUNS=1 forge test -v

hotshot-state-prover/src/service.rs

@@ -498,15 +498,19 @@ mod test {
         let signer = Wallet::from(anvil.keys()[0].clone());
         let l1_wallet = Arc::new(L1Wallet::new(provider.clone(), signer));
 
-        let status = Command::new("just")
+        let output = Command::new("just")
             .arg("dev-deploy")
             .arg(anvil.endpoint())
             .arg(TEST_MNEMONIC)
             .arg(BLOCKS_PER_EPOCH.to_string())
             .arg(NUM_INIT_VALIDATORS.to_string())
-            .status()
+            .output()
             .expect("fail to deploy");
-        assert!(status.success());
+
+        if !output.status.success() {
+            tracing::error!("{}", String::from_utf8(output.stderr).unwrap());
+            return Err(anyhow!("failed to deploy contract"));
+        }
 
         let last_blk_num = provider.get_block_number().await?;
         // the first tx deploys PlonkVerifier.sol library, the second deploys LightClient.sol

justfile

@@ -79,7 +79,6 @@ sol-test:
 
 # Deploy contracts to local blockchain for development and testing
 dev-deploy url="http://localhost:8545" mnemonics="test test test test test test test test test test test junk" num_blocks_per_epoch="10" num_init_validators="5":
-    forge build
-    MNEMONICS="{{mnemonics}}" forge script 'contracts/test/LightClientTest.s.sol' \
+    MNEMONICS="{{mnemonics}}" forge script contracts/test/LightClientTest.s.sol:DeployLightClientTestScript \
     --sig "run(uint32 numBlocksPerEpoch, uint32 numInitValidators)" {{num_blocks_per_epoch}} {{num_init_validators}} \
     --fork-url {{url}} --broadcast

scripts/ubuntu-install-test-no-nix

@@ -0,0 +1,25 @@
+#!/usr/bin/env bash
+# This script is to test the installation instructions in doc/ubuntu.md without
+# using Nix on ubuntu. It's not recommended to run it outside of a container or
+# otherwise ephemeral environment like the CI.
+set -e
+#
+# Execute the ubuntu installation instructions from doc/ubuntu.md
+#
+# Skip cloning the repo and changing directories, because we want to test with
+# the current code and not what's in the main branch.
+SCRIPT="$(grep '^    ' doc/ubuntu.md | sed 's/^    //' | sed 's/^cd .*//' | sed 's/^git clone .*//')"
+
+# Remove sudo if we're already root
+if [ "$(whoami)" == root ]; then
+    SCRIPT="$(echo "$SCRIPT" | sed 's/^sudo //' | sed 's/| sudo /| /')"
+fi
+
+# Workaround: foundryup installs into XDG_CONFIG_HOME if that's set HOME otherwise.
+export XDG_CONFIG_HOME=$HOME
+
+echo "Will execute:"
+echo "$SCRIPT"
+echo
+
+echo "$SCRIPT" | bash -exo pipefail

scripts/ubuntu-install-test-no-nix-docker

@@ -0,0 +1,24 @@
+#!/usr/bin/env bash
+# This script is to test the ubuntu installation instructions locally on
+# non-ubuntu machines by running through the installation and tests inside an
+# ubuntu docker contrainer.
+set -e
+
+# Create a temporary copy of the repository to be mounted into the container.
+REPO_COPY_DIR="$(mktemp -d)"
+
+# Remove repo copy on exit, error etc.
+trap "exit" INT TERM
+trap cleanup EXIT
+cleanup(){
+    echo "Cleaning up repo copy: $REPO_COPY_DIR, need sudo"
+    sudo rm -rf "$REPO_COPY_DIR"
+}
+
+git clone --recursive . "$REPO_COPY_DIR"
+
+# Run the installation and tests inside the container.
+docker run -it --rm \
+    -v "$REPO_COPY_DIR:/code" \
+    ubuntu:latest \
+    bash -c "cd /code && ./scripts/ubuntu-install-test-no-nix"