update ci

Author: samricotta

.github/workflows/ci.yml

@@ -9,11 +9,11 @@ jobs:
   lint_test:
     uses: babylonlabs-io/.github/.github/workflows/[email protected]
     with:
-     go-version: '1.23'
-     go-lint-version: 'v1.60.2'
-     run-unit-tests: true
-     run-integration-tests: true
-     run-lint: true
-     install-dependencies-command: |
-       sudo apt-get update
-       sudo apt-get install -y libzmq3-dev
+      go-version: '1.23'
+      go-lint-version: 'v1.60.2'
+      run-unit-tests: true
+      run-integration-tests: true
+      run-lint: true
+      install-dependencies-command: |
+        sudo apt-get update
+        sudo apt-get install -y libzmq3-dev

Makefile

@@ -65,7 +65,7 @@ run-dgd: build
 	@./build/dgd generate --total-fps 5 --total-stakers 150
 
 test-e2e:
-	go test -mod=readonly -failfast -timeout=15m -v $(PACKAGES_E2E) -count=1
+	go test -mod=readonly -failfast -timeout=35m -v $(PACKAGES_E2E) -count=1
 
 .PHONY: build help
 

container/container.go

@@ -8,7 +8,7 @@ import (
 	"github.com/babylonlabs-io/babylon-benchmark/lib"
 	"github.com/docker/docker/api/types"
 	"regexp"
-	"strconv"
+	"strings"
 	"time"
 
 	bbn "github.com/babylonlabs-io/babylon/types"
@@ -71,7 +71,7 @@ func NewManager() (mgr *Manager, err error) {
 
 func (m *Manager) ExecBitcoindCliCmd(ctx context.Context, command []string) (bytes.Buffer, bytes.Buffer, error) {
 	// this is currently hardcoded, as it will be the same for all tests
-	cmd := []string{"bitcoin-cli", "-chain=regtest", "-rpcuser=user", "-rpcpassword=pass"}
+	cmd := []string{"bitcoin-cli", "-chain=regtest", "-datadir=/data/.bitcoin", "-rpcconnect=127.0.0.1", "-rpcport=18443", "-rpcuser=user", "-rpcpassword=pass"}
 	cmd = append(cmd, command...)
 	return m.ExecCmd(ctx, bitcoindContainerName, cmd)
 }
@@ -85,21 +85,35 @@ func (m *Manager) ExecCmd(ctx context.Context, containerName string, command []s
 	}
 	containerId := m.resources[containerName].Container.ID
 
-	var (
-		outBuf bytes.Buffer
-		errBuf bytes.Buffer
-	)
+	containerInfo, err := m.rawDcClient.ContainerInspect(ctx, containerId)
+	if err != nil {
+		fmt.Printf("🚫 Failed to inspect container %s: %v\n", containerName, err)
+		return bytes.Buffer{}, bytes.Buffer{}, fmt.Errorf("failed to inspect container: %v", err)
+	}
 
-	timeout := 5 * time.Second
+	fmt.Printf("📊 Container %s status: %s, running: %v\n", containerName, containerInfo.State.Status, containerInfo.State.Running)
+
+	if !containerInfo.State.Running {
+		return bytes.Buffer{}, bytes.Buffer{}, fmt.Errorf("container %s is not running (status: %s)", containerName, containerInfo.State.Status)
+	}
+
+	timeout := 120 * time.Second
 	innerCtx, cancel := context.WithTimeout(ctx, timeout)
 	defer cancel()
 
 	// We use the `require.Eventually` function because it is only allowed to do one transaction per block without
 	// sequence numbers. For simplicity, we avoid keeping track of the sequence number and just use the `require.Eventually`.
-	var innerErr error
-	err := lib.Eventually(
+	var (
+		innerErr error
+		outBuf   bytes.Buffer
+		errBuf   bytes.Buffer
+	)
+	err = lib.Eventually(
 		innerCtx,
 		func() bool {
+			// Reset buffers for each retry attempt
+			outBuf.Reset()
+			errBuf.Reset()
 			exec, err := m.pool.Client.CreateExec(docker.CreateExecOptions{
 				Context:      ctx,
 				AttachStdout: true,
@@ -110,35 +124,61 @@ func (m *Manager) ExecCmd(ctx context.Context, containerName string, command []s
 			})
 
 			if err != nil {
+				fmt.Printf("🚫 CreateExec failed with error: %v\n", err)
 				innerErr = fmt.Errorf("failed to create exec: %v", err)
 				return false
 			}
 
+			if exec == nil {
+				fmt.Printf("🚫 CreateExec returned nil exec object\n")
+				innerErr = fmt.Errorf("CreateExec returned nil exec object")
+				return false
+			}
+
 			err = m.pool.Client.StartExec(exec.ID, docker.StartExecOptions{
 				Context:      ctx,
 				Detach:       false,
 				OutputStream: &outBuf,
 				ErrorStream:  &errBuf,
 			})
 			if err != nil {
-				innerErr = fmt.Errorf("failed to create exec: %v", err)
-
+				fmt.Printf("🚫 StartExec failed with error: %v\n", err)
+				innerErr = fmt.Errorf("failed to start exec: %v", err)
 				return false
 			}
 
 			errBufString := errBuf.String()
+			outBufString := outBuf.String()
+			
+			fmt.Printf("🔍 Debug - stdout: %q, stderr: %q\n", outBufString, errBufString)
+			
 			// Note that this does not match all errors.
 			// This only works if CLI outputs "Error" or "error"
 			// to stderr.
+			// Check for real errors vs temporary connection issues during startup
 			if errRegex.MatchString(errBufString) {
-				innerErr = fmt.Errorf("failed to create exec: %v", err)
+				// Allow connection errors during startup - bitcoind process exists but RPC isn't ready
+				if strings.Contains(errBufString, "Could not connect to the server") ||
+					strings.Contains(errBufString, "timeout on transient error") {
+					fmt.Printf("⏳ Bitcoin RPC not ready yet, retrying...\n")
+					innerErr = fmt.Errorf("bitcoin RPC not ready: %s", errBufString)
+					return false // retry
+				}
+				// Real errors that shouldn't be retried
+				fmt.Printf("🚫 Real error detected in stderr, treating as failure\n")
+				innerErr = fmt.Errorf("bitcoin-cli error: %s", errBufString)
 				return false
 			}
+			
+			// Success case - RPC command worked
+			if outBufString != "" || errBufString == "" {
+				fmt.Printf("✅ Bitcoin RPC command succeeded - stdout: %q\n", outBufString)
+			}
 
 			return true
 		},
 		timeout,
-		500*time.Millisecond,
+		2*time.Second,
 	)
 
 	if err != nil {
@@ -176,11 +216,12 @@ func (m *Manager) RunBitcoindResource(
 				"-rpcbind=0.0.0.0",
 				"-zmqpubsequence=tcp://0.0.0.0:28333",
 				"-fallbackfee=0.0002",
+				"-datadir=/data/.bitcoin",
 			},
 		},
 		func(config *docker.HostConfig) {
 			config.PortBindings = map[docker.Port][]docker.PortBinding{
-				"18443/tcp": {{HostIP: "", HostPort: strconv.Itoa(AllocateUniquePort())}}, // only expose what we need
+				"18443/tcp": {{HostIP: "", HostPort: "18443"}}, // use static port mapping, empty HostIP binds to all interfaces
 			}
 			config.PublishAllPorts = false // because in dockerfile they already expose them
 		},

e2e/e2e_test.go

@@ -10,7 +10,7 @@ import (
 )
 
 func TestCreatesDelegations(t *testing.T) {
-	ctx, cancel := context.WithTimeout(context.Background(), 10*time.Minute)
+	ctx, cancel := context.WithTimeout(context.Background(), 20*time.Minute)
 	defer cancel()
 
 	cfg := config.Config{

harness/bitcoindsetup.go

@@ -14,7 +14,7 @@ import (
 )
 
 var (
-	startTimeout = 30 * time.Second
+	startTimeout = 300 * time.Second
 )
 
 type CreateWalletResponse struct {

harness/manager.go

@@ -34,7 +34,7 @@ import (
 
 var (
 	babylonTag            = []byte{1, 2, 3, 4} //nolint:unused
-	eventuallyWaitTimeOut = 180 * time.Second
+	eventuallyWaitTimeOut = 900 * time.Second
 	eventuallyPollTime    = 2 * time.Second
 	regtestParams         = &chaincfg.RegressionNetParams
 )
@@ -82,15 +82,15 @@ func StartManager(ctx context.Context, outputsInWallet uint32, epochInterval uin
 	cfg := defaultConfig()
 
 	btcHandler := NewBitcoindHandler(manager)
-	bitcoind, err := btcHandler.Start(ctx)
+	_, err = btcHandler.Start(ctx)
 	if err != nil {
 		return nil, err
 	}
 
 	passphrase := "pass"
 	_ = btcHandler.CreateWallet(ctx, "default", passphrase)
 
-	cfg.BTC.Endpoint = fmt.Sprintf("127.0.0.1:%s", bitcoind.GetPort("18443/tcp"))
+	cfg.BTC.Endpoint = "127.0.0.1:18443" // static port mapping
 
 	testRpcClient, err := rpcclient.New(&rpcclient.ConnConfig{
 		Host:                 cfg.BTC.Endpoint,