Merge pull request #648 from hyperweb-io/anmol/faucet-fix

fix: remove after log, no longer required, nor is it actually correct

Author: Anmol1696

.github/workflows/pr-tests.yaml

@@ -87,7 +87,7 @@ jobs:
 
       - name: Setup Test infra
         id: starship-action
-        uses: hyperweb-io/starship-action@0.5.6
+        uses: hyperweb-io/starship-action@c810d7760328a55474594aebaa7cb834e219f896
         with:
           config: ${{ env.CONFIG_FILE }}
           chart: ./starship/charts/devnet
@@ -101,23 +101,21 @@ jobs:
           TEST_CONFIG_FILE: ${{ env.CONFIG_FILE }}
         working-directory: starship
 
-      - name: Log if tests step fails
+      - name: Log pods
         if: failure()
         run: |
-          set -euxo pipefail
-
-          echo "Tests failed Logging everything"
-          kubectl get pods -n $DEVNET_NAMESPACE
-
-          for i in `kubectl get po -n $DEVNET_NAMESPACE -o json | jq -r '.items[].metadata.name'`; do
+          kubectl get pods -n $NAMESPACE
+          for i in `kubectl get po -n $NAMESPACE -o json | jq -r '.items[].metadata.name'`; do
             echo "==================================================="
             echo "Logs for $i"
-            kubectl describe pods $i -n $DEVNET_NAMESPACE
-            kubectl logs $i -n $DEVNET_NAMESPACE --all-containers --tail=10000
+            kubectl describe pods $i -n $NAMESPACE
+            kubectl logs $i -n $NAMESPACE -c faucet --tail=10000
+            echo "=====================Previous======================"
+            kubectl logs $i -n $NAMESPACE -c faucet --previous --tail=10000
             echo "==================================================="
           done
         env:
-          DEVNET_NAMESPACE: ${{ steps.starship-action.outputs.namespace }}
+          NAMESPACE: ${{ steps.starship-action.outputs.namespace }}
 
       # todo: change this to be post step of the action
       - name: Cleanup cluster

starship/charts/devnet/defaults.yaml

@@ -993,7 +993,7 @@ defaultFaucet:
     image: ghcr.io/cosmology-tech/starship/cosmjs-faucet:v0.31.1
     concurrency: 5
   starship:
-    image: ghcr.io/hyperweb-io/starship/faucet:20250325-9eb67d7
+    image: anmol1696/faucet:20250325-e08407f # debug faucet # ghcr.io/hyperweb-io/starship/faucet:20250325-9eb67d7
     concurrency: 5
 
 defaultCometmock:

starship/charts/devnet/templates/chains/cosmos/genesis.yaml

@@ -444,10 +444,9 @@ spec:
 
               export | grep "FAUCET"
 
-              while [ $(curl -sw '%{http_code}' http://localhost:26657/status -o /dev/null) -ne 200 ]; do
-                echo "Validator node does not seem to be ready. Waiting for it to start..."
-                echo "Checking: http://localhost:26657/status"
-                sleep 10;
+              until [[ $(curl -s --connect-timeout 2 http://localhost:26657/status | jq -r '.result.sync_info.latest_block_height // "0"' | grep -E '^[1-9][0-9]*$') ]]; do
+                echo "Validator node not ready or block height is still 0. Retrying in 5s..."
+                sleep 5
               done
 
               curl http://localhost:26657/status

starship/faucet/distributor.go

@@ -8,6 +8,7 @@ import (
 	"net/http"
 	"strings"
 	"sync"
+	"time"
 
 	"go.uber.org/zap"
 
@@ -293,11 +294,44 @@ func (a *Account) queryTx(txhash string) (map[string]interface{}, error) {
 	return txMap, nil
 }
 
-// sendTokens performs chain binary send txn from account
-func (a *Account) sendTokens(address string, denom string, amount string) error {
+// queryTxWithRetry wraps queryTx with retry logic on tx "not found" error
+func (a *Account) queryTxWithRetry(txhash string, maxRetries int) (map[string]interface{}, error) {
+	var (
+		txMap map[string]interface{}
+		err   error
+	)
+
+	for attempt := 1; attempt <= maxRetries; attempt++ {
+		txMap, err = a.queryTx(txhash)
+		if err == nil {
+			return txMap, nil
+		}
+
+		// Check explicitly for the "not found" error condition
+		if strings.Contains(err.Error(), "not found") {
+			a.logger.Warn("transaction not found, retrying",
+				zap.String("txhash", txhash),
+				zap.Int("attempt", attempt),
+				zap.Int("maxRetries", maxRetries),
+			)
+
+			// Sleep 2-3 seconds before retry
+			time.Sleep(2*time.Second + time.Duration(rand.Intn(1000))*time.Millisecond)
+			continue
+		}
+
+		// If any other error occurs, return immediately
+		return nil, err
+	}
+
+	return nil, fmt.Errorf("transaction not found after retries: txhash: %s", txhash)
+}
+
+// sendTokens performs chain binary send txn from account, returns txhash
+func (a *Account) sendTokens(address string, denom string, amount string) (string, error) {
 	ok := a.mu.TryLock()
 	if !ok {
-		return fmt.Errorf("account %s busy: %w", a, ErrResourceInUse)
+		return "", fmt.Errorf("account %s busy: %w", a, ErrResourceInUse)
 	}
 	defer a.mu.Unlock()
 
@@ -306,29 +340,35 @@ func (a *Account) sendTokens(address string, denom string, amount string) error
 	output, err := runCommand(cmdStr)
 	if err != nil {
 		a.logger.Error("send token failed", zap.String("cmd", cmdStr), zap.Error(err))
-		return err
+		return "", err
 	}
 	a.logger.Info("ran cmd to send tokens", zap.String("cmd", cmdStr), zap.String("stdout", string(output)))
 
 	// Find the JSON line and extract the txhash using a regular expression
 	txhash, err := extractTxHash(string(output))
 	if err != nil {
 		a.logger.Error("failed to extract txhash", zap.Error(err))
-		return err
+		return "", err
 	}
 
 	a.logger.Debug("send tokens txhash", zap.String("txhash", txhash))
 
+	return txhash, nil
+}
+
+// confirmTx checks if the tx has gone through by checking the event
+func (a *Account) confirmTx(txhash, eventType string) error {
 	// query tx to check if the tx was successful
-	txMap, err := a.queryTx(txhash)
+	txMap, err := a.queryTxWithRetry(txhash, 3)
 	if err != nil {
 		return err
 	}
 
 	// check if the tx has the event
-	err = hasEvent(txMap, "transfer")
+	err = hasEvent(txMap, eventType)
 	if err != nil {
-		a.logger.Error("transfer event not found in tx",
+		a.logger.Error("event not found in tx",
+			zap.String("eventType", eventType),
 			zap.String("txhash", txhash),
 			zap.Any("txMap", txMap))
 		return err
@@ -339,16 +379,23 @@ func (a *Account) sendTokens(address string, denom string, amount string) error
 
 // SendTokens will perform send tokens with retries based on errors
 func (a *Account) SendTokens(address string, denom string, amount string) error {
-	err := a.sendTokens(address, denom, amount)
+	txHash, err := a.sendTokens(address, denom, amount)
 	if err == nil {
 		return nil
 	}
+
 	if strings.Contains(err.Error(), "account sequence mismatch") {
 		// retry sendTokens
 		a.logger.Debug("got account sequence missmatch error, retrying send tokens recursively")
 		return a.SendTokens(address, denom, amount)
 	}
 
+	// Confirm tx has gone through
+	err = a.confirmTx(txHash, "transfer")
+	if err != nil {
+		return err
+	}
+
 	return err
 }
 

starship/tests/e2e/faucet_test.go

@@ -4,6 +4,7 @@ import (
 	"bytes"
 	"encoding/json"
 	"fmt"
+	"io"
 	"net/http"
 	urlpkg "net/url"
 	"strconv"
@@ -131,9 +132,17 @@ func (s *TestSuite) creditAccount(chain *Chain, addr, denom string) error {
 	if err != nil {
 		return err
 	}
+
 	if resp.StatusCode != 200 {
-		return fmt.Errorf("unexpected status code: %d", resp.StatusCode)
+		bodyBytes, err := io.ReadAll(resp.Body)
+		if err != nil {
+			return fmt.Errorf("unexpected status code: %d, and failed to read response body: %w", resp.StatusCode, err)
+		}
+		defer resp.Body.Close()
+
+		return fmt.Errorf("unexpected status code: %d, response: %s", resp.StatusCode, string(bodyBytes))
 	}
+
 	return nil
 }
 
@@ -186,6 +195,7 @@ func (s *TestSuite) TestFaucet_Credit_MultipleRequests() {
 			// Send multiple requests
 			numRequests := 3
 			for i := 0; i < numRequests; i++ {
+				s.T().Log("crediting account for request: ", i)
 				err := s.creditAccount(chain, addr, denom)
 				s.Require().NoError(err)
 			}