Add Ethereum Consensus Layer Support (#42)

Author: dennis-tra

.github/workflows/pull_request_main.yml

@@ -18,7 +18,7 @@ jobs:
         ports:
           - "2345:5432"
         env:
-          POSTGRES_PASSWORD: password_test
+          POSTGRES_PASSWORD: password
           POSTGRES_USER: nebula_test
           POSTGRES_DB: nebula_test
         options: >-

.gitignore

@@ -180,5 +180,6 @@ override.tf.json
 terraform.rc
 
 # End of https://www.toptal.com/developers/gitignore/api/terraform
-
+out
 report/db.toml
+.tool-versions

Dockerfile

@@ -12,10 +12,9 @@ RUN CGO_ENABLED=0 GOOS=linux go build -ldflags "-X main.RawVersion=`cat version`
 # Create lightweight container to run nebula
 FROM alpine:latest
 
-# Create user ot
+# Create user nebula
 RUN adduser -D -H nebula
 WORKDIR /home/nebula
-RUN mkdir .config && chown nebula:nebula .config
 USER nebula
 
 COPY --from=builder /build/nebula /usr/local/bin/nebula

Makefile

@@ -28,21 +28,25 @@ tools:
 	go install github.com/volatiletech/sqlboiler/v4@v4.13.0
 	go install github.com/volatiletech/sqlboiler/v4/drivers/sqlboiler-psql@v4.13.0
 
-database-reset: migrate-down migrate-up models
+database-reset: database-stop databased migrate-up models
 
 database:
-	docker run --rm -p 5432:5432 -e POSTGRES_PASSWORD=password -e POSTGRES_USER=nebula -e POSTGRES_DB=nebula postgres:14
+	docker run --rm -p 2345:5432 -e POSTGRES_PASSWORD=password -e POSTGRES_USER=nebula_test -e POSTGRES_DB=nebula_test --name nebula_test_db postgres:14
 
-database-test:
-	docker run --rm -p 2345:5432 -e POSTGRES_PASSWORD=password_test -e POSTGRES_USER=nebula_test -e POSTGRES_DB=nebula_test postgres:14
+databased:
+	docker run --rm -d -p 2345:5432 -e POSTGRES_PASSWORD=password -e POSTGRES_USER=nebula_test -e POSTGRES_DB=nebula_test --name nebula_test_db postgres:14
+	sleep 1
+
+database-stop:
+	docker stop nebula_test_db || true
 
 models:
 	sqlboiler --no-tests psql
 
 migrate-up:
-	migrate -database 'postgres://nebula:password@localhost:5432/nebula?sslmode=disable' -path pkg/db/migrations up
+	migrate -database 'postgres://nebula_test:password@localhost:2345/nebula_test?sslmode=disable' -path pkg/db/migrations up
 
 migrate-down:
-	migrate -database 'postgres://nebula:password@localhost:5432/nebula?sslmode=disable' -path pkg/db/migrations down
+	migrate -database 'postgres://nebula_test:password@localhost:2345/nebula_test?sslmode=disable' -path pkg/db/migrations down
 
 .PHONY: all clean test format tools models migrate-up migrate-down

README.md

@@ -2,8 +2,10 @@ package main
 
 import (
 	"context"
+	"fmt"
 	"os"
 	"os/signal"
+	"strings"
 	"syscall"
 	"time"
 
@@ -19,37 +21,40 @@ const (
 	flagCategoryDatabase  = "Database Configuration:"
 	flagCategoryDebugging = "Debugging Configuration:"
 	flagCategoryCache     = "Cache Configuration:"
+	flagCategoryNetwork   = "Network Specific Configuration:"
 )
 
-var (
-	// RawVersion and build tag of the
-	// Nebula command line tool.
-	RawVersion = "dev"
-)
+// RawVersion and build tag of the Nebula command line tool.
+var RawVersion = "dev"
 
 var rootConfig = &config.Root{
-	RawVersion:             RawVersion,
-	Debug:                  false,
-	LogLevel:               4,
-	DialTimeout:            time.Minute,
-	TelemetryHost:          "0.0.0.0",
-	TelemetryPort:          6666,
-	DatabaseHost:           "localhost",
-	DatabasePort:           5432,
-	DatabaseName:           "nebula",
-	DatabasePassword:       "password",
-	DatabaseUser:           "nebula",
-	DatabaseSSLMode:        "disable",
-	AgentVersionsCacheSize: 200,
-	ProtocolsCacheSize:     100,
-	ProtocolsSetCacheSize:  200,
+	RawVersion:      RawVersion,
+	Debug:           false,
+	LogLevel:        4,
+	LogFormat:       "text",
+	LogDisableColor: false,
+	DialTimeout:     time.Minute,
+	TelemetryHost:   "0.0.0.0",
+	TelemetryPort:   6666,
+	Database: &config.Database{
+		DryRun:                 false,
+		JSONOut:                "",
+		DatabaseHost:           "localhost",
+		DatabasePort:           5432,
+		DatabaseName:           "nebula",
+		DatabasePassword:       "password",
+		DatabaseUser:           "nebula",
+		DatabaseSSLMode:        "disable",
+		AgentVersionsCacheSize: 200,
+		ProtocolsCacheSize:     100,
+		ProtocolsSetCacheSize:  200,
+	},
 }
 
 func main() {
-
 	app := &cli.App{
 		Name:      "nebula",
-		Usage:     "A libp2p DHT crawler, monitor, and measurement tool that exposes timely information about DHT networks.",
+		Usage:     "A DHT crawler, monitor, and measurement tool that exposes timely information about DHT networks.",
 		UsageText: "nebula [global options] command [command options] [arguments...]",
 		Authors: []*cli.Author{
 			{
@@ -76,6 +81,29 @@ func main() {
 				Destination: &rootConfig.LogLevel,
 				Category:    flagCategoryDebugging,
 			},
+			&cli.StringFlag{
+				Name:        "log-format",
+				Usage:       "Define the formatting of the log output (values: text, json)",
+				EnvVars:     []string{"NEBULA_LOG_FORMAT"},
+				Value:       rootConfig.LogFormat,
+				Destination: &rootConfig.LogFormat,
+				Category:    flagCategoryDebugging,
+			},
+			&cli.BoolFlag{
+				Name:        "log-disable-color",
+				Usage:       "Whether to have colorized log output (only text log format)",
+				EnvVars:     []string{"NEBULA_LOG_COLOR"},
+				Value:       rootConfig.LogDisableColor,
+				Destination: &rootConfig.LogDisableColor,
+				Category:    flagCategoryDebugging,
+			},
+			&cli.DurationFlag{
+				Name:        "dial-timeout",
+				Usage:       "Global timeout when trying to connect to or dial another peer in the network.",
+				EnvVars:     []string{"NEBULA_DIAL_TIMEOUT"},
+				Value:       rootConfig.DialTimeout,
+				Destination: &rootConfig.DialTimeout,
+			},
 			&cli.StringFlag{
 				Name:        "telemetry-host",
 				Usage:       "To which network address should the telemetry (prometheus, pprof) server bind",
@@ -92,76 +120,92 @@ func main() {
 				Destination: &rootConfig.TelemetryPort,
 				Category:    flagCategoryDebugging,
 			},
+			&cli.BoolFlag{
+				Name:        "dry-run",
+				Usage:       "Don't write anything to disk",
+				EnvVars:     []string{"NEBULA_DRY_RUN", "NEBULA_CRAWL_DRY_RUN" /*<-legacy*/},
+				Value:       rootConfig.Database.DryRun,
+				Destination: &rootConfig.Database.DryRun,
+				Category:    flagCategoryDatabase,
+			},
+			&cli.StringFlag{
+				Name:        "json-out",
+				Usage:       "If set, stores results as JSON documents at `DIR` (takes precedence over database settings).",
+				EnvVars:     []string{"NEBULA_JSON_OUT", "NEBULA_CRAWL_JSON_OUT" /*<-legacy*/},
+				Value:       rootConfig.Database.JSONOut,
+				Destination: &rootConfig.Database.JSONOut,
+				Category:    flagCategoryDatabase,
+			},
 			&cli.StringFlag{
 				Name:        "db-host",
 				Usage:       "On which host address can nebula reach the database",
 				EnvVars:     []string{"NEBULA_DATABASE_HOST"},
-				Value:       rootConfig.DatabaseHost,
-				Destination: &rootConfig.DatabaseHost,
+				Value:       rootConfig.Database.DatabaseHost,
+				Destination: &rootConfig.Database.DatabaseHost,
 				Category:    flagCategoryDatabase,
 			},
 			&cli.IntFlag{
 				Name:        "db-port",
 				Usage:       "On which port can nebula reach the database",
 				EnvVars:     []string{"NEBULA_DATABASE_PORT"},
-				Value:       rootConfig.DatabasePort,
-				Destination: &rootConfig.DatabasePort,
+				Value:       rootConfig.Database.DatabasePort,
+				Destination: &rootConfig.Database.DatabasePort,
 				Category:    flagCategoryDatabase,
 			},
 			&cli.StringFlag{
 				Name:        "db-name",
 				Usage:       "The name of the database to use",
 				EnvVars:     []string{"NEBULA_DATABASE_NAME"},
-				Value:       rootConfig.DatabaseName,
-				Destination: &rootConfig.DatabaseName,
+				Value:       rootConfig.Database.DatabaseName,
+				Destination: &rootConfig.Database.DatabaseName,
 				Category:    flagCategoryDatabase,
 			},
 			&cli.StringFlag{
 				Name:        "db-password",
 				Usage:       "The password for the database to use",
 				EnvVars:     []string{"NEBULA_DATABASE_PASSWORD"},
-				Value:       rootConfig.DatabasePassword,
-				Destination: &rootConfig.DatabasePassword,
+				Value:       rootConfig.Database.DatabasePassword,
+				Destination: &rootConfig.Database.DatabasePassword,
 				Category:    flagCategoryDatabase,
 			},
 			&cli.StringFlag{
 				Name:        "db-user",
 				Usage:       "The user with which to access the database to use",
 				EnvVars:     []string{"NEBULA_DATABASE_USER"},
-				Value:       rootConfig.DatabaseUser,
-				Destination: &rootConfig.DatabaseUser,
+				Value:       rootConfig.Database.DatabaseUser,
+				Destination: &rootConfig.Database.DatabaseUser,
 				Category:    flagCategoryDatabase,
 			},
 			&cli.StringFlag{
 				Name:        "db-sslmode",
 				Usage:       "The sslmode to use when connecting the the database",
 				EnvVars:     []string{"NEBULA_DATABASE_SSL_MODE"},
-				Value:       rootConfig.DatabaseSSLMode,
-				Destination: &rootConfig.DatabaseSSLMode,
+				Value:       rootConfig.Database.DatabaseSSLMode,
+				Destination: &rootConfig.Database.DatabaseSSLMode,
 				Category:    flagCategoryDatabase,
 			},
 			&cli.IntFlag{
 				Name:        "agent-versions-cache-size",
 				Usage:       "The cache size to hold agent versions in memory",
 				EnvVars:     []string{"NEBULA_AGENT_VERSIONS_CACHE_SIZE"},
-				Value:       rootConfig.AgentVersionsCacheSize,
-				Destination: &rootConfig.AgentVersionsCacheSize,
+				Value:       rootConfig.Database.AgentVersionsCacheSize,
+				Destination: &rootConfig.Database.AgentVersionsCacheSize,
 				Category:    flagCategoryCache,
 			},
 			&cli.IntFlag{
 				Name:        "protocols-cache-size",
 				Usage:       "The cache size to hold protocols in memory",
 				EnvVars:     []string{"NEBULA_PROTOCOLS_CACHE_SIZE"},
-				Value:       rootConfig.ProtocolsCacheSize,
-				Destination: &rootConfig.ProtocolsCacheSize,
+				Value:       rootConfig.Database.ProtocolsCacheSize,
+				Destination: &rootConfig.Database.ProtocolsCacheSize,
 				Category:    flagCategoryCache,
 			},
 			&cli.IntFlag{
 				Name:        "protocols-set-cache-size",
 				Usage:       "The cache size to hold sets of protocols in memory",
 				EnvVars:     []string{"NEBULA_PROTOCOLS_SET_CACHE_SIZE"},
-				Value:       rootConfig.ProtocolsSetCacheSize,
-				Destination: &rootConfig.ProtocolsSetCacheSize,
+				Value:       rootConfig.Database.ProtocolsSetCacheSize,
+				Destination: &rootConfig.Database.ProtocolsSetCacheSize,
 				Category:    flagCategoryCache,
 			},
 		},
@@ -170,6 +214,7 @@ func main() {
 			CrawlCommand,
 			MonitorCommand,
 			ResolveCommand,
+			NetworksCommand,
 		},
 	}
 
@@ -179,7 +224,7 @@ func main() {
 	signal.Notify(sigs, syscall.SIGINT, syscall.SIGTERM, os.Interrupt, os.Kill)
 	go func() {
 		sig := <-sigs
-		log.Printf("Received %s signal - Stopping...\n", sig.String())
+		log.Infof("Received %s signal - Stopping...\n", sig.String())
 		signal.Stop(sigs)
 		cancel()
 	}()
@@ -205,6 +250,17 @@ func Before(c *cli.Context) error {
 		}
 	}
 
+	switch strings.ToLower(c.String("log-format")) {
+	case "text":
+		log.SetFormatter(&log.TextFormatter{
+			DisableColors: c.Bool("log-disable-color"),
+		})
+	case "json":
+		log.SetFormatter(&log.JSONFormatter{})
+	default:
+		return fmt.Errorf("unknown log format: %q", c.String("log-format"))
+	}
+
 	// Start prometheus metrics endpoint
 	go metrics.ListenAndServe(rootConfig.TelemetryHost, rootConfig.TelemetryPort)