keep track of unknown crawl errors

Author: dennis-tra

discv5/crawler.go

@@ -71,6 +71,16 @@ func (c *Crawler) Work(ctx context.Context, task PeerInfo) (core.CrawlResult[Pee
 		properties["gen_tcp_addr"] = true
 	}
 
+	// keep track of all unknown connection errors
+	if libp2pResult.ConnectErrorStr == models.NetErrorUnknown && libp2pResult.ConnectError != nil {
+		properties["connect_error"] = libp2pResult.ConnectError.Error()
+	}
+
+	// keep track of all unknown crawl errors
+	if discV5Result.ErrorStr == models.NetErrorUnknown && discV5Result.Error != nil {
+		properties["crawl_error"] = discV5Result.Error.Error()
+	}
+
 	data, err := json.Marshal(properties)
 	if err != nil {
 		log.WithError(err).WithField("properties", properties).Warnln("Could not marshal peer properties")
@@ -263,8 +273,6 @@ func (c *Crawler) connect(ctx context.Context, pi peer.AddrInfo) error {
 		err := c.host.Connect(timeoutCtx, pi)
 		cancel()
 
-		// if we still don't have an error (despite the above custom error
-		// handling), we return to the caller.
 		if err == nil {
 			return nil
 		}

libp2p/crawler.go

@@ -13,6 +13,7 @@ import (
 
 	"github.com/dennis-tra/nebula-crawler/config"
 	"github.com/dennis-tra/nebula-crawler/core"
+	"github.com/dennis-tra/nebula-crawler/db/models"
 	"github.com/dennis-tra/nebula-crawler/kubo"
 	"github.com/dennis-tra/nebula-crawler/utils"
 )
@@ -110,6 +111,16 @@ func mergeResults(r *core.CrawlResult[PeerInfo], p2pRes P2PResult, apiRes APIRes
 		properties["direct_close"] = true
 	}
 
+	// keep track of all unknown connection errors
+	if p2pRes.ConnectErrorStr == models.NetErrorUnknown && p2pRes.ConnectError != nil {
+		properties["connect_error"] = p2pRes.ConnectError.Error()
+	}
+
+	// keep track of all unknown crawl errors
+	if p2pRes.CrawlErrorStr == models.NetErrorUnknown && p2pRes.CrawlError != nil {
+		properties["crawl_error"] = p2pRes.CrawlError.Error()
+	}
+
 	var err error
 	r.Properties, err = json.Marshal(properties)
 	if err != nil {