feat: opt-in http retrieval client (#10772)

* Feat: http retrieval as experimental feature

This introduces the http-retrieval capability as an experimental feature.

It can be enabled in the configuration `Experimental.HTTPRetrieval.Enabled = true`.

Documentation and changelog to be added later.

* refactor: HTTPRetrieval.Enabled as Flag

* docs(config): HTTPRetrieval section

* refactor: reusable MockHTTPContentRouter

* feat: HTTPRetrieval.TLSInsecureSkipVerify

allows self-signed certificates in tests

* feat(config): HTTPRetrieval.MaxBlockSize

* test: end-to-end HTTPRetrieval.Enabled

this spawns two http services on localhost:
1. HTTP router that returns HTTP provider when /routing/v1/providers/cid  i queried
2. HTTP provider that returns a block when /ipfs/cid is queried
3. Configures Kubo to use (1) instead of cid.contact

this seems to work (running test with DEBUG=true shows (1) was queried
for the test CID and returned multiaddr of (2), but Kubo never requested
test CID block from (2) – needs investigation

* fix: enable /routing/v1/peers for non-cid.contact

we artificially limited every delegated routing endpoint because of
cid.contact being limited to one endpoint

* feat: Routing.DelegatedRouters

make it easy to override the hardcoded implicit HTTP routeur URL
without having to set the entire custom Router.Routers and
Router.Methods

(http_retrieval_client_test.go still needs to be fixed in future commit)

* test: flag remaining work

* docs: review feedback

* refactor: providerQueryMgr with bitswapNetworks

this fixes two regressions:

(1) introduced in #10717
    where we only used bitswapLib2p query manager
    (this is why E2E did not act on http provider)

(2) introduced in #10765
    where it was not possible to set binary peerID in IgnoreProviders
    (we changed to []string)

* refactor: Bitswap.Libp2pEnabled

replaces Bitswap.Enabled with Bitswap.Libp2pEnabled
adds tests that confirm it is possible to disable libp2p bitswap fully
and only keep http in client mode

also, removes the need for passing empty blockstore in client-only mode

* docs: changelog

---------

Co-authored-by: Marcin Rataj <lidel@lidel.org>

Author: hsanjuan

cmd/ipfs/kubo/daemon.go

@@ -458,6 +458,7 @@ func daemonFunc(req *cmds.Request, re cmds.ResponseEmitter, env cmds.Environment
 			cfg.Identity.PeerID,
 			cfg.Addresses,
 			cfg.Identity.PrivKey,
+			cfg.HTTPRetrieval.Enabled.WithDefault(config.DefaultHTTPRetrievalEnabled),
 		)
 	default:
 		return fmt.Errorf("unrecognized routing option: %s", routingOption)

config/bitswap.go

@@ -2,13 +2,14 @@ package config
 
 // Bitswap holds Bitswap configuration options
 type Bitswap struct {
-	// Enabled controls both client and server (enabled by default)
-	Enabled Flag `json:",omitempty"`
-	// ServerEnabled controls if the node responds to WANTs (depends on Enabled, enabled by default)
+	// Libp2pEnabled controls if the node initializes bitswap over libp2p (enabled by default)
+	// (This can be disabled if HTTPRetrieval.Enabled is set to true)
+	Libp2pEnabled Flag `json:",omitempty"`
+	// ServerEnabled controls if the node responds to WANTs (depends on Libp2pEnabled, enabled by default)
 	ServerEnabled Flag `json:",omitempty"`
 }
 
 const (
-	DefaultBitswapEnabled       = true
+	DefaultBitswapLibp2pEnabled = true
 	DefaultBitswapServerEnabled = true
 )

config/config.go

@@ -33,13 +33,14 @@ type Config struct {
 	DNS       DNS
 	Migration Migration
 
-	Provider     Provider
-	Reprovider   Reprovider
-	Experimental Experiments
-	Plugins      Plugins
-	Pinning      Pinning
-	Import       Import
-	Version      Version
+	Provider      Provider
+	Reprovider    Reprovider
+	HTTPRetrieval HTTPRetrieval
+	Experimental  Experiments
+	Plugins       Plugins
+	Pinning       Pinning
+	Import        Import
+	Version       Version
 
 	Internal Internal // experimental/unstable options
 

config/http_retrieval.go

@@ -0,0 +1,19 @@
+package config
+
+// HTTPRetrieval is the configuration object for HTTP Retrieval settings.
+// Implicit defaults can be found in core/node/bitswap.go
+type HTTPRetrieval struct {
+	Enabled               Flag             `json:",omitempty"`
+	Allowlist             []string         `json:",omitempty"`
+	Denylist              []string         `json:",omitempty"`
+	NumWorkers            *OptionalInteger `json:",omitempty"`
+	MaxBlockSize          *OptionalString  `json:",omitempty"`
+	TLSInsecureSkipVerify Flag             `json:",omitempty"`
+}
+
+const (
+	DefaultHTTPRetrievalEnabled               = false // opt-in for now, until we figure out https://github.com/ipfs/specs/issues/496
+	DefaultHTTPRetrievalNumWorkers            = 16
+	DefaultHTTPRetrievalTLSInsecureSkipVerify = false  // only for testing with self-signed HTTPS certs
+	DefaultHTTPRetrievalMaxBlockSize          = "2MiB" // matching bitswap: https://specs.ipfs.tech/bitswap-protocol/#block-sizes
+)

config/init.go

@@ -48,13 +48,6 @@ func InitWithIdentity(identity Identity) (*Config, error) {
 			},
 		},
 
-		Routing: Routing{
-			Type:            nil,
-			Methods:         nil,
-			Routers:         nil,
-			IgnoreProviders: []peer.ID{},
-		},
-
 		// setup the node mount points.
 		Mounts: Mounts{
 			IPFS: "/ipfs",

config/routing.go

@@ -6,27 +6,30 @@ import (
 	"os"
 	"runtime"
 	"strings"
-
-	peer "github.com/libp2p/go-libp2p/core/peer"
 )
 
 const (
 	DefaultAcceleratedDHTClient      = false
 	DefaultLoopbackAddressesOnLanDHT = false
+	CidContactRoutingURL             = "https://cid.contact"
+	PublicGoodDelegatedRoutingURL    = "https://delegated-ipfs.dev" // cid.contact + amino dht (incl. IPNS PUTs)
+	EnvHTTPRouters                   = "IPFS_HTTP_ROUTERS"
+	EnvHTTPRoutersFilterProtocols    = "IPFS_HTTP_ROUTERS_FILTER_PROTOCOLS"
 )
 
 var (
 	// Default HTTP routers used in parallel to DHT when Routing.Type = "auto"
-	DefaultHTTPRouters = getEnvOrDefault("IPFS_HTTP_ROUTERS", []string{
-		"https://cid.contact", // https://github.com/ipfs/kubo/issues/9422#issuecomment-1338142084
+	DefaultHTTPRouters = getEnvOrDefault(EnvHTTPRouters, []string{
+		CidContactRoutingURL, // https://github.com/ipfs/kubo/issues/9422#issuecomment-1338142084
 	})
 
 	// Default filter-protocols to pass along with delegated routing requests (as defined in IPIP-484)
 	// and also filter out locally
-	DefaultHTTPRoutersFilterProtocols = getEnvOrDefault("IPFS_HTTP_ROUTERS_FILTER_PROTOCOLS", []string{
+	DefaultHTTPRoutersFilterProtocols = getEnvOrDefault(EnvHTTPRoutersFilterProtocols, []string{
 		"unknown", // allow results without protocol list, we can do libp2p identify to test them
 		"transport-bitswap",
-		// TODO: add 'transport-ipfs-gateway-http' once https://github.com/ipfs/rainbow/issues/125 is addressed
+		// http is added dynamically in routing/delegated.go.
+		// 'transport-ipfs-gateway-http'
 	})
 )
 
@@ -43,11 +46,13 @@ type Routing struct {
 
 	LoopbackAddressesOnLanDHT Flag `json:",omitempty"`
 
-	IgnoreProviders []peer.ID
+	IgnoreProviders []string `json:",omitempty"`
+
+	DelegatedRouters []string `json:",omitempty"`
 
-	Routers Routers
+	Routers Routers `json:",omitempty"`
 
-	Methods Methods
+	Methods Methods `json:",omitempty"`
 }
 
 type Router struct {

core/node/bitswap.go

@@ -2,24 +2,28 @@ package node
 
 import (
 	"context"
+	"errors"
 	"io"
 	"time"
 
+	"github.com/dustin/go-humanize"
 	"github.com/ipfs/boxo/bitswap"
 	"github.com/ipfs/boxo/bitswap/client"
+	"github.com/ipfs/boxo/bitswap/network"
 	bsnet "github.com/ipfs/boxo/bitswap/network/bsnet"
+	"github.com/ipfs/boxo/bitswap/network/httpnet"
 	blockstore "github.com/ipfs/boxo/blockstore"
 	exchange "github.com/ipfs/boxo/exchange"
 	"github.com/ipfs/boxo/exchange/providing"
 	provider "github.com/ipfs/boxo/provider"
 	rpqm "github.com/ipfs/boxo/routing/providerquerymanager"
 	"github.com/ipfs/go-cid"
-	"github.com/ipfs/go-datastore"
 	ipld "github.com/ipfs/go-ipld-format"
+	version "github.com/ipfs/kubo"
 	"github.com/ipfs/kubo/config"
 	irouting "github.com/ipfs/kubo/routing"
 	"github.com/libp2p/go-libp2p/core/host"
-	"github.com/libp2p/go-libp2p/core/routing"
+	peer "github.com/libp2p/go-libp2p/core/peer"
 	"go.uber.org/fx"
 
 	blocks "github.com/ipfs/go-block-format"
@@ -79,38 +83,63 @@ type bitswapIn struct {
 // Bitswap creates the BitSwap server/client instance.
 // If Bitswap.ServerEnabled is false, the node will act only as a client
 // using an empty blockstore to prevent serving blocks to other peers.
-func Bitswap(provide bool) interface{} {
+func Bitswap(serverEnabled bool) interface{} {
 	return func(in bitswapIn, lc fx.Lifecycle) (*bitswap.Bitswap, error) {
-		bitswapNetwork := bsnet.NewFromIpfsHost(in.Host)
-		var blockstoree blockstore.Blockstore = in.Bs
-		var provider routing.ContentDiscovery
+		var bitswapNetworks, bitswapLibp2p network.BitSwapNetwork
+		var bitswapBlockstore blockstore.Blockstore = in.Bs
 
-		if provide {
+		libp2pEnabled := in.Cfg.Bitswap.Libp2pEnabled.WithDefault(config.DefaultBitswapLibp2pEnabled)
+		if libp2pEnabled {
+			bitswapLibp2p = bsnet.NewFromIpfsHost(in.Host)
+		}
 
-			var maxProviders int = DefaultMaxProviders
-			if in.Cfg.Internal.Bitswap != nil {
-				maxProviders = int(in.Cfg.Internal.Bitswap.ProviderSearchMaxResults.WithDefault(DefaultMaxProviders))
+		if httpCfg := in.Cfg.HTTPRetrieval; httpCfg.Enabled.WithDefault(config.DefaultHTTPRetrievalEnabled) {
+			maxBlockSize, err := humanize.ParseBytes(httpCfg.MaxBlockSize.WithDefault(config.DefaultHTTPRetrievalMaxBlockSize))
+			if err != nil {
+				return nil, err
 			}
-
-			pqm, err := rpqm.New(bitswapNetwork,
-				in.Rt,
-				rpqm.WithMaxProviders(maxProviders),
-				rpqm.WithIgnoreProviders(in.Cfg.Routing.IgnoreProviders...),
+			bitswapHTTP := httpnet.New(in.Host,
+				httpnet.WithHTTPWorkers(int(httpCfg.NumWorkers.WithDefault(config.DefaultHTTPRetrievalNumWorkers))),
+				httpnet.WithAllowlist(httpCfg.Allowlist),
+				httpnet.WithDenylist(httpCfg.Denylist),
+				httpnet.WithInsecureSkipVerify(httpCfg.TLSInsecureSkipVerify.WithDefault(config.DefaultHTTPRetrievalTLSInsecureSkipVerify)),
+				httpnet.WithMaxBlockSize(int64(maxBlockSize)),
+				httpnet.WithUserAgent(version.GetUserAgentVersion()),
 			)
+			bitswapNetworks = network.New(in.Host.Peerstore(), bitswapLibp2p, bitswapHTTP)
+		} else if libp2pEnabled {
+			bitswapNetworks = bitswapLibp2p
+		} else {
+			return nil, errors.New("invalid configuration: Bitswap.Libp2pEnabled and HTTPRetrieval.Enabled are both disabled, unable to initialize Bitswap")
+		}
+
+		// Kubo uses own, customized ProviderQueryManager
+		in.BitswapOpts = append(in.BitswapOpts, bitswap.WithClientOption(client.WithDefaultProviderQueryManager(false)))
+		var maxProviders int = DefaultMaxProviders
+		if in.Cfg.Internal.Bitswap != nil {
+			maxProviders = int(in.Cfg.Internal.Bitswap.ProviderSearchMaxResults.WithDefault(DefaultMaxProviders))
+		}
+		ignoredPeerIDs := make([]peer.ID, 0, len(in.Cfg.Routing.IgnoreProviders))
+		for _, str := range in.Cfg.Routing.IgnoreProviders {
+			pid, err := peer.Decode(str)
 			if err != nil {
 				return nil, err
 			}
-			in.BitswapOpts = append(in.BitswapOpts, bitswap.WithClientOption(client.WithDefaultProviderQueryManager(false)))
-			in.BitswapOpts = append(in.BitswapOpts, bitswap.WithServerEnabled(true))
-			provider = pqm
-		} else {
-			provider = nil
-			// When server is disabled, use an empty blockstore to prevent serving blocks
-			blockstoree = blockstore.NewBlockstore(datastore.NewMapDatastore())
-			in.BitswapOpts = append(in.BitswapOpts, bitswap.WithServerEnabled(false))
+			ignoredPeerIDs = append(ignoredPeerIDs, pid)
+		}
+		providerQueryMgr, err := rpqm.New(bitswapNetworks,
+			in.Rt,
+			rpqm.WithMaxProviders(maxProviders),
+			rpqm.WithIgnoreProviders(ignoredPeerIDs...),
+		)
+		if err != nil {
+			return nil, err
 		}
 
-		bs := bitswap.New(helpers.LifecycleCtx(in.Mctx, lc), bitswapNetwork, provider, blockstoree, in.BitswapOpts...)
+		// Explicitly enable/disable server to ensure desired provide mode
+		in.BitswapOpts = append(in.BitswapOpts, bitswap.WithServerEnabled(serverEnabled))
+
+		bs := bitswap.New(helpers.LifecycleCtx(in.Mctx, lc), bitswapNetworks, providerQueryMgr, bitswapBlockstore, in.BitswapOpts...)
 
 		lc.Append(fx.Hook{
 			OnStop: func(ctx context.Context) error {

core/node/groups.go

@@ -335,17 +335,18 @@ func Online(bcfg *BuildCfg, cfg *config.Config, userResourceOverrides rcmgr.Part
 		recordLifetime = d
 	}
 
-	isBitswapEnabled := cfg.Bitswap.Enabled.WithDefault(config.DefaultBitswapEnabled)
+	isBitswapLibp2pEnabled := cfg.Bitswap.Libp2pEnabled.WithDefault(config.DefaultBitswapLibp2pEnabled)
 	isBitswapServerEnabled := cfg.Bitswap.ServerEnabled.WithDefault(config.DefaultBitswapServerEnabled)
-	// Don't provide from bitswap when the strategic provider service is active
-	shouldBitswapProvide := isBitswapEnabled && isBitswapServerEnabled && !cfg.Experimental.StrategicProviding
+
+	// Don't provide from bitswap when the legacy noop experiment "strategic provider service" is active
+	isBitswapServerEnabled = isBitswapServerEnabled && !cfg.Experimental.StrategicProviding
 
 	return fx.Options(
 		fx.Provide(BitswapOptions(cfg)),
-		fx.Provide(Bitswap(shouldBitswapProvide)),
-		fx.Provide(OnlineExchange(isBitswapEnabled)),
+		fx.Provide(Bitswap(isBitswapServerEnabled)),
+		fx.Provide(OnlineExchange(isBitswapLibp2pEnabled)),
 		// Replace our Exchange with a Providing exchange!
-		fx.Decorate(ProvidingExchange(shouldBitswapProvide)),
+		fx.Decorate(ProvidingExchange(isBitswapServerEnabled)),
 		fx.Provide(DNSResolver),
 		fx.Provide(Namesys(ipnsCacheSize, cfg.Ipns.MaxCacheTTL.WithDefault(config.DefaultIpnsMaxCacheTTL))),
 		fx.Provide(Peering),

core/node/libp2p/routingopt.go

@@ -2,6 +2,7 @@ package libp2p
 
 import (
 	"context"
+	"os"
 	"time"
 
 	"github.com/ipfs/go-datastore"
@@ -29,21 +30,44 @@ type RoutingOptionArgs struct {
 
 type RoutingOption func(args RoutingOptionArgs) (routing.Routing, error)
 
+var noopRouter = routinghelpers.Null{}
+
 func constructDefaultHTTPRouters(cfg *config.Config) ([]*routinghelpers.ParallelRouter, error) {
 	var routers []*routinghelpers.ParallelRouter
+	httpRetrievalEnabled := cfg.HTTPRetrieval.Enabled.WithDefault(config.DefaultHTTPRetrievalEnabled)
+
+	// Use config.DefaultHTTPRouters if custom override was sent via config.EnvHTTPRouters
+	// or if user did not set any preference in cfg.Routing.DelegatedRouters
+	var httpRouterEndpoints []string
+	if os.Getenv(config.EnvHTTPRouters) != "" || len(cfg.Routing.DelegatedRouters) == 0 {
+		httpRouterEndpoints = config.DefaultHTTPRouters
+	} else {
+		httpRouterEndpoints = cfg.Routing.DelegatedRouters
+	}
+
 	// Append HTTP routers for additional speed
-	for _, endpoint := range config.DefaultHTTPRouters {
-		httpRouter, err := irouting.ConstructHTTPRouter(endpoint, cfg.Identity.PeerID, httpAddrsFromConfig(cfg.Addresses), cfg.Identity.PrivKey)
+	for _, endpoint := range httpRouterEndpoints {
+		httpRouter, err := irouting.ConstructHTTPRouter(endpoint, cfg.Identity.PeerID, httpAddrsFromConfig(cfg.Addresses), cfg.Identity.PrivKey, httpRetrievalEnabled)
 		if err != nil {
 			return nil, err
 		}
-
+		// Mapping router to /routing/v1/* endpoints
+		// https://specs.ipfs.tech/routing/http-routing-v1/
 		r := &irouting.Composer{
-			GetValueRouter:      routinghelpers.Null{},
-			PutValueRouter:      routinghelpers.Null{},
-			ProvideRouter:       routinghelpers.Null{}, // modify this when indexers supports provide
-			FindPeersRouter:     routinghelpers.Null{},
-			FindProvidersRouter: httpRouter,
+			GetValueRouter:      httpRouter, // GET /routing/v1/ipns
+			PutValueRouter:      httpRouter, // PUT /routing/v1/ipns
+			ProvideRouter:       noopRouter, // we don't have spec for sending provides to /routing/v1 (revisit once https://github.com/ipfs/specs/pull/378 or similar is ratified)
+			FindPeersRouter:     httpRouter, // /routing/v1/peers
+			FindProvidersRouter: httpRouter, // /routing/v1/providers
+		}
+
+		if endpoint == config.CidContactRoutingURL {
+			// Special-case: cid.contact only supports /routing/v1/providers/cid
+			// we disable other endpoints to avoid sending requests that always fail
+			r.GetValueRouter = noopRouter
+			r.PutValueRouter = noopRouter
+			r.ProvideRouter = noopRouter
+			r.FindPeersRouter = noopRouter
 		}
 
 		routers = append(routers, &routinghelpers.ParallelRouter{
@@ -119,7 +143,7 @@ func constructDHTRouting(mode dht.ModeOpt) RoutingOption {
 }
 
 // ConstructDelegatedRouting is used when Routing.Type = "custom"
-func ConstructDelegatedRouting(routers config.Routers, methods config.Methods, peerID string, addrs config.Addresses, privKey string) RoutingOption {
+func ConstructDelegatedRouting(routers config.Routers, methods config.Methods, peerID string, addrs config.Addresses, privKey string, httpRetrieval bool) RoutingOption {
 	return func(args RoutingOptionArgs) (routing.Routing, error) {
 		return irouting.Parse(routers, methods,
 			&irouting.ExtraDHTParams{
@@ -130,9 +154,10 @@ func ConstructDelegatedRouting(routers config.Routers, methods config.Methods, p
 				Context:        args.Ctx,
 			},
 			&irouting.ExtraHTTPParams{
-				PeerID:     peerID,
-				Addrs:      httpAddrsFromConfig(addrs),
-				PrivKeyB64: privKey,
+				PeerID:        peerID,
+				Addrs:         httpAddrsFromConfig(addrs),
+				PrivKeyB64:    privKey,
+				HTTPRetrieval: httpRetrieval,
 			},
 		)
 	}