Skip to content

Commit a26b503

Browse files
hacdiashacdias
authored
gateway: remote car gateway backend (ipfs#587)
1 parent 63c3ec6 commit a26b503

33 files changed

+3956
-267
lines changed

‎.github/workflows/gateway-conformance.yml‎

Lines changed: 158 additions & 16 deletions
Original file line numberDiff line numberDiff line change
@@ -1,17 +1,23 @@
11
name: Gateway Conformance
2+
# This workflow runs https://github.com/ipfs/gateway-conformance
3+
# against different backend implementations of boxo/gateway
24

35
on:
46
push:
57
branches:
68
- main
79
pull_request:
10+
workflow_dispatch:
811

912
concurrency:
1013
group: ${{ github.workflow }}-${{ github.event_name }}-${{ github.event_name == 'push' && github.sha || github.ref }}
1114
cancel-in-progress: true
1215

1316
jobs:
14-
gateway-conformance:
17+
# This test uses a static CAR file as a local blockstore,
18+
# allowing us to test conformance against BlocksBackend (gateway/backend_blocks.go)
19+
# which is used by implementations like Kubo
20+
local-block-backend:
1521
runs-on: ubuntu-latest
1622
steps:
1723
# 1. Download the gateway-conformance fixtures
@@ -21,49 +27,185 @@ jobs:
2127
output: fixtures
2228
merged: true
2329

24-
# 2. Build the car-gateway
30+
# 2. Build the gateway binary
31+
- name: Checkout boxo
32+
uses: actions/checkout@v4
33+
with:
34+
path: boxo
35+
- name: Setup Go
36+
uses: actions/setup-go@v5
37+
with:
38+
go-version-file: 'boxo/examples/go.mod'
39+
cache-dependency-path: "boxo/**/*.sum"
40+
- name: Build test-gateway
41+
run: go build -o test-gateway
42+
working-directory: boxo/examples/gateway/car-file
43+
44+
# 3. Start the gateway binary
45+
- name: Start test-gateway
46+
run: boxo/examples/gateway/car-file/test-gateway -c fixtures/fixtures.car -p 8040 &
47+
48+
# 4. Run the gateway-conformance tests
49+
- name: Run gateway-conformance tests
50+
uses: ipfs/gateway-conformance/.github/actions/test@v0.5
51+
with:
52+
gateway-url: http://127.0.0.1:8040
53+
json: output.json
54+
xml: output.xml
55+
html: output.html
56+
markdown: output.md
57+
subdomain-url: http://example.net
58+
specs: -trustless-ipns-gateway,-path-ipns-gateway,-subdomain-ipns-gateway,-dnslink-gateway
59+
60+
# 5. Upload the results
61+
- name: Upload MD summary
62+
if: failure() || success()
63+
run: cat output.md >> $GITHUB_STEP_SUMMARY
64+
- name: Upload HTML report
65+
if: failure() || success()
66+
uses: actions/upload-artifact@v4
67+
with:
68+
name: gateway-conformance_local-block-backend.html
69+
path: output.html
70+
- name: Upload JSON report
71+
if: failure() || success()
72+
uses: actions/upload-artifact@v4
73+
with:
74+
name: gateway-conformance_local-block-backend.json
75+
path: output.json
76+
77+
# This test uses remote block gateway (?format=raw) as a remote blockstore,
78+
# allowing us to test conformance against RemoteBlocksBackend
79+
# (gateway/backend_blocks.go) which is used by implementations like
80+
# rainbow configured to use with remote block backend
81+
# Ref. https://specs.ipfs.tech/http-gateways/trustless-gateway/#block-responses-application-vnd-ipld-raw
82+
remote-block-backend:
83+
runs-on: ubuntu-latest
84+
steps:
85+
# 1. Download the gateway-conformance fixtures
86+
- name: Download gateway-conformance fixtures
87+
uses: ipfs/gateway-conformance/.github/actions/extract-fixtures@v0.5
88+
with:
89+
output: fixtures
90+
merged: true
91+
92+
# 2. Build the gateway binaries
93+
- name: Checkout boxo
94+
uses: actions/checkout@v4
95+
with:
96+
path: boxo
2597
- name: Setup Go
26-
uses: actions/setup-go@v4
98+
uses: actions/setup-go@v5
99+
with:
100+
go-version-file: 'boxo/examples/go.mod'
101+
cache-dependency-path: "boxo/**/*.sum"
102+
- name: Build remote-block-backend # it will act as a trustless CAR gateway
103+
run: go build -o remote-block-backend
104+
working-directory: boxo/examples/gateway/car-file
105+
- name: Build test-gateway # this one will be used for tests, it will use previous one as its remote block backend
106+
run: go build -o test-gateway
107+
working-directory: boxo/examples/gateway/proxy-blocks
108+
109+
# 3. Start the gateway binaries
110+
- name: Start remote HTTP backend that serves application/vnd.ipld.raw
111+
run: boxo/examples/gateway/car-file/remote-block-backend -c fixtures/fixtures.car -p 8030 & # this endpoint will respond to application/vnd.ipld.car requests
112+
- name: Start gateway that uses the remote block backend
113+
run: boxo/examples/gateway/proxy-blocks/test-gateway -g http://127.0.0.1:8030 -p 8040 &
114+
115+
# 4. Run the gateway-conformance tests
116+
- name: Run gateway-conformance tests
117+
uses: ipfs/gateway-conformance/.github/actions/test@v0.5
118+
with:
119+
gateway-url: http://127.0.0.1:8040 # we test gateway that is backed by a remote block gateway
120+
json: output.json
121+
xml: output.xml
122+
html: output.html
123+
markdown: output.md
124+
subdomain-url: http://example.net
125+
specs: -trustless-ipns-gateway,-path-ipns-gateway,-subdomain-ipns-gateway,-dnslink-gateway
126+
args: -skip 'TestGatewayCache/.*_for_%2Fipfs%2F_with_only-if-cached_succeeds_when_in_local_datastore'
127+
128+
# 5. Upload the results
129+
- name: Upload MD summary
130+
if: failure() || success()
131+
run: cat output.md >> $GITHUB_STEP_SUMMARY
132+
- name: Upload HTML report
133+
if: failure() || success()
134+
uses: actions/upload-artifact@v4
27135
with:
28-
go-version: 1.21.x
136+
name: gateway-conformance_remote-block-backend.html
137+
path: output.html
138+
- name: Upload JSON report
139+
if: failure() || success()
140+
uses: actions/upload-artifact@v4
141+
with:
142+
name: gateway-conformance_remote-block-backend.json
143+
path: output.json
144+
145+
# This test uses remote CAR gateway (?format=car, IPIP-402)
146+
# allowing us to test conformance against remote CarFetcher backend.
147+
# (gateway/backend_car_fetcher.go) which is used by implementations like
148+
# rainbow configured to use with remote car backend
149+
# Ref. https://specs.ipfs.tech/http-gateways/trustless-gateway/#car-responses-application-vnd-ipld-car
150+
remote-car-backend:
151+
runs-on: ubuntu-latest
152+
steps:
153+
# 1. Download the gateway-conformance fixtures
154+
- name: Download gateway-conformance fixtures
155+
uses: ipfs/gateway-conformance/.github/actions/extract-fixtures@v0.5
156+
with:
157+
output: fixtures
158+
merged: true
159+
160+
# 2. Build the gateway binaries
29161
- name: Checkout boxo
30162
uses: actions/checkout@v4
31163
with:
32164
path: boxo
33-
- name: Build car-gateway
34-
run: go build -o car-gateway
35-
working-directory: boxo/examples/gateway/car
165+
- name: Setup Go
166+
uses: actions/setup-go@v5
167+
with:
168+
go-version-file: 'boxo/examples/go.mod'
169+
cache-dependency-path: "boxo/**/*.sum"
170+
- name: Build remote-car-backend # it will act as a trustless CAR gateway
171+
run: go build -o remote-car-backend
172+
working-directory: boxo/examples/gateway/car-file
173+
- name: Build test-gateway # this one will be used for tests, it will use previous one as its remote CAR backend
174+
run: go build -o test-gateway
175+
working-directory: boxo/examples/gateway/proxy-car
36176

37-
# 3. Start the car-gateway
38-
- name: Start car-gateway
39-
run: boxo/examples/gateway/car/car-gateway -c fixtures/fixtures.car -p 8040 &
177+
# 3. Start the gateway binaries
178+
- name: Start remote HTTP backend that serves application/vnd.ipld.car (IPIP-402)
179+
run: boxo/examples/gateway/car-file/remote-car-backend -c fixtures/fixtures.car -p 8030 & # this endpoint will respond to application/vnd.ipld.raw requests
180+
- name: Start gateway that uses the remote CAR backend
181+
run: boxo/examples/gateway/proxy-car/test-gateway -g http://127.0.0.1:8030 -p 8040 &
40182

41183
# 4. Run the gateway-conformance tests
42184
- name: Run gateway-conformance tests
43185
uses: ipfs/gateway-conformance/.github/actions/test@v0.5
44186
with:
45-
gateway-url: http://127.0.0.1:8040
187+
gateway-url: http://127.0.0.1:8040 # we test gateway that is backed by a remote car gateway
46188
json: output.json
47189
xml: output.xml
48190
html: output.html
49191
markdown: output.md
50192
subdomain-url: http://example.net
51193
specs: -trustless-ipns-gateway,-path-ipns-gateway,-subdomain-ipns-gateway,-dnslink-gateway
52-
args: -skip 'TestGatewayCar/GET_response_for_application/vnd.ipld.car/Header_Content-Length'
194+
args: -skip 'TestGatewayCache/.*_for_%2Fipfs%2F_with_only-if-cached_succeeds_when_in_local_datastore'
53195

54196
# 5. Upload the results
55197
- name: Upload MD summary
56198
if: failure() || success()
57199
run: cat output.md >> $GITHUB_STEP_SUMMARY
58200
- name: Upload HTML report
59201
if: failure() || success()
60-
uses: actions/upload-artifact@v3
202+
uses: actions/upload-artifact@v4
61203
with:
62-
name: gateway-conformance.html
204+
name: gateway-conformance_remote-car-backend.html
63205
path: output.html
64206
- name: Upload JSON report
65207
if: failure() || success()
66-
uses: actions/upload-artifact@v3
208+
uses: actions/upload-artifact@v4
67209
with:
68-
name: gateway-conformance.json
210+
name: gateway-conformance_remote-car-backend.json
69211
path: output.json

‎CHANGELOG.md‎

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -16,7 +16,9 @@ The following emojis are used to highlight certain changes:
1616

1717
### Added
1818

19-
* `gateway` now includes `NewRemoteBlocksBackend` which allows you to create a gateway backend that uses one or multiple other gateways as backend. These gateways must support RAW block requests (`application/vnd.ipld.raw`), as well as IPNS Record requests (`application/vnd.ipfs.ipns-record`). With this, we also introduced a `NewCacheBlockStore`, `NewRemoteBlockstore` and `NewRemoteValueStore`.
19+
* ✨ `gateway` has new backend possibilities:
20+
* `NewRemoteBlocksBackend` allows you to create a gateway backend that uses one or multiple other gateways as backend. These gateways must support RAW block requests (`application/vnd.ipld.raw`), as well as IPNS Record requests (`application/vnd.ipfs.ipns-record`). With this, we also introduced `NewCacheBlockStore`, `NewRemoteBlockstore` and `NewRemoteValueStore`.
21+
* `NewRemoteCarBackend` allows you to create a gateway backend that uses one or multiple Trustless Gateways as backend. These gateways must support CAR requests (`application/vnd.ipld.car`), as well as the extensions describe in [IPIP-402](https://specs.ipfs.tech/ipips/ipip-0402/). With this, we also introduced `NewCarBackend`, `NewRemoteCarFetcher` and `NewRetryCarFetcher`.
2022

2123
### Changed
2224

‎examples/README.md‎

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -27,6 +27,7 @@ Once you have your example finished, do not forget to run `go mod tidy` and addi
2727
## Examples and Tutorials
2828

2929
- [Fetching a UnixFS file by CID](./unixfs-file-cid)
30-
- [Gateway backed by a CAR file](./gateway/car)
31-
- [Gateway backed by a remote blockstore and IPNS resolver](./gateway/proxy)
30+
- [Gateway backed by a local blockstore in form of a CAR file](./gateway/car-file)
31+
- [Gateway backed by a remote (HTTP) blockstore and IPNS resolver](./gateway/proxy-blocks)
32+
- [Gateway backed by a remote (HTTP) CAR Gateway](./gateway/proxy-car)
3233
- [Delegated Routing V1 Command Line Client](./routing/delegated-routing-client/)

examples/gateway/car/README.md renamed to examples/gateway/car-file/README.md

Lines changed: 6 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1,13 +1,16 @@
1-
# HTTP Gateway backed by a CAR File
1+
# HTTP Gateway backed by a CAR File as BlocksBackend
22

33
This is an example that shows how to build a Gateway backed by the contents of
44
a CAR file. A [CAR file](https://ipld.io/specs/transport/car/) is a Content
55
Addressable aRchive that contains blocks.
66

7+
The `main.go` sets up a `blockService` backed by a static CAR file,
8+
and then uses it to initialize `gateway.NewBlocksBackend(blockService)`.
9+
710
## Build
811

912
```bash
10-
> go build -o car-gateway
13+
> go build -o gateway
1114
```
1215

1316
## Usage
@@ -23,7 +26,7 @@ Then, you can start the gateway with:
2326

2427

2528
```
26-
./car-gateway -c data.car -p 8040
29+
./gateway -c data.car -p 8040
2730
```
2831

2932
### Subdomain gateway

examples/gateway/car/main.go renamed to examples/gateway/car-file/main.go

File renamed without changes.

examples/gateway/car/main_test.go renamed to examples/gateway/car-file/main_test.go

File renamed without changes.

examples/gateway/car/test.car renamed to examples/gateway/car-file/test.car

File renamed without changes.

examples/gateway/proxy/README.md renamed to examples/gateway/proxy-blocks/README.md

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -18,7 +18,7 @@ gateway using `?format=ipns-record`. In addition, DNSLink lookups are done local
1818
## Build
1919

2020
```bash
21-
> go build -o verifying-proxy
21+
> go build -o gateway
2222
```
2323

2424
## Usage
@@ -28,7 +28,7 @@ types. Once you have it, run the proxy gateway with its address as the host para
2828

2929

3030
```
31-
./verifying-proxy -g https://ipfs.io -p 8040
31+
./gateway -g https://trustless-gateway.link -p 8040
3232
```
3333

3434
### Subdomain gateway

examples/gateway/proxy/main.go renamed to examples/gateway/proxy-blocks/main.go

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -28,7 +28,7 @@ func main() {
2828
defer (func() { _ = tp.Shutdown(ctx) })()
2929

3030
// Creates the gateway with the remote block store backend.
31-
backend, err := gateway.NewRemoteBlocksBackend([]string{*gatewayUrlPtr})
31+
backend, err := gateway.NewRemoteBlocksBackend([]string{*gatewayUrlPtr}, nil)
3232
if err != nil {
3333
log.Fatal(err)
3434
}

examples/gateway/proxy/main_test.go renamed to examples/gateway/proxy-blocks/main_test.go

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -21,7 +21,7 @@ const (
2121
)
2222

2323
func newProxyGateway(t *testing.T, rs *httptest.Server) *httptest.Server {
24-
backend, err := gateway.NewRemoteBlocksBackend([]string{rs.URL})
24+
backend, err := gateway.NewRemoteBlocksBackend([]string{rs.URL}, nil)
2525
require.NoError(t, err)
2626
handler := common.NewHandler(backend)
2727
ts := httptest.NewServer(handler)

0 commit comments

Comments
 (0)