feat: Packet forward middleware

.github/workflows/e2e-test-workflow-call.yml

@@ -36,6 +36,16 @@ on:
         description: 'The tag to use for chain B'
         required: true
         type: string
+      chain-c-tag:
+        description: 'The tag to use for chain C'
+        required: true
+        type: string
+        default: main
+      chain-d-tag:
+        default: main
+        description: 'The tag to use for chain D'
+        required: true
+        type: string
       # upgrade-plan-name is only required during upgrade tests, and is otherwise ignored.
       upgrade-plan-name:
         default: ''
@@ -78,6 +88,8 @@ jobs:
           echo "Chain Image:       ${{ inputs.chain-image }}"
           echo "Chain A Tag:       ${{ inputs.chain-a-tag }}"
           echo "Chain B Tag:       ${{ inputs.chain-b-tag }}"
+          echo "Chain C Tag:       ${{ inputs.chain-c-tag }}"
+          echo "Chain D Tag:       ${{ inputs.chain-d-tag }}"
           echo "Upgrade Plan Name: ${{ inputs.upgrade-plan-name }}"
           echo "Test Entry Point:  ${{ inputs.test-entry-point }}"
           echo "Test:              ${{ inputs.test }}"
@@ -205,6 +217,8 @@ jobs:
       CHAIN_UPGRADE_PLAN: '${{ inputs.upgrade-plan-name }}'
       CHAIN_A_TAG: '${{ inputs.chain-a-tag }}'
       CHAIN_B_TAG: '${{ inputs.chain-b-tag }}'
+      CHAIN_C_TAG: '${{ inputs.chain-c-tag }}'
+      CHAIN_D_TAG: '${{ inputs.chain-d-tag }}'
       E2E_CONFIG_PATH: '${{ inputs.e2e-config-path }}'
     strategy:
       fail-fast: false
@@ -243,6 +257,8 @@ jobs:
       CHAIN_IMAGE: '${{ inputs.chain-image }}'
       CHAIN_A_TAG: '${{ inputs.chain-a-tag }}'
       CHAIN_B_TAG: '${{ inputs.chain-b-tag }}'
+      CHAIN_C_TAG: '${{ inputs.chain-c-tag }}'
+      CHAIN_D_TAG: '${{ inputs.chain-d-tag }}'
       E2E_CONFIG_PATH: '${{ inputs.e2e-config-path }}'
     strategy:
       fail-fast: false
@@ -256,6 +272,7 @@ jobs:
           - entrypoint: TestTransferLocalhostTestSuite
           - entrypoint: TestConnectionTestSuite
           - entrypoint: TestInterchainAccountsGovTestSuite
+          - entrypoint: TestForwardTransferSuite
     steps:
       - uses: actions/checkout@v4
         with:

.github/workflows/e2e-upgrade.yaml

@@ -68,6 +68,8 @@ jobs:
           CHAIN_IMAGE: '${{ env.DOCKER_IMAGE_NAME }}'
           CHAIN_A_TAG: '${{ matrix.test-config.tag }}'
           CHAIN_B_TAG: '${{ matrix.test-config.tag }}'
+          CHAIN_C_TAG: '${{ matrix.test-config.tag }}'
+          CHAIN_D_TAG: '${{ matrix.test-config.tag }}'
           CHAIN_UPGRADE_PLAN: '${{ matrix.test-config.upgrade-plan }}'
           E2E_CONFIG_PATH: 'ci-e2e-config.yaml'
         run: |

.github/workflows/e2e.yaml

@@ -104,6 +104,8 @@ jobs:
           CHAIN_IMAGE: '${{ env.DOCKER_IMAGE_NAME }}'
           CHAIN_A_TAG: '${{ needs.determine-image-tag.outputs.simd-tag }}'
           CHAIN_B_TAG: '${{ needs.determine-image-tag.outputs.simd-tag }}'
+          CHAIN_C_TAG: '${{ needs.determine-image-tag.outputs.simd-tag }}'
+          CHAIN_D_TAG: '${{ needs.determine-image-tag.outputs.simd-tag }}'
           E2E_CONFIG_PATH: 'ci-e2e-config.yaml'
         run: |
           cd e2e

CHANGELOG.md

@@ -38,6 +38,8 @@ Ref: https://keepachangelog.com/en/1.0.0/
 
 ### Features
 
+* [\#8285](https://github.com/cosmos/ibc-go/pull/8285) Packet forward middleware.
+
 ### Dependencies
 
 * [\#8369](https://github.com/cosmos/ibc-go/pull/8369) Bump **github.com/CosmWasm/wasmvm** to **2.2.4**

docs/docs/02-apps/03-packet-forward-middleware/integration.md

@@ -0,0 +1,172 @@
+# Integration
+
+This document provides instructions on integrating and configuring the Packet Forward Middleware (PFM) within your
+existing chain implementation. This document is *NOT* a guide on developing with the Cosmos SDK or ibc-go and makes
+the assumption that you have some existing codebase for your chain with IBC already enabled.
+
+The integration steps include the following:
+
+1. Import the PFM, initialize the PFM Module & Keeper, initialize the store keys and module params, and initialize the Begin/End Block logic and InitGenesis order.
+2. Configure the IBC application stack including the transfer module.
+3. Configuration of additional options such as timeout period, number of retries on timeout, refund timeout period, and fee percentage.
+
+Integration of the PFM should take approximately 20 minutes.
+
+## Example integration of the Packet Forward Middleware
+
+```go
+// app.go
+
+// Import the packet forward middleware
+import (
+    "github.com/cosmos/ibc-apps/middleware/packet-forward-middleware/v10/packetforward"
+    packetforwardkeeper "github.com/cosmos/ibc-apps/middleware/packet-forward-middleware/v10/packetforward/keeper"
+    packetforwardtypes "github.com/cosmos/ibc-apps/middleware/packet-forward-middleware/v10/packetforward/types"
+)
+
+...
+
+// Register the AppModule for the packet forward middleware module
+ModuleBasics = module.NewBasicManager(
+    ...
+    packetforward.AppModuleBasic{},
+    ...
+)
+
+...
+
+// Add packet forward middleware Keeper
+type App struct {
+ ...
+ PacketForwardKeeper *packetforwardkeeper.Keeper
+ ...
+}
+
+...
+
+// Create store keys
+keys := sdk.NewKVStoreKeys(
+    ...
+    packetforwardtypes.StoreKey,
+    ...
+)
+
+...
+
+// Initialize the packet forward middleware Keeper
+// It's important to note that the PFM Keeper must be initialized before the Transfer Keeper
+app.PacketForwardKeeper = packetforwardkeeper.NewKeeper(
+    appCodec,
+    keys[packetforwardtypes.StoreKey],
+    nil, // will be zero-value here, reference is set later on with SetTransferKeeper.
+    app.IBCKeeper.ChannelKeeper,
+    appKeepers.DistrKeeper,
+    app.BankKeeper,
+    app.IBCKeeper.ChannelKeeper,
+    authtypes.NewModuleAddress(govtypes.ModuleName).String(),
+)
+
+// Initialize the transfer module Keeper
+app.TransferKeeper = ibctransferkeeper.NewKeeper(
+    appCodec,
+    keys[ibctransfertypes.StoreKey],
+    app.GetSubspace(ibctransfertypes.ModuleName),
+    app.PacketForwardKeeper,
+    app.IBCKeeper.ChannelKeeper,
+    &app.IBCKeeper.PortKeeper,
+    app.AccountKeeper,
+    app.BankKeeper,
+    scopedTransferKeeper,
+)
+
+app.PacketForwardKeeper.SetTransferKeeper(app.TransferKeeper)
+
+// See the section below for configuring an application stack with the packet forward middleware
+
+...
+
+// Register packet forward middleware AppModule
+app.moduleManager = module.NewManager(
+    ...
+    packetforward.NewAppModule(app.PacketForwardKeeper, app.GetSubspace(packetforwardtypes.ModuleName)),
+)
+
+...
+
+// Add packet forward middleware to begin blocker logic
+app.moduleManager.SetOrderBeginBlockers(
+    ...
+    packetforwardtypes.ModuleName,
+    ...
+)
+
+// Add packet forward middleware to end blocker logic
+app.moduleManager.SetOrderEndBlockers(
+    ...
+    packetforwardtypes.ModuleName,
+    ...
+)
+
+// Add packet forward middleware to init genesis logic
+app.moduleManager.SetOrderInitGenesis(
+    ...
+    packetforwardtypes.ModuleName,
+    ...
+)
+
+// Add packet forward middleware to init params keeper
+func initParamsKeeper(appCodec codec.BinaryCodec, legacyAmino *codec.LegacyAmino, key, tkey storetypes.StoreKey) paramskeeper.Keeper {
+    ...
+    paramsKeeper.Subspace(packetforwardtypes.ModuleName).WithKeyTable(packetforwardtypes.ParamKeyTable())
+    ...
+}
+```
+
+## Configuring the transfer application stack with Packet Forward Middleware
+
+Here is an example of how to create an application stack using `transfer` and `packet-forward-middleware`.
+The following `transferStack` is configured in `app/app.go` and added to the IBC `Router`.
+The in-line comments describe the execution flow of packets between the application stack and IBC core.
+
+For more information on configuring an IBC application stack see the ibc-go docs [here](https://github.com/cosmos/ibc-go/blob/e69a833de764fa0f5bdf0338d9452fd6e579a675/docs/docs/04-middleware/01-ics29-fee/02-integration.md#configuring-an-application-stack-with-fee-middleware).
+
+```go
+// Create Transfer Stack
+// SendPacket, since it is originating from the application to core IBC:
+// transferKeeper.SendPacket -> packetforward.SendPacket -> channel.SendPacket
+
+// RecvPacket, message that originates from core IBC and goes down to app, the flow is the other way
+// channel.RecvPacket -> packetforward.OnRecvPacket -> transfer.OnRecvPacket
+
+// transfer stack contains (from top to bottom):
+// - Packet Forward Middleware
+// - Transfer
+var transferStack ibcporttypes.IBCModule
+transferStack = transfer.NewIBCModule(app.TransferKeeper)
+transferStack = packetforward.NewIBCMiddleware(
+    transferStack,
+    app.PacketForwardKeeper,
+    0, // retries on timeout
+    packetforwardkeeper.DefaultForwardTransferPacketTimeoutTimestamp, // forward timeout
+)
+
+// Add transfer stack to IBC Router
+ibcRouter.AddRoute(ibctransfertypes.ModuleName, transferStack)
+```
+
+## Configurable options in the Packet Forward Middleware
+
+The Packet Forward Middleware has several configurable options available when initializing the IBC application stack.
+You can see these passed in as arguments to `packetforward.NewIBCMiddleware` and they include the number of retries that
+will be performed on a forward timeout, the timeout period that will be used for a forward, and the timeout period that
+will be used for performing refunds in the case that a forward is taking too long.
+
+Additionally, there is a fee percentage parameter that can be set in `InitGenesis`, this is an optional parameter that
+can be used to take a fee from each forwarded packet which will then be distributed to the community pool. In the
+`OnRecvPacket` callback `ForwardTransferPacket` is invoked which will attempt to subtract a fee from the forwarded
+packet amount if the fee percentage is non-zero.
+
+- Retries On Timeout - how many times will a forward be re-attempted in the case of a timeout.
+- Timeout Period - how long can a forward be in progress before giving up.
+- Refund Timeout - how long can a forward be in progress before issuing a refund back to the original source chain.
+- Fee Percentage - % of the forwarded packet amount which will be subtracted and distributed to the community pool.

e2e/go.mod

@@ -152,6 +152,7 @@ require (
 	github.com/hdevalence/ed25519consensus v0.2.0 // indirect
 	github.com/holiman/uint256 v1.3.2 // indirect
 	github.com/huandu/skiplist v1.2.1 // indirect
+	github.com/iancoleman/orderedmap v0.3.0 // indirect
 	github.com/iancoleman/strcase v0.3.0 // indirect
 	github.com/icza/dyno v0.0.0-20230330125955-09f820a8d9c0 // indirect
 	github.com/improbable-eng/grpc-web v0.15.0 // indirect

e2e/go.sum

@@ -1213,6 +1213,8 @@ github.com/huandu/go-assert v1.1.5/go.mod h1:yOLvuqZwmcHIC5rIzrBhT7D3Q9c3GFnd0Jr
 github.com/huandu/skiplist v1.2.1 h1:dTi93MgjwErA/8idWTzIw4Y1kZsMWx35fmI2c8Rij7w=
 github.com/huandu/skiplist v1.2.1/go.mod h1:7v3iFjLcSAzO4fN5B8dvebvo/qsfumiLiDXMrPiHF9w=
 github.com/hudl/fargo v1.3.0/go.mod h1:y3CKSmjA+wD2gak7sUSXTAoopbhU08POFhmITJgmKTg=
+github.com/iancoleman/orderedmap v0.3.0 h1:5cbR2grmZR/DiVt+VJopEhtVs9YGInGIxAoMJn+Ichc=
+github.com/iancoleman/orderedmap v0.3.0/go.mod h1:XuLcCUkdL5owUCQeF2Ue9uuw1EptkJDkXXS7VoV7XGE=
 github.com/iancoleman/strcase v0.2.0/go.mod h1:iwCmte+B7n89clKwxIoIXy/HfoL7AsD47ZCWhYzw7ho=
 github.com/iancoleman/strcase v0.3.0 h1:nTXanmYxhfFAMjZL34Ov6gkzEsSJZ5DbhxWjvSASxEI=
 github.com/iancoleman/strcase v0.3.0/go.mod h1:iwCmte+B7n89clKwxIoIXy/HfoL7AsD47ZCWhYzw7ho=

e2e/sample.config.extended.yaml

@@ -7,6 +7,8 @@
 # | CHAIN_IMAGE          | The image that will be used for the chain | ghcr.io/cosmos/ibc-go-simd   |
 # | CHAIN_A_TAG          | The tag used for chain A                  | N/A (must be set)            |
 # | CHAIN_B_TAG          | The tag used for chain B                  | N/A (must be set)            |
+# | CHAIN_C_TAG          | The tag used for chain C                  | Optional (fallback to A)     |
+# | CHAIN_D_TAG          | The tag used for chain D                  | Optional (fallback to A)     |
 # | CHAIN_BINARY         | The binary used in the container          | simd                         |
 # | RELAYER_TAG          | The tag used for the relayer              | 1.10.4                       |
 # | RELAYER_ID           | The type of relayer to use (rly/hermes)   | hermes                       |
@@ -23,13 +25,29 @@ chains:
     tag: main # override with CHAIN_A_TAG
     binary: simd # override with CHAIN_BINARY
 
-    # the entry at index 1 corresponds to CHAIN_B
+  # the entry at index 1 corresponds to CHAIN_B
   - chainId: chainB-1
     numValidators: 4
     numFullNodes: 1
     image: ghcr.io/cosmos/ibc-go-simd # override with CHAIN_IMAGE
     tag: main # override with CHAIN_B_TAG
     binary: simd # override with CHAIN_BINARY
+
+  # the entry at index 2 corresponds to CHAIN_C
+  - chainId: chainC-1
+    numValidators: 4
+    numFullNodes: 1
+    image: ghcr.io/cosmos/ibc-go-simd # override with CHAIN_IMAGE
+    tag: main # override with CHAIN_C_TAG
+    binary: simd # override with CHAIN_BINARY
+
+  # the entry at index 3 corresponds to CHAIN_D
+  - chainId: chainD-1
+    numValidators: 4
+    numFullNodes: 1
+    image: ghcr.io/cosmos/ibc-go-simd # override with CHAIN_IMAGE
+    tag: main # override with CHAIN_D_TAG
+    binary: simd # override with CHAIN_BINARY
 
 # activeRelayer must match the id of a relayer specified in the relayers list below.
 activeRelayer: hermes # override with RELAYER_ID

e2e/sample.config.yaml

@@ -6,3 +6,7 @@ chains:
     chainId: chainA-1
   - tag: main # override with CHAIN_B_TAG
     chainId: chainB-1
+  - tag: main # override with CHAIN_C_TAG
+    chainId: chainC-1
+  - tag: main # override with CHAIN_D_TAG
+    chainId: chainD-1

e2e/tests/core/03-connection/connection_test.go

@@ -40,7 +40,7 @@ func (s *ConnectionTestSuite) SetupSuite() {
 }
 
 func (s *ConnectionTestSuite) CreateConnectionTestPath(testName string) (ibc.Relayer, ibc.ChannelOutput) {
-	return s.CreatePaths(ibc.DefaultClientOpts(), s.TransferChannelOptions(), testName), s.GetChainAChannelForTest(testName)
+	return s.CreatePaths(ibc.DefaultClientOpts(), s.TransferChannelOptions(), testName), s.GetChainAToChainBChannel(testName)
 }
 
 // QueryMaxExpectedTimePerBlockParam queries the on-chain max expected time per block param for 03-connection