Less things in the Types module (#297)

* Less things in the Types module
* remove Utils, clean up Types (#298)
---------

Co-authored-by: Brendan Brown <brendan.brown@iohk.io>

Author: berewt

doc/read-the-docs-site/doc/marconi-as-a-library/tutorials/BasicApp.hs

@@ -21,8 +21,9 @@ module Main where
 
 import Control.Concurrent (MVar)
 import Control.Concurrent.Async (race_)
+import Control.Exception (throwIO)
 import Control.Lens ((^.))
-import Control.Monad.Except (ExceptT, MonadError)
+import Control.Monad.Except (ExceptT, MonadError, runExceptT)
 import Control.Monad.IO.Class (MonadIO, liftIO)
 import Data.Aeson (FromJSON, ToJSON, (.=))
 import Data.Aeson qualified as Aeson
@@ -40,14 +41,13 @@ import Cardano.Api qualified as C
 import Cardano.BM.Setup qualified as Trace
 import Cardano.BM.Tracing qualified as Trace
 import Data.Void (Void)
-import Marconi.Cardano.ChainIndex.Utils qualified as Utils
-import Marconi.Cardano.Core.Logger (mkMarconiTrace)
+import Marconi.Cardano.ChainIndex.SecurityParam qualified as SecurityParam
+import Marconi.Cardano.Core.Logger (MarconiTrace, mkMarconiTrace)
 import Marconi.Cardano.Core.Node.Client.Retry qualified as Core
 import Marconi.Cardano.Core.Orphans ()
 import Marconi.Cardano.Core.Runner qualified as Core
 import Marconi.Cardano.Core.Types (
   BlockEvent (BlockEvent),
-  MarconiTrace,
   RetryConfig (RetryConfig),
   SecurityParam,
  )
@@ -104,7 +104,8 @@ withIndexerBuildEnv worker action = do
     -- We query the local node for the security parameter with a retry mechanism
     -- in case the node has not been started.
     k <- Core.withNodeConnectRetry marconiTrace (RetryConfig 30 Nothing) socketFilePath $ do
-      Utils.toException $ Utils.querySecurityParam @Void networkId socketFilePath
+      runExceptT (SecurityParam.querySecurityParam @Void networkId socketFilePath)
+        >>= either throwIO pure
 
     action (Env marconiTrace socketFilePath networkId k worker)
 
@@ -307,7 +308,7 @@ mkBlockInfoSqliteIndexerWorker
           -- Storage type of the indexer
       )
 mkBlockInfoSqliteIndexerWorker dbPath = do
-  ix <- Utils.toException $ mkBlockInfoSqliteIndexer dbPath
+  ix <- either throwIO pure =<< runExceptT (mkBlockInfoSqliteIndexer dbPath)
   Core.createWorker "BlockInfo" getEventsFromBlock ix
 
 -- | Creation of 'Core.SQLiteIndexer' for the 'BlockInfoEvent' indexer.

marconi-cardano-chain-index/marconi-cardano-chain-index.cabal

@@ -89,8 +89,8 @@ library
     Marconi.Cardano.ChainIndex.Git.RevFromGit
     Marconi.Cardano.ChainIndex.Indexers
     Marconi.Cardano.ChainIndex.Run
+    Marconi.Cardano.ChainIndex.SecurityParam
     Marconi.Cardano.ChainIndex.Snapshot.Run
-    Marconi.Cardano.ChainIndex.Utils
 
   other-modules:   Paths_marconi_cardano_chain_index
   autogen-modules: Paths_marconi_cardano_chain_index
@@ -111,7 +111,6 @@ library
     , cardano-api             ^>=8.20
     , cardano-crypto-class
     , cardano-ledger-shelley
-    , cardano-slotting        ^>=0.1.1.1
     , iohk-monitoring
     , ouroboros-consensus
 
@@ -189,7 +188,6 @@ test-suite marconi-cardano-chain-index-test
     , cardano-ledger-core
     , cardano-ledger-shelley
     , cardano-node-emulator
-    , cardano-slotting             ^>=0.1.1.1
     , hedgehog-extras
     , iohk-monitoring
     , ouroboros-consensus
@@ -302,7 +300,7 @@ library marconi-cardano-chain-index-test-lib
     , aeson
     , aeson-pretty
     , async
-    , base             >=4.9 && <5
+    , base                 >=4.9 && <5
     , bytestring
     , containers
     , contra-tracer
@@ -313,6 +311,7 @@ library marconi-cardano-chain-index-test-lib
     , hedgehog-extras
     , lens
     , mtl
+    , nonempty-containers
     , process
     , streaming
     , temporary

marconi-cardano-chain-index/src/Marconi/Cardano/ChainIndex/Api/JsonRpc/Endpoint/EpochState.hs

@@ -26,7 +26,6 @@ import Marconi.Cardano.ChainIndex.Indexers (
   queryableEpochNonce,
   queryableEpochSDD,
  )
-import Marconi.Cardano.ChainIndex.Utils qualified as Util
 import Marconi.Cardano.Indexers.EpochNonce qualified as EpochState
 import Marconi.Cardano.Indexers.EpochSDD qualified as EpochState
 import Marconi.Core qualified as Core
@@ -122,8 +121,8 @@ getEpochStakePoolDelegationHandler epochNo
       ActiveSDDResult
         (Lens.view EpochState.sddPoolId epochSDD)
         (Lens.view EpochState.sddLovelace epochSDD)
-        (Util.chainPointToSlotNo chainPoint)
-        (Util.chainPointToHash chainPoint)
+        (C.chainPointToSlotNo chainPoint)
+        (C.chainPointToHeaderHash chainPoint)
         (Lens.view EpochState.sddBlockNo epochSDD)
         (Lens.view EpochState.sddEpochNo epochSDD)
 
@@ -151,10 +150,10 @@ getEpochNonceHandler epochNo
     toEpochNonceResult Nothing = EpochNonceResult Nothing Nothing Nothing Nothing Nothing
     toEpochNonceResult (Just (Core.Timed chainPoint epochNonce)) =
       EpochNonceResult
-        (Util.chainPointToHash chainPoint)
+        (C.chainPointToHeaderHash chainPoint)
         (Just $ Lens.view EpochState.nonceBlockNo epochNonce)
         (Just $ Lens.view EpochState.nonceEpochNo epochNonce)
-        (Util.chainPointToSlotNo chainPoint)
+        (C.chainPointToSlotNo chainPoint)
         (nonceToMaybe $ Lens.view EpochState.nonceNonce epochNonce)
 
 nonceToMaybe :: Ledger.Nonce -> Maybe (Crypto.Hash Crypto.Blake2b_256 Ledger.Nonce)

marconi-cardano-chain-index/src/Marconi/Cardano/ChainIndex/CLI.hs

@@ -13,6 +13,7 @@ import Data.ByteString.Char8 qualified as C8
 import Data.List (nub)
 import Data.List.NonEmpty (NonEmpty)
 import Data.Proxy (Proxy (Proxy))
+import Data.Set.NonEmpty (NESet)
 import Data.Text (Text)
 import Data.Text qualified as Text
 import Data.Text.Encoding qualified as Text
@@ -30,17 +31,9 @@ import GHC.Generics (Generic)
 import Marconi.Cardano.ChainIndex.Git.Rev (gitRev)
 import Marconi.Cardano.Core.Orphans ()
 import Marconi.Cardano.Core.Types (
-  BlockRange,
   RetryConfig (RetryConfig),
-  TargetAddresses,
-  UtxoIndexerConfig (UtxoIndexerConfig),
-  addressDatumDbName,
-  epochStateDbName,
-  mintBurnDbName,
-  mkBlockRange,
-  scriptTxDbName,
-  utxoDbName,
  )
+import Marconi.Cardano.Indexers.SnapshotBlockEvent (BlockRange, mkBlockRange)
 import Options.Applicative (ReadM, eitherReader, execParserPure)
 import Paths_marconi_cardano_chain_index (version)
 
@@ -370,6 +363,9 @@ commonBatchSizeParser =
       <> Opt.help "Number of blocks sent as a batch to the indexers"
       <> Opt.showDefault
 
+-- | Type represents non empty set of Bech32 Shelley compatible addresses
+type TargetAddresses = NESet (C.Address C.ShelleyAddr)
+
 {- | Parse the addresses to index. Addresses should be given in Bech32 format
  Several addresses can be given in a single string, if they are separated by a space
 -}
@@ -473,9 +469,17 @@ commonRetryConfigParser =
               <> Opt.value 1_800
           )
 
+-- | Type defining the shape of UtxoIndexerConfig from CLI arguments.
+data UtxoIndexerCLIConfig = UtxoIndexerCLIConfig
+  { ucTargetAddresses :: Maybe TargetAddresses
+  -- ^ List of address utxo indexer to follow
+  , ucEnableUtxoTxOutRef :: Bool
+  -- ^ enable utxo indexer to store txOut refScript
+  }
+
 -- | Extract UtxoIndexerConfig from CLI Options
-mkUtxoIndexerConfig :: Options -> UtxoIndexerConfig
-mkUtxoIndexerConfig o = UtxoIndexerConfig (optionsTargetAddresses o) (optionsEnableUtxoTxOutRef o)
+mkUtxoIndexerConfig :: Options -> UtxoIndexerCLIConfig
+mkUtxoIndexerConfig o = UtxoIndexerCLIConfig (optionsTargetAddresses o) (optionsEnableUtxoTxOutRef o)
 
 -- | CL options for the marconi-chain-snapshot executable.
 data SnapshotOptions = SnapshotOptions
@@ -536,3 +540,20 @@ readBlockRange rawBlockRange =
         "Block range is not formatted correctly. "
           <> "Please provide a pair of positive numbers, "
           <> "without any spaces."
+
+-- * Database file names
+
+utxoDbName :: FilePath
+utxoDbName = "utxo.db"
+
+addressDatumDbName :: FilePath
+addressDatumDbName = "addressdatum.db"
+
+scriptTxDbName :: FilePath
+scriptTxDbName = "scripttx.db"
+
+epochStateDbName :: FilePath
+epochStateDbName = "epochstate.db"
+
+mintBurnDbName :: FilePath
+mintBurnDbName = "mintburn.db"

marconi-cardano-chain-index/src/Marconi/Cardano/ChainIndex/Indexers.hs

@@ -27,16 +27,14 @@ import Marconi.Cardano.Core.Indexer.Worker (
   StandardWorker (StandardWorker),
   StandardWorkerConfig (StandardWorkerConfig, eventExtractor, logger, workerName),
  )
+import Marconi.Cardano.Core.Logger (MarconiTrace)
 import Marconi.Cardano.Core.Transformer.WithSyncStats (WithSyncStats)
 import Marconi.Cardano.Core.Types (
   AnyTxBody (AnyTxBody),
   BlockEvent (BlockEvent),
-  BlockRange,
-  MarconiTrace,
   SecurityParam,
   TipAndBlock (TipAndBlock),
   TxIndexInBlock,
-  blockRangeFst,
  )
 import Marconi.Cardano.Indexers.BlockInfo qualified as BlockInfo
 import Marconi.Cardano.Indexers.ChainTip qualified as ChainTip
@@ -309,7 +307,7 @@ snapshotBlockEventBuilder
   -> Core.CatchupConfig
   -> BM.Trace m Text
   -> FilePath
-  -> BlockRange
+  -> SnapshotBlockEvent.BlockRange
   -> FilePath
   -> n
       ( Core.WorkerIndexer
@@ -353,7 +351,7 @@ buildIndexersForSnapshot
   -> BM.Trace IO Text
   -> MarconiTrace IO
   -> FilePath
-  -> [BlockRange]
+  -> [SnapshotBlockEvent.BlockRange]
   -> FilePath
   -> ExceptT
       Core.IndexerError
@@ -421,7 +419,7 @@ snapshotExtLedgerStateEventBuilder
   -> Core.CatchupConfig
   -> BM.Trace m Text
   -> FilePath
-  -> BlockRange
+  -> SnapshotBlockEvent.BlockRange
   -> FilePath
   -> n
       ( Core.WorkerIndexer
@@ -474,7 +472,10 @@ snapshotExtLedgerStateEventWorker standardWorkerConfig snapshotBlockEventWorkerC
   Core.createWorkerWithPreprocessing (workerName standardWorkerConfig) preprocessor indexer
 
 justBeforeBlockRangePreprocessor
-  :: (Monad m) => (a -> C.BlockNo) -> BlockRange -> Core.Preprocessor m C.ChainPoint a a
+  :: (Monad m)
+  => (a -> C.BlockNo)
+  -> SnapshotBlockEvent.BlockRange
+  -> Core.Preprocessor m C.ChainPoint a a
 justBeforeBlockRangePreprocessor toBlockNo br =
   Core.scanMaybeEvent filterJustBeforeBlockRange Nothing
   where
@@ -486,5 +487,5 @@ justBeforeBlockRangePreprocessor toBlockNo br =
         offset
           | leftRange == 0 = 0
           | otherwise = 1
-        leftRange = Lens.view blockRangeFst br
+        leftRange = Lens.view SnapshotBlockEvent.blockRangeFst br
         inputWord = C.unBlockNo . toBlockNo $ input

marconi-cardano-chain-index/src/Marconi/Cardano/ChainIndex/Run.hs

@@ -8,8 +8,8 @@ import Cardano.BM.Setup qualified as BM
 import Cardano.BM.Trace (logError, logInfo)
 import Cardano.BM.Tracing qualified as BM
 import Control.Concurrent.Async (race_)
-import Control.Exception (finally)
-import Control.Monad (guard, unless)
+import Control.Exception (finally, throwIO)
+import Control.Monad (guard, unless, (>=>))
 import Control.Monad.Except (runExceptT)
 import Control.Monad.Reader (runReaderT)
 import Data.Aeson (toJSON)
@@ -27,11 +27,11 @@ import Marconi.Cardano.ChainIndex.Api.Types (
  )
 import Marconi.Cardano.ChainIndex.CLI qualified as Cli
 import Marconi.Cardano.ChainIndex.Indexers (buildIndexers)
-import Marconi.Cardano.ChainIndex.Utils qualified as Utils
+import Marconi.Cardano.ChainIndex.SecurityParam qualified as SecurityParam
 import Marconi.Cardano.Core.Logger (defaultStdOutLogger, mkMarconiTrace)
 import Marconi.Cardano.Core.Node.Client.Retry (withNodeConnectRetry)
 import Marconi.Cardano.Core.Runner qualified as Runner
-import Marconi.Cardano.Core.Types (SecurityParam (SecurityParam), TargetAddresses)
+import Marconi.Cardano.Core.Types (SecurityParam (SecurityParam))
 import Marconi.Cardano.Indexers.MintTokenEvent qualified as MintTokenEvent
 import Marconi.Cardano.Indexers.Utxo qualified as Utxo
 import Marconi.Core qualified as Core
@@ -117,8 +117,8 @@ run appName = withGracefulTermination_ $ do
 
   securityParam <-
     withNodeConnectRetry marconiTrace retryConfig socketPath $
-      Utils.toException $
-        Utils.querySecurityParam @Void networkId socketPath
+      (runExceptT >=> either throwIO pure) $
+        SecurityParam.querySecurityParam @Void networkId socketPath
 
   let SecurityParam stopCatchupDistance = securityParam
       extLedgerStateAsEvent previousLedgerStateEvent ledgerStateEvent _blockEvent = do
@@ -184,7 +184,7 @@ run appName = withGracefulTermination_ $ do
     runHttpServer'
     `finally` BM.shutdown sb
 
-shelleyAddressesToAddressAny :: Maybe TargetAddresses -> [C.AddressAny]
+shelleyAddressesToAddressAny :: Maybe Cli.TargetAddresses -> [C.AddressAny]
 shelleyAddressesToAddressAny Nothing = []
 shelleyAddressesToAddressAny (Just targetAddresses) =
   fmap C.AddressShelley $ NEList.toList $ NESet.toList targetAddresses

marconi-cardano-chain-index/src/Marconi/Cardano/ChainIndex/SecurityParam.hs

@@ -0,0 +1,84 @@
+{-# LANGUAGE DerivingStrategies #-}
+{-# LANGUAGE OverloadedStrings #-}
+
+{- |
+Query the 'SecurityParam' from a running local node.
+-}
+module Marconi.Cardano.ChainIndex.SecurityParam (
+  querySecurityParam,
+  querySecurityParamEra,
+) where
+
+import Cardano.Api qualified as C
+import Cardano.Api.Extended.IPC qualified as CE
+import Cardano.Api.Extended.Shelley qualified as CE
+import Control.Monad.Except (
+  ExceptT,
+  throwError,
+ )
+import Control.Monad.Trans (MonadTrans (lift))
+import Data.Text (pack)
+import Marconi.Cardano.ChainIndex.Error (IndexerError (CantStartIndexer))
+import Marconi.Cardano.Core.Types (
+  SecurityParam,
+ )
+import Ouroboros.Consensus.HardFork.Combinator.AcrossEras (EraMismatch)
+
+{- | Query security param from the local node.
+ It queries the current era first, and uses that to query the security parameter.
+-}
+querySecurityParam
+  :: C.NetworkId
+  -> FilePath
+  -- ^ Node socket file path
+  -> ExceptT (IndexerError err) IO SecurityParam
+querySecurityParam networkId socketPath = do
+  (C.AnyCardanoEra era) <- queryCurrentEra networkId socketPath
+  case CE.cardanoEraToShelleyBasedEra era of
+    Nothing -> throwError $ CantStartIndexer "The security parameter can only be queried in shelley based era."
+    Just shelleyBasedEra -> querySecurityParamEra shelleyBasedEra networkId socketPath
+
+-- | Query the current era of the local node's current state.
+queryCurrentEra
+  :: C.NetworkId
+  -> FilePath
+  -- ^ Node socket file path
+  -> ExceptT (IndexerError err) IO C.AnyCardanoEra
+queryCurrentEra networkId socketPath = do
+  let localNodeConnectInfo :: C.LocalNodeConnectInfo C.CardanoMode
+      localNodeConnectInfo = CE.mkLocalNodeConnectInfo networkId socketPath
+
+      queryInMode :: C.QueryInMode C.CardanoMode C.AnyCardanoEra
+      queryInMode = C.QueryCurrentEra C.CardanoModeIsMultiEra
+
+  result <- lift $ C.queryNodeLocalState localNodeConnectInfo Nothing queryInMode
+  case result of
+    Left err -> toError err
+    Right x -> pure x
+
+-- | Query security param from the local node given a Shelley based era.
+querySecurityParamEra
+  :: forall era err
+   . C.ShelleyBasedEra era
+  -> C.NetworkId
+  -> FilePath
+  -- ^ Node socket file path
+  -> ExceptT (IndexerError err) IO SecurityParam
+querySecurityParamEra shelleyBasedEra networkId socketPath = do
+  result <- lift $ C.queryNodeLocalState localNodeConnectInfo Nothing queryInMode
+  genesisParameters <- case result of
+    Left err -> toError err
+    Right (Left err) -> toError err
+    Right (Right x) -> pure x
+  pure $ fromIntegral $ C.protocolParamSecurity genesisParameters
+  where
+    localNodeConnectInfo :: C.LocalNodeConnectInfo C.CardanoMode
+    localNodeConnectInfo = CE.mkLocalNodeConnectInfo networkId socketPath
+
+    queryInMode :: C.QueryInMode C.CardanoMode (Either EraMismatch (C.GenesisParameters C.ShelleyEra))
+    queryInMode =
+      C.QueryInEra (CE.toShelleyEraInCardanoMode shelleyBasedEra) $
+        C.QueryInShelleyBasedEra shelleyBasedEra C.QueryGenesisParameters
+
+toError :: (Show a) => a -> ExceptT (IndexerError err) IO b
+toError = throwError . CantStartIndexer . pack . show