Skip to content
This repository was archived by the owner on Dec 2, 2024. It is now read-only.

Commit 7a4f8d3

Browse files
kayvankkayvank
committed
report-marconi-target-addresses-v2 PLT-1009
Changed Cardano.Api From/To Filed SQL mappings to RawByteString
1 parent 25adf2e commit 7a4f8d3

File tree

9 files changed

+105
-90
lines changed

9 files changed

+105
-90
lines changed

‎marconi-mamba/db-utils/exe/Main.hs‎

Lines changed: 2 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -4,8 +4,7 @@ module Main where
44
import Cardano.Api (NetworkId)
55
import Marconi.Api.Types (TargetAddresses)
66
import Marconi.Api.UtxoIndexersQuery qualified as Q.Utxo
7-
import Marconi.Bootstrap (targetAddressParser)
8-
import Marconi.CLI (pNetworkId)
7+
import Marconi.CLI (pNetworkId, targetAddressParser)
98
import Marconi.DB.SqlUtils (freqShelleyTable, freqUtxoTable)
109
import Options.Applicative (Parser, execParser, help, helper, info, long, metavar, short, showDefault, strOption, value,
1110
(<**>))
@@ -25,7 +24,7 @@ cliParser = CliOptions
2524
<> short 'd'
2625
<> metavar "FILENAME"
2726
<> showDefault
28-
<> value "./.marconidb/2/utxo-db"
27+
<> value "./.marconidb/2/utxodb"
2928
<> help "Path to the utxo database.")
3029
<*> pNetworkId
3130

@@ -39,5 +38,4 @@ main = do
3938
freqUtxoTable dbEnv
4039
as <- freqShelleyTable dbEnv
4140
print (length as)
42-
4341
print "end"

‎marconi-mamba/db-utils/src/Marconi/DB/SqlUtils.hs‎

Lines changed: 3 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -52,7 +52,7 @@ freqUtxoTable env =
5252
execute_ conn "drop table if exists frequtxos"
5353
>> execute_ conn "drop table if exists shelleyaddresses"
5454
>> execute_ conn "create table frequtxos as select address, count (address) as frequency from utxos group by address order by frequency DESC"
55-
>> execute_ conn "delete from frequtxos where frequency < 50"
55+
>> execute_ conn "delete from frequtxos where frequency < 50" -- we only want `intersing` data
5656
>> execute_ conn
5757
"create TABLE shelleyaddresses (address text not null, frequency int not null)"
5858
)
@@ -93,10 +93,9 @@ freqShelleyTable env = do
9393
-- first conver to cardano address, then seriase to text
9494
--
9595
toShelley :: C.NetworkId -> ShelleyFrequencyTable C.AddressAny -> Maybe (ShelleyFrequencyTable Text)
96-
toShelley nid (ShelleyFrequencyTable (C.AddressShelley a) f) =
96+
toShelley _ (ShelleyFrequencyTable (C.AddressShelley a) f) =
9797
let
9898
addrTxt = C.serialiseAddress a
9999
in
100100
Just ( ShelleyFrequencyTable addrTxt f)
101-
toShelley nid _ = Nothing
102-
101+
toShelley _ _ = Nothing

‎marconi-mamba/examples/json-rpc-client/src/Main.hs‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -9,9 +9,9 @@ module Main where
99
import Control.Monad (void)
1010
import Control.Monad.IO.Class (liftIO)
1111
import Data.Proxy (Proxy (Proxy))
12-
import Ledger.Tx (TxOutRef)
1312
import Marconi.Client.Types (JsonRpcResponse)
1413
import Marconi.JsonRpc.Types (JsonRpc, JsonRpcNotification, RawJsonRpc)
14+
import Marconi.Types (TxOutRef)
1515
import Network.HTTP.Client (defaultManagerSettings, newManager)
1616
import Servant.API (Get, NoContent, PlainText, Post, ReqBody, (:<|>) ((:<|>)), (:>))
1717
import Servant.Client (ClientM, client, mkClientEnv, parseBaseUrl, runClientM)

‎marconi-mamba/marconi-mamba.cabal‎

Lines changed: 7 additions & 27 deletions
Original file line numberDiff line numberDiff line change
@@ -62,7 +62,6 @@ library
6262
build-depends:
6363
, json-rpc
6464
, marconi
65-
, plutus-ledger
6665
, rewindable-index
6766

6867
------------------------
@@ -105,11 +104,6 @@ executable marconi-mamba
105104
hs-source-dirs: app
106105
main-is: Main.hs
107106

108-
--------------------------
109-
-- Other IOG dependencies
110-
--------------------------
111-
build-depends: cardano-api
112-
113107
--------------------
114108
-- Local components
115109
--------------------
@@ -141,21 +135,16 @@ executable examples-json-rpc-server
141135
build-depends:
142136
, marconi
143137
, marconi-mamba
144-
, plutus-ledger
145-
, rewindable-index
146138

147139
------------------------
148140
-- Non-IOG dependencies
149141
------------------------
150142
build-depends:
151143
, async
152144
, base >=4.9 && <5
153-
, containers
154145
, lens
155146
, optparse-applicative
156147
, stm
157-
, vector
158-
, warp
159148

160149
executable examples-json-rpc-client
161150
import: lang
@@ -167,7 +156,7 @@ executable examples-json-rpc-client
167156
--------------------
168157
build-depends:
169158
, json-rpc
170-
, plutus-ledger
159+
, marconi
171160

172161
------------------------
173162
-- Non-IOG dependencies
@@ -186,31 +175,23 @@ library db-utils
186175
--------------------------
187176
-- Other IOG dependencies
188177
--------------------------
189-
build-depends:
190-
, cardano-api
191-
, cardano-streaming
178+
build-depends: cardano-api
192179

193180
--------------------
194181
-- Local components
195182
--------------------
196183
build-depends:
197-
, json-rpc
198184
, marconi
199185
, marconi-mamba
200-
, plutus-ledger
201186

187+
-- , json-rpc
202188
build-depends:
203-
, aeson
204189
, async
205-
, base >=4.9 && <5
206-
, containers
190+
, base >=4.9 && <5
207191
, lens
208-
, servant
209-
, servant-server
210192
, sqlite-simple
211193
, text
212194
, time
213-
, warp
214195

215196
executable db-utils-exe
216197
import: lang
@@ -220,9 +201,7 @@ executable db-utils-exe
220201
--------------------------
221202
-- Other IOG dependencies
222203
--------------------------
223-
build-depends:
224-
, cardano-api
225-
, cardano-streaming
204+
build-depends: cardano-api
226205

227206
--------------------
228207
-- Local components
@@ -233,7 +212,8 @@ executable db-utils-exe
233212
, marconi-mamba
234213

235214
------------------------
236-
215+
-- Non-IOG dependencies
216+
------------------------
237217
build-depends:
238218
, base >=4.9 && <5
239219
, optparse-applicative

‎marconi-mamba/src/Marconi/Api/HttpServer.hs‎

Lines changed: 8 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -16,9 +16,9 @@ import Data.Proxy (Proxy (Proxy))
1616
import Data.Set (Set)
1717
import Data.Text (Text, pack)
1818
import Data.Time (defaultTimeLocale, formatTime, getCurrentTime)
19-
import Ledger.Tx.CardanoAPI (ToCardanoError)
2019
import Marconi.Api.Routes (API)
21-
import Marconi.Api.Types (DBQueryEnv, HasJsonRpcEnv (httpSettings, queryEnv), JsonRpcEnv, UtxoTxOutReport)
20+
import Marconi.Api.Types (DBQueryEnv, HasJsonRpcEnv (httpSettings, queryEnv), JsonRpcEnv, QueryExceptions,
21+
UtxoTxOutReport)
2222
import Marconi.Api.UtxoIndexersQuery qualified as Q.Utxo (findAll, findByAddress, reportBech32Addresses)
2323
import Marconi.JsonRpc.Types (JsonRpcErr (JsonRpcErr, errorCode, errorData, errorMessage), parseErrorCode)
2424
import Marconi.Server.Types ()
@@ -81,7 +81,7 @@ utxoTxOutReport
8181
-> String -- ^ bech32 addressCredential
8282
-> Handler (Either (JsonRpcErr String) UtxoTxOutReport )
8383
utxoTxOutReport env address = liftIO $
84-
bimap cardanoErrToRpcErr id <$> (Q.Utxo.findByAddress env . pack $ address)
84+
bimap toRpcErr id <$> (Q.Utxo.findByAddress env . pack $ address)
8585

8686
-- | Retrieves a set of TxOutRef
8787
-- TODO convert this to stream
@@ -98,12 +98,12 @@ targetAddressesReport
9898
-> Handler (Either (JsonRpcErr String) (Set Text) )
9999
targetAddressesReport env _ = pure . Right . Q.Utxo.reportBech32Addresses $ env
100100

101-
-- | convert form cardano error, to jsonrpc protocal error
102-
cardanoErrToRpcErr
103-
:: ToCardanoError
101+
-- | convert form to jsonrpc protocal error
102+
toRpcErr
103+
:: QueryExceptions
104104
-> JsonRpcErr String
105-
cardanoErrToRpcErr e = JsonRpcErr {
105+
toRpcErr e = JsonRpcErr {
106106
errorCode = parseErrorCode
107-
, errorMessage = "address deserialization or conversion related error."
107+
, errorMessage = "marconi RPC query related error!"
108108
, errorData = Just . show $ e
109109
}

‎marconi-mamba/src/Marconi/Api/Types.hs‎

Lines changed: 13 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,9 +1,12 @@
11
{-# LANGUAGE DataKinds #-}
2+
{-# LANGUAGE DeriveAnyClass #-}
23
{-# LANGUAGE DeriveGeneric #-}
4+
{-# LANGUAGE DerivingStrategies #-}
35
{-# LANGUAGE FlexibleInstances #-}
46
{-# LANGUAGE LambdaCase #-}
57
{-# LANGUAGE MultiParamTypeClasses #-}
68
{-# LANGUAGE RankNTypes #-}
9+
{-# LANGUAGE StandaloneDeriving #-}
710
{-# LANGUAGE TemplateHaskell #-}
811
{-# LANGUAGE TypeApplications #-}
912
{-# LANGUAGE TypeFamilies #-}
@@ -27,8 +30,10 @@ module Marconi.Api.Types
2730
, UtxoTxOutReport (..)
2831
, UtxoQueryComm (..)
2932
, HasUtxoQueryComm (..)
33+
, QueryExceptions (..)
3034
) where
3135
import Cardano.Api (AddressAny, NetworkId, anyAddressInShelleyBasedEra)
36+
import Control.Exception (Exception)
3237
import Control.Lens (makeClassy)
3338
import Data.Aeson (ToJSON (toEncoding, toJSON), defaultOptions, genericToEncoding)
3439
import Data.Set (Set)
@@ -94,3 +99,11 @@ instance ToJSON UtxoRowWrapper where
9499

95100
instance ToJSON UtxoRow where
96101
toEncoding = genericToEncoding defaultOptions
102+
103+
data QueryExceptions
104+
= AddressNotInListError QueryExceptions
105+
| AddressConversionError QueryExceptions
106+
| TxRefConversionError QueryExceptions
107+
| QueryError String
108+
deriving stock Show
109+
deriving anyclass Exception

‎marconi-mamba/src/Marconi/Api/UtxoIndexersQuery.hs‎

Lines changed: 53 additions & 36 deletions
Original file line numberDiff line numberDiff line change
@@ -24,21 +24,23 @@ import Data.Set (Set, fromList, toList, union)
2424
import Data.Text (Text, intercalate, unpack)
2525
import Database.SQLite.Simple (NamedParam ((:=)), open)
2626
import Database.SQLite.Simple qualified as SQL
27-
import Ledger.Tx.CardanoAPI (ToCardanoError (DeserialisationError, Tag, TxBodyError))
2827
import Marconi.Api.Types (DBConfig (DBConfig, utxoConn),
2928
DBQueryEnv (DBQueryEnv, _DbConf, _Network, _QueryAddresses, _QueryComm),
3029
HasDBQueryEnv (dbConf, queryAddresses, queryComm), HasUtxoQueryComm (indexer, queryReq),
31-
TargetAddresses, UtxoQueryComm (UtxoQueryComm, _Indexer, _QueryReq),
32-
UtxoTxOutReport (UtxoTxOutReport))
30+
QueryExceptions (AddressNotInListError, QueryError), TargetAddresses,
31+
UtxoQueryComm (UtxoQueryComm, _Indexer, _QueryReq), UtxoTxOutReport (UtxoTxOutReport))
3332
import Marconi.Index.Utxo (UtxoIndex, UtxoRow (UtxoRow, _reference), toRows)
3433
import Marconi.Types (CardanoAddress, TxOutRef)
3534
import RewindableIndex.Index.VSqlite qualified as Ix
3635

36+
-- | Bootstraps the utxo query environment.
37+
-- The module is responsible for accessing SQLite for quries.
38+
-- The main issue we try to avoid here is mixing inserts and quries in SQLite to avoid locking the database
3739
bootstrap
38-
:: FilePath
39-
-> TargetAddresses
40-
-> CApi.NetworkId
41-
-> IO DBQueryEnv
40+
:: FilePath -- ^ file path the SQLite utxos database
41+
-> TargetAddresses -- ^ user provided target addresses
42+
-> CApi.NetworkId -- ^ cardano networkId
43+
-> IO DBQueryEnv -- ^ returns Query runtime environment
4244
bootstrap dbPath targetAddresses nId = do
4345
dbconf <- DBConfig <$> open dbPath
4446
_QueryReq <- atomically (newEmptyTMVar :: STM (TMVar () ) )
@@ -49,68 +51,81 @@ bootstrap dbPath targetAddresses nId = do
4951
, _QueryAddresses = targetAddresses
5052
, _Network = nId
5153
}
52-
54+
-- | finds reports for all user-provided addresses.
55+
-- TODO we need to use streaming
56+
--
5357
findAll
54-
:: DBQueryEnv
55-
-> IO (Set UtxoTxOutReport) -- ^ set of corresponding TxOutRefs
58+
:: DBQueryEnv -- ^ Query run time environment
59+
-> IO (Set UtxoTxOutReport) -- ^ set of corresponding TxOutRefs
5660
findAll env = fromList <$> forConcurrently addresses f
5761
where
5862
addresses = NonEmpty.toList (env ^. queryAddresses)
5963
f :: CardanoAddress -> IO (UtxoTxOutReport)
6064
f addr = UtxoTxOutReport (CApi.serialiseAddress addr) <$> findByCardanoAddress env (CApi.toAddressAny addr)
6165

66+
-- | Query utxos address address
67+
--
6268
utxoQuery:: DBConfig -> CApi.AddressAny -> IO (Set TxOutRef)
63-
utxoQuery dbConfig address = SQL.queryNamed (utxoConn dbConfig)
64-
"SELECT * FROM utxos WHERE utxos.address=:address"
69+
utxoQuery dbConfig address = putStrLn "utxoquery" >> SQL.queryNamed (utxoConn dbConfig)
70+
"SELECT txid, inputIx FROM utxos WHERE utxos.address=:address"
6571
[":address" := address]
66-
>>= pure . fromList . fmap _reference
72+
>>= pure . fromList -- . fmap _reference
6773

74+
-- | Query utxos by Cardano Address
75+
-- To Cardano error may occure
6876
findByCardanoAddress
69-
:: DBQueryEnv
70-
-> CApi.AddressAny
71-
-> IO (Set TxOutRef) -- ^ To Plutus address conversion error may occure
77+
:: DBQueryEnv -- ^ Query run time environment
78+
-> CApi.AddressAny -- ^ Cardano address to query
79+
-> IO (Set TxOutRef)
7280
findByCardanoAddress env address = withQueryAction env address utxoQuery
7381

7482
-- | Retrieve a Set of TxOutRefs associated with the given Cardano Era address
7583
-- We return an empty Set if no address is found
7684
findByAddress
77-
:: DBQueryEnv
85+
:: DBQueryEnv -- ^ Query run time environment
7886
-> Text -- ^ Bech32 Address
79-
-> IO (Either ToCardanoError UtxoTxOutReport) -- ^ To Plutus address conversion error may occure
87+
-> IO (Either QueryExceptions UtxoTxOutReport) -- ^ To Plutus address conversion error may occure
8088
findByAddress env addressText =
8189
let
82-
f :: Either CApi.Bech32DecodeError CardanoAddress -> IO (Either ToCardanoError UtxoTxOutReport)
90+
f :: Either CApi.Bech32DecodeError CardanoAddress -> IO (Either QueryExceptions UtxoTxOutReport)
8391
f (Right address)
84-
| address `elem` (env ^. queryAddresses) =
92+
| address `elem` (env ^. queryAddresses) = -- allow for targetAddress search only
8593
(pure . CApi.toAddressAny $ address)
8694
>>= findByCardanoAddress env
8795
>>= pure . Right . UtxoTxOutReport addressText
88-
| otherwise = pure . Left . TxBodyError $
96+
| otherwise = pure . Left . AddressNotInListError . QueryError $
8997
unpack addressText <> " not in the provided target addresses"
90-
f (Left e) = pure . Left $ Tag (unpack addressText
98+
f (Left e) = pure . Left $ QueryError (unpack addressText
9199
<> " generated error: "
92100
<> show e)
93-
DeserialisationError
101+
94102
in
95103
f $ CApi.deserialiseFromBech32 CApi.AsShelleyAddress addressText
96104

97-
98-
queryInMemory :: CApi.AddressAny -> UtxoIndex-> IO ( Set TxOutRef )
105+
-- | query in-momory utxos for the given address
106+
--
107+
queryInMemory
108+
:: CApi.AddressAny -- ^ address to query
109+
-> UtxoIndex -- ^ inmemory, hot-store, storage for utxos
110+
-> IO ( Set TxOutRef )
99111
queryInMemory address ix =
100-
Ix.getBuffer (ix ^. Ix.storage)
101-
>>= pure
112+
let
113+
isTargetAddress :: UtxoRow -> Bool
114+
isTargetAddress (UtxoRow a _ ) = address == a
115+
in
116+
Ix.getBuffer (ix ^. Ix.storage)
117+
>>= pure
102118
. fromList
103119
. fmap _reference
104-
. filter (isTargetAddress address)
120+
. filter isTargetAddress
105121
. concatMap toRows
106122

107-
isTargetAddress :: CApi.AddressAny -> UtxoRow -> Bool
108-
isTargetAddress address (UtxoRow a _ ) = address == a
109-
123+
-- | Execute the query function
124+
-- We must stop the utxo insers before doing the query
110125
withQueryAction
111-
:: DBQueryEnv
112-
-> CApi.AddressAny
113-
-> (DBConfig -> CApi.AddressAny -> IO (Set TxOutRef) )
126+
:: DBQueryEnv -- ^ Query run time environment
127+
-> CApi.AddressAny -- ^ Cardano address to query
128+
-> (DBConfig -> CApi.AddressAny -> IO (Set TxOutRef) ) -- ^ Query function to run
114129
-> IO (Set TxOutRef)
115130
withQueryAction env address qAction =
116131
let
@@ -123,10 +138,12 @@ withQueryAction env address qAction =
123138
pure . union fromColdStore $ fromHotStore
124139
in
125140
bracket_
126-
(atomically $ putTMVar qreq () )
127-
(atomically $ putTMVar qreq () )
141+
(atomically $ putTMVar qreq () ) -- block inserts
142+
(atomically $ putTMVar qreq () ) -- unblock inserts
128143
action
129144

145+
-- | report target addresses
146+
-- Used by JSON-RPC
130147
reportQueryAddresses
131148
:: DBQueryEnv
132149
-> IO (Set CardanoAddress)

0 commit comments

Comments
 (0)