REST API changes for unlimited assets (#872)

Adds new REST endpoints and "exclude" parameter for account information endpoint, similar to algorand/go-algorand#3542. Relies on #870

Update endpoint: GET /v2/accounts/{address}

Supports new query parameter “exclude”, which can be a combination of all, created-apps, created-assets, apps-local-state, assets
Removes empty “creator” string from AssetHolding type — was not implemented (removed associated TODO)
Update endpoint: GET /v2/assets/{asset-id}/balances

Remove "round" query parameter — I noticed support is not implemented, and the parameter was being ignored
New endpoints:
GET /v2/accounts/{address}/created-apps
GET /v2/accounts/{address}/created-assets
GET /v2/accounts/{address}/assets
GET /v2/accounts/{address}/apps-local-state

Supported query parameters: asset-id / application-id (look up a single asset), include-all (include deleted), limit, next (for pagination)

.circleci/config.yml

@@ -83,7 +83,7 @@ commands:
           name: Install python and other python dependencies
           command: |
             sudo apt update
-            sudo apt -y install python3 python3-pip python3-setuptools python3-wheel libboost-all-dev libffi-dev
+            sudo apt -y install python3 python3-pip python3-setuptools python3-wheel libboost-math-dev libffi-dev
             pip3 install -r misc/requirements.txt
 
       - run:

Makefile

@@ -11,6 +11,8 @@ GOLDFLAGS += -X github.com/algorand/indexer/version.CompileTime=$(shell date -u
 GOLDFLAGS += -X github.com/algorand/indexer/version.GitDecorateBase64=$(shell git log -n 1 --pretty="%D"|base64|tr -d ' \n')
 GOLDFLAGS += -X github.com/algorand/indexer/version.ReleaseVersion=$(shell cat .version)
 
+COVERPKG := $(shell go list ./...  | grep -v '/cmd/' | egrep -v '(testing|test|mocks)$$' |  paste -s -d, - )
+
 # Used for e2e test
 export GO_IMAGE = golang:$(shell go version | cut -d ' ' -f 3 | tail -c +3 )
 
@@ -45,7 +47,7 @@ fakepackage: go-algorand
 	misc/release.py --host-only --outdir $(PKG_DIR) --fake-release
 
 test: idb/mocks/IndexerDb.go cmd/algorand-indexer/algorand-indexer
-	go test ./... -coverprofile=coverage.txt -covermode=atomic
+	go test -coverpkg=$(COVERPKG) ./... -coverprofile=coverage.txt -covermode=atomic
 
 lint: go-algorand
 	golint -set_exit_status ./...

README.md

@@ -134,20 +134,33 @@ If the maximum number of connections/active queries is reached, subsequent conne
 
 Settings can be provided from the command line, a configuration file, or an environment variable
 
-| Command Line Flag (long) | (short) | Config File                | Environment Variable               |
-| ------------------------ | ------- | -------------------------- | ---------------------------------- |
-| postgres                 | P       | postgres-connection-string | INDEXER_POSTGRES_CONNECTION_STRING |
-| pidfile                  |         | pidfile                    | INDEXER_PIDFILE                    |
-| algod                    | d       | algod-data-dir             | INDEXER_ALGOD_DATA_DIR             |
-| algod-net                |         | algod-address              | INDEXER_ALGOD_ADDRESS              |
-| algod-token              |         | algod-token                | INDEXER_ALGOD_TOKEN                |
-| genesis                  | g       | genesis                    | INDEXER_GENESIS                    |
-| server                   | S       | server-address             | INDEXER_SERVER_ADDRESS             |
-| no-algod                 |         | no-algod                   | INDEXER_NO_ALGOD                   |
-| token                    | t       | api-token                  | INDEXER_API_TOKEN                  |
-| dev-mode                 |         | dev-mode                   | INDEXER_DEV_MODE                   |
-| metrics-mode             |         | metrics-mode               | INDEXER_METRICS_MODE               |
-| max-conn                 |         | max-conn                   | INDEXER_MAX_CONN                   |
+| Command Line Flag (long)   | (short) | Config File                | Environment Variable               |
+|----------------------------|---------|----------------------------|------------------------------------|
+| postgres                   | P       | postgres-connection-string | INDEXER_POSTGRES_CONNECTION_STRING |
+| pidfile                    |         | pidfile                    | INDEXER_PIDFILE                    |
+| algod                      | d       | algod-data-dir             | INDEXER_ALGOD_DATA_DIR             |
+| algod-net                  |         | algod-address              | INDEXER_ALGOD_ADDRESS              |
+| algod-token                |         | algod-token                | INDEXER_ALGOD_TOKEN                |
+| genesis                    | g       | genesis                    | INDEXER_GENESIS                    |
+| server                     | S       | server-address             | INDEXER_SERVER_ADDRESS             |
+| no-algod                   |         | no-algod                   | INDEXER_NO_ALGOD                   |
+| token                      | t       | api-token                  | INDEXER_API_TOKEN                  |
+| dev-mode                   |         | dev-mode                   | INDEXER_DEV_MODE                   |
+| metrics-mode               |         | metrics-mode               | INDEXER_METRICS_MODE               |
+| max-conn                   |         | max-conn                   | INDEXER_MAX_CONN                   |
+| write-timeout              |         | write-timeout              | INDEXER_WRITE_TIMEOUT              |
+| read-timeout               |         | read-timeout               | INDEXER_READ_TIMEOUT               |
+| max-account-nested-objects |         | max-account-nested-objects | INDEXER_MAX_ACCOUNT_NESTED_OBJECTS |
+| max-transactions-limit     |         | max-transactions-limit     | INDEXER_MAX_TRANSACTIONS_LIMIT     |
+| default-transactions-limit |         | default-transactions-limit | INDEXER_DEFAULT_TRANSACTIONS_LIMIT |
+| max-accounts-limit         |         | max-accounts-limit         | INDEXER_MAX_ACCOUNTS_LIMIT         |
+| default-accounts-limit     |         | default-accounts-limit     | INDEXER_DEFAULT_ACCOUNTS_LIMIT     |
+| max-assets-limit           |         | max-assets-limit           | INDEXER_MAX_ASSETS_LIMIT           |
+| default-assets-limit       |         | default-assets-limit       | INDEXER_DEFAULT_ASSETS_LIMIT       |
+| max-balances-limit         |         | max-balances-limit         | INDEXER_MAX_BALANCES_LIMIT         |
+| default-balances-limit     |         | default-balances-limit     | INDEXER_DEFAULT_BALANCES_LIMIT     |
+| max-applications-limit     |         | max-applications-limit     | INDEXER_MAX_APPLICATIONS_LIMIT     |
+| default-applications-limit |         | default-applications-limit | INDEXER_DEFAULT_APPLICATIONS_LIMIT |
 
 ## Command line
 

api/converter_utils.go

@@ -525,7 +525,7 @@ func signedTxnWithAdToTransaction(stxn *transactions.SignedTxnWithAD, extra rowD
 	return txn, nil
 }
 
-func assetParamsToAssetQuery(params generated.SearchForAssetsParams) (idb.AssetsQuery, error) {
+func (si *ServerImplementation) assetParamsToAssetQuery(params generated.SearchForAssetsParams) (idb.AssetsQuery, error) {
 	creator, errorArr := decodeAddress(params.Creator, "creator", make([]string, 0))
 	if len(errorArr) != 0 {
 		return idb.AssetsQuery{}, errors.New(errUnableToParseAddress)
@@ -548,21 +548,45 @@ func assetParamsToAssetQuery(params generated.SearchForAssetsParams) (idb.Assets
 		Unit:               strOrDefault(params.Unit),
 		Query:              "",
 		IncludeDeleted:     boolOrDefault(params.IncludeAll),
-		Limit:              min(uintOrDefaultValue(params.Limit, defaultAssetsLimit), maxAssetsLimit),
+		Limit:              min(uintOrDefaultValue(params.Limit, si.opts.DefaultAssetsLimit), si.opts.MaxAssetsLimit),
 	}
 
 	return query, nil
 }
 
-func transactionParamsToTransactionFilter(params generated.SearchForTransactionsParams) (filter idb.TransactionFilter, err error) {
+func (si *ServerImplementation) appParamsToApplicationQuery(params generated.SearchForApplicationsParams) (idb.ApplicationQuery, error) {
+	addr, errorArr := decodeAddress(params.Creator, "creator", make([]string, 0))
+	if len(errorArr) != 0 {
+		return idb.ApplicationQuery{}, errors.New(errUnableToParseAddress)
+	}
+
+	var appGreaterThan uint64 = 0
+	if params.Next != nil {
+		agt, err := strconv.ParseUint(*params.Next, 10, 64)
+		if err != nil {
+			return idb.ApplicationQuery{}, fmt.Errorf("%s: %v", errUnableToParseNext, err)
+		}
+		appGreaterThan = agt
+	}
+
+	return idb.ApplicationQuery{
+		ApplicationID:            uintOrDefault(params.ApplicationId),
+		ApplicationIDGreaterThan: appGreaterThan,
+		Address:                  addr,
+		IncludeDeleted:           boolOrDefault(params.IncludeAll),
+		Limit:                    min(uintOrDefaultValue(params.Limit, si.opts.DefaultApplicationsLimit), si.opts.MaxApplicationsLimit),
+	}, nil
+}
+
+func (si *ServerImplementation) transactionParamsToTransactionFilter(params generated.SearchForTransactionsParams) (filter idb.TransactionFilter, err error) {
 	var errorArr = make([]string, 0)
 
 	// Integer
 	filter.MaxRound = uintOrDefault(params.MaxRound)
 	filter.MinRound = uintOrDefault(params.MinRound)
 	filter.AssetID = uintOrDefault(params.AssetId)
 	filter.ApplicationID = uintOrDefault(params.ApplicationId)
-	filter.Limit = min(uintOrDefaultValue(params.Limit, defaultTransactionsLimit), maxTransactionsLimit)
+	filter.Limit = min(uintOrDefaultValue(params.Limit, si.opts.DefaultTransactionsLimit), si.opts.MaxTransactionsLimit)
 
 	// filter Algos or Asset but not both.
 	if filter.AssetID != 0 {
@@ -610,3 +634,17 @@ func transactionParamsToTransactionFilter(params generated.SearchForTransactions
 
 	return
 }
+
+func (si *ServerImplementation) maxAccountsErrorToAccountsErrorResponse(maxErr idb.MaxAccountNestedObjectsError) generated.AccountsErrorResponse {
+	addr := maxErr.Address.String()
+	max := uint64(si.opts.MaxAccountNestedObjects)
+	return generated.AccountsErrorResponse{
+		Message:            "Result limit exceeded",
+		MaxResults:         &max,
+		Address:            &addr,
+		TotalAssetsOptedIn: &maxErr.TotalAssets,
+		TotalCreatedAssets: &maxErr.TotalAssetParams,
+		TotalAppsOptedIn:   &maxErr.TotalAppLocalStates,
+		TotalCreatedApps:   &maxErr.TotalAppParams,
+	}
+}