Merge branch 'release/2.1.0-beta'

Author: onetechnical

.travis.yml

@@ -16,10 +16,12 @@ addons:
       - python3-wheel
 
 install:
-  - pip3 install boto3
-    #- docker run -d -e POSTGES_USER=algorand -e POSTGRES_PASSWORD=indexer -e POSTGRES_DB=indexer-db -p 5432:5432 -t postgres
+  - pip3 install boto3 markdown2
 
 script:
   - make
   - go test ./...
-    #- python3 misc/e2etest.py "host=localhost port=5432 user=algorand password=indexer dbname='indexer-db' sslmode=disable"
+  - make package
+  - docker build -t e2e_tests -f docker/Dockerfile.mule .
+  - docker run --rm -t e2e_tests mule/e2e.sh
+

.version

@@ -1 +1 @@
-2.0.3
+2.1.0-beta

Jenkinsfile

@@ -0,0 +1,4 @@
+@Library('go-algorand-ci')_
+
+muleCI('mule.yaml')
+

Makefile

@@ -1,7 +1,8 @@
 SRCPATH		:= $(shell pwd)
-OS_TYPE		:= $(shell ./scripts/ostype.sh)
-   ARCH		:= $(shell ./scripts/archtype.sh)
-PKG_DIR		= $(SRCPATH)/tmp/node_pkgs/$(OS_TYPE)/$(ARCH)
+VERSION		:= $(shell $(SRCPATH)/mule/scripts/compute_build_number.sh)
+OS_TYPE		:= $(shell $(SRCPATH)/mule/scripts/ostype.sh)
+ARCH		:= $(shell $(SRCPATH)/mule/scripts/archtype.sh)
+PKG_DIR		= $(SRCPATH)/tmp/node_pkgs/$(OS_TYPE)/$(ARCH)/$(VERSION)
 
 # This is the default target, build the indexer:
 cmd/algorand-indexer/algorand-indexer:	idb/setup_postgres_sql.go types/protocols_json.go .PHONY
@@ -13,22 +14,29 @@ idb/setup_postgres_sql.go:	idb/setup_postgres.sql
 types/protocols_json.go:	types/protocols.json types/consensus.go
 	cd types && go generate
 
-mocks:	idb/dummy.go
+idb/mocks/IndexerDb.go:	idb/dummy.go
+	go get github.com/vektra/mockery/.../
 	cd idb && mockery -name=IndexerDb
 
+deploy:
+	mule/deploy.sh
+
 package: clean setup
 	misc/release.py --outdir $(PKG_DIR)
 
 setup:
 	mkdir -p $(PKG_DIR)
 
-test:	mocks
+sign:
+	mule/sign.sh
+
+test: idb/mocks/IndexerDb.go
 	go test ./...
 
+test-package:
+	mule/e2e.sh
+
 clean:
 	rm -rf $(PKG_DIR)
 
 .PHONY:
-
-###### TARGETS FOR CICD PROCESS ######
-include ./mule/Makefile.mule

README.md

@@ -3,6 +3,11 @@
 
 The Indexer is a standalone service reads committed blocks from the Algorand blockchain and maintains a database of transactions and accounts that are searchable and indexed.
 
+# Minimum Version Requirements
+
+* go 1.13
+* Postgres 11
+
 # Quickstart
 
 We prepared a docker compose file to bring up indexer and postgres preloaded with some data. From the root directory run:
@@ -12,13 +17,14 @@ We prepared a docker compose file to bring up indexer and postgres preloaded wit
 
 Once running, here are a few commands to try out:
 ```bash
-~$ curl "localhost:8980/assets?name=bogo"
-~$ curl "localhost:8980/transactions?limit=1"
-~$ curl "localhost:8980/transactions?round=10"
-~$ curl "localhost:8980/transactions?tx-type=acfg"
-~$ curl "localhost:8980/accounts?asset-id=9"
-~$ curl "localhost:8980/accounts/ZBBRQD73JH5KZ7XRED6GALJYJUXOMBBP3X2Z2XFA4LATV3MUJKKMKG7SHA?round=15"
-~$ curl "localhost:8980/assets/9/balances"
+~$ curl "localhost:8980/v2/assets?name=bogo"
+~$ curl "localhost:8980/v2/transactions?limit=1"
+~$ curl "localhost:8980/v2/transactions?round=10"
+~$ curl "localhost:8980/v2/transactions?tx-type=acfg"
+~$ curl "localhost:8980/v2/accounts?asset-id=9"
+~$ curl "localhost:8980/v2/accounts/ZBBRQD73JH5KZ7XRED6GALJYJUXOMBBP3X2Z2XFA4LATV3MUJKKMKG7SHA?round=15"
+~$ curl "localhost:8980/v2/assets/9/balances"
+~$ curl "localhost:8980/health"
 ```
 
 # Features
@@ -73,12 +79,12 @@ You should use a process manager, like systemd, to ensure the daemon is always r
 
 To start indexer as a daemon in update mode, provide the required fields:
 ```
-~$ algorand-indexer daemon --algodAddr yournode.com:1234 --algodToken token --genesis ~/path/to/genesis.json  --postgres "user=readonly password=YourPasswordHere {other connection string options for your database}"
+~$ algorand-indexer daemon --algod-net yournode.com:1234 --algod-token token --genesis ~/path/to/genesis.json  --postgres "user=readonly password=YourPasswordHere {other connection string options for your database}"
 ```
 
 Alternatively if indexer is running on the same host as the archival node, a simplified command may be used:
 ```
-~$ algorand-indexer daemon --algodAddr yournode.com:1234 -d /path/to/algod/data/dir --postgres "user=readonly password=YourPasswordHere {other connection string options for your database}"
+~$ algorand-indexer daemon --algod-net yournode.com:1234 -d /path/to/algod/data/dir --postgres "user=readonly password=YourPasswordHere {other connection string options for your database}"
 ```
 
 ### Read only
@@ -107,6 +113,7 @@ When `--token your-token` is provided, an authentication header is required. For
 
 ```
 [Service]
+ExecStart=
 ExecStart=/usr/bin/algorand-indexer daemon --pidfile /var/lib/algorand/algorand-indexer.pid --algod /var/lib/algorand --postgres "host=mydb.mycloud.com user=postgres password=password dbname=mainnet"
 PIDFile=/var/lib/algorand/algorand-indexer.pid
 

accounting/accounting.go

@@ -70,15 +70,7 @@ func (accounting *AccountingState) commitRound() error {
 	if err != nil {
 		return err
 	}
-	accounting.AlgoUpdates = nil
-	accounting.AccountTypes = nil
-	accounting.AccountDataUpdates = nil
-	accounting.AssetUpdates = nil
-	accounting.AcfgUpdates = nil
-	accounting.TxnAssetUpdates = nil
-	accounting.FreezeUpdates = nil
-	accounting.AssetCloses = nil
-	accounting.AssetDestroys = nil
+	accounting.RoundUpdates.Clear()
 	accounting.dirty = false
 	return nil
 }
@@ -203,7 +195,10 @@ func blankLsig(lsig atypes.LogicSig) bool {
 	return len(lsig.Logic) == 0
 }
 
-func (accounting *AccountingState) AddTransaction(round uint64, intra int, txnbytes []byte) (err error) {
+func (accounting *AccountingState) AddTransaction(txnr *idb.TxnRow) (err error) {
+	round := txnr.Round
+	intra := txnr.Intra
+	txnbytes := txnr.TxnBytes
 	var stxn types.SignedTxnWithAD
 	err = msgpack.Decode(txnbytes, &stxn)
 	if err != nil {
@@ -295,19 +290,14 @@ func (accounting *AccountingState) AddTransaction(round uint64, intra int, txnby
 		}
 	} else if stxn.Txn.Type == "acfg" {
 		assetId := uint64(stxn.Txn.ConfigAsset)
-		if assetId == 0 {
-			// create an asset
-			txnCounter, err := accounting.getTxnCounter(round - 1)
-			if err != nil {
-				return fmt.Errorf("acfg get TxnCounter round %d, %v", round, err)
-			}
-			assetId = txnCounter + uint64(intra) + 1
-			accounting.updateTxnAsset(round, intra, assetId)
+		isNew := assetId == 0
+		if isNew {
+			assetId = txnr.AssetId
 		}
 		if stxn.Txn.AssetParams.IsZero() {
 			accounting.destroyAsset(assetId)
 		} else {
-			accounting.AcfgUpdates = append(accounting.AcfgUpdates, idb.AcfgUpdate{AssetId: assetId, Creator: stxn.Txn.Sender, Params: stxn.Txn.AssetParams})
+			accounting.AcfgUpdates = append(accounting.AcfgUpdates, idb.AcfgUpdate{AssetId: assetId, IsNew: isNew, Creator: stxn.Txn.Sender, Params: stxn.Txn.AssetParams})
 			accounting.defaultFrozen[assetId] = stxn.Txn.AssetParams.DefaultFrozen
 			if stxn.Txn.ConfigAsset == 0 {
 				// initial creation, give all initial value to creator
@@ -333,6 +323,69 @@ func (accounting *AccountingState) AddTransaction(round uint64, intra int, txnby
 		}
 	} else if stxn.Txn.Type == "afrz" {
 		accounting.freezeAsset(stxn.Txn.FreezeAccount, uint64(stxn.Txn.FreezeAsset), stxn.Txn.AssetFrozen)
+	} else if stxn.Txn.Type == "appl" {
+		hasGlobal := (len(stxn.EvalDelta.GlobalDelta) > 0) || (len(stxn.Txn.ApprovalProgram) > 0) || (len(stxn.Txn.ClearStateProgram) > 0) || stxn.Txn.OnCompletion == atypes.DeleteApplicationOC
+		appid := uint64(stxn.Txn.ApplicationID)
+		if appid == 0 {
+			// creation
+			appid = txnr.AssetId
+		}
+		if hasGlobal {
+			agd := idb.AppDelta{
+				AppIndex: int64(appid),
+				Round:    round,
+				Intra:    intra,
+				//Address:           nil,
+				Delta:             stxn.EvalDelta.GlobalDelta,
+				OnCompletion:      stxn.Txn.OnCompletion,
+				ApprovalProgram:   stxn.Txn.ApprovalProgram,
+				ClearStateProgram: stxn.Txn.ClearStateProgram,
+				LocalStateSchema:  stxn.Txn.LocalStateSchema,
+				GlobalStateSchema: stxn.Txn.GlobalStateSchema,
+			}
+			if stxn.Txn.ApplicationID == 0 {
+				// app creation
+				agd.Creator = stxn.Txn.Sender[:]
+			}
+			//fmt.Printf("agset %d %s\n", appid, agd.String())
+			accounting.AppGlobalDeltas = append(
+				accounting.AppGlobalDeltas,
+				agd,
+			)
+		}
+		for accountIndex, ldelts := range stxn.EvalDelta.LocalDeltas {
+			var addr []byte
+			if accountIndex == 0 {
+				addr = stxn.Txn.Sender[:]
+			} else {
+				addr = stxn.Txn.Accounts[accountIndex-1][:]
+			}
+			accounting.AppLocalDeltas = append(
+				accounting.AppLocalDeltas,
+				idb.AppDelta{
+					AppIndex:     int64(appid),
+					Round:        round,
+					Intra:        intra,
+					Address:      addr,
+					AddrIndex:    accountIndex,
+					Delta:        ldelts,
+					OnCompletion: stxn.Txn.OnCompletion,
+				},
+			)
+		}
+		// if there's no other content change, but a state change of opt-in/close-out/clear-state, record that
+		if len(stxn.EvalDelta.LocalDeltas) == 0 && (stxn.Txn.OnCompletion == atypes.OptInOC || stxn.Txn.OnCompletion == atypes.CloseOutOC || stxn.Txn.OnCompletion == atypes.ClearStateOC) {
+			accounting.AppLocalDeltas = append(
+				accounting.AppLocalDeltas,
+				idb.AppDelta{
+					AppIndex:     int64(appid),
+					Address:      stxn.Txn.Sender[:],
+					Round:        round,
+					Intra:        intra,
+					OnCompletion: stxn.Txn.OnCompletion,
+				},
+			)
+		}
 	} else {
 		return fmt.Errorf("txn r=%d i=%d UNKNOWN TYPE %#v\n", round, intra, stxn.Txn.Type)
 	}

accounting/rewind.go

@@ -21,9 +21,11 @@ func assetUpdate(account *models.Account, assetid uint64, add, sub uint64) {
 			ah.Amount += add
 			ah.Amount -= sub
 			assets[i] = ah
+			// found and updated asset, done
 			return
 		}
 	}
+	// add asset to list
 	assets = append(assets, models.AssetHolding{
 		Amount:  add - sub,
 		AssetId: assetid,
@@ -71,8 +73,13 @@ func AccountAtRound(account models.Account, round uint64, db idb.IndexerDb) (acc
 				acct.AmountWithoutPendingRewards -= uint64(stxn.ReceiverRewards)
 			}
 			if addr == stxn.Txn.CloseRemainderTo {
+				// unwind receiving a close-to
 				acct.AmountWithoutPendingRewards -= uint64(stxn.ClosingAmount)
 				acct.AmountWithoutPendingRewards -= uint64(stxn.CloseRewards)
+			} else if !stxn.Txn.CloseRemainderTo.IsZero() {
+				// unwind sending a close-to
+				acct.AmountWithoutPendingRewards += uint64(stxn.ClosingAmount)
+				acct.AmountWithoutPendingRewards += uint64(stxn.CloseRewards)
 			}
 		case atypes.KeyRegistrationTx:
 			// TODO: keyreg does not rewind. workaround: query for txns on an account with typeenum=2 to find previous values it was set to.

api/Makefile

@@ -23,6 +23,6 @@ indexer.oas3.yml:	.3tmp.json
 	curl -s -X POST "https://converter.swagger.io/api/convert" -H "accept: application/json" -H "Content-Type: application/json" -d @./indexer.oas2.json -o .3tmp.json
 
 oapi-codegen:	.PHONY
-	GO111MODULE=on go get -u "github.com/algorand/oapi-codegen/...@v1.3.5-algorand4"
+	GO111MODULE=on go get -u "github.com/algorand/oapi-codegen/...@v1.3.5-algorand5"
 
 .PHONY:

api/README.md

@@ -7,7 +7,7 @@ The API is defined using [OpenAPI v2](https://swagger.io/specification/v2/) in *
 ## Updating REST API
 You need to have our fork of **oapi-codegen** installed:
 ```
-GO111MODULE=on go get -u github.com/algorand/oapi-codegen/...@v1.3.5-algorand2
+GO111MODULE=on go get -u github.com/algorand/oapi-codegen/...@v1.3.5-algorand5
 ```
 
 1. Document your changes by editing **indexer.oas2.yml**