diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 000000000..80fbb03d3 --- /dev/null +++ b/Dockerfile @@ -0,0 +1,36 @@ +FROM gcr.io/whiteblock/base:ubuntu1804 + +RUN apt-get update +RUN apt-get install -y openjdk-8-jdk + +# Harmony implementation depends on JVM libp2p implementation, which is not in a public repository yet. +# So, one should build it manually and push to a local maven repository, to be able to build the node command. +RUN git clone https://github.com/libp2p/jvm-libp2p --branch feature/cleanup +WORKDIR /jvm-libp2p +RUN ./gradlew build -x test +RUN ./gradlew publishToMavenLocal +WORKDIR / + +# Cloning Harmony +RUN git clone https://github.com/harmony-dev/beacon-chain-java.git +WORKDIR /beacon-chain-java +# TODO: switch to develop when merged +RUN git checkout interop +WORKDIR / + +# Building Harmony +WORKDIR /beacon-chain-java +RUN ./gradlew build -x test +WORKDIR start/node/build/distributions/ +RUN tar -xf node*.tar +RUN ln -s /beacon-chain-java/start/node/build/distributions/node-*/bin/node /usr/bin/harmony +WORKDIR / + +# Copying start script +RUN mkdir /launch +RUN cp /beacon-chain-java/scripts/whiteblock_start.sh /launch/start.sh +RUN chmod +x /launch/start.sh + +EXPOSE 8545 8546 9000 30303 30303/udp + +ENTRYPOINT ["/bin/bash"] diff --git a/scripts/whiteblock_start.sh b/scripts/whiteblock_start.sh new file mode 100755 index 000000000..0c49672e8 --- /dev/null +++ b/scripts/whiteblock_start.sh @@ -0,0 +1,87 @@ +#!/bin/bash + +<" + echo "--peers=" + echo "--validator-keys=" + echo "--gen-state=" + echo "--port=" +} + +CMD_LINE_ADDON="" +while [ "$1" != "" ]; +do + PARAM=`echo $1 | awk -F= '{print $1}'` + VALUE=`echo $1 | sed 's/^[^=]*=//g'` + + case $PARAM in + --identity) + IDENTITY=$VALUE + CMD_LINE_ADDON=$CMD_LINE_ADDON"--node-key=$IDENTITY " + ;; + --peers) + [ ! -z "$PEERS" ] && PEERS+="," + PEERS+="$VALUE" + CMD_LINE_ADDON=$CMD_LINE_ADDON"--connect=$PEERS " + ;; + --validator-keys) + VALIDATOR_KEYS=$VALUE + CMD_LINE_ADDON=$CMD_LINE_ADDON"--validators-file=$VALIDATOR_KEYS " + ;; + --gen-state) + GEN_STATE=$VALUE + CMD_LINE_ADDON=$CMD_LINE_ADDON"--initial-state=$GEN_STATE " + ;; + --port) + PORT=$VALUE + CMD_LINE_ADDON=$CMD_LINE_ADDON"--listen=$PORT " + ;; + --help) + usage + exit + ;; + *) + echo "ERROR: unknown parameter \"$PARAM\"" + usage + exit 1 + ;; + esac + shift +done + +/usr/bin/harmony default $CMD_LINE_ADDON + +trap 'trap - SIGTERM && kill 0' SIGINT SIGTERM EXIT + diff --git a/start/config/src/main/java/org/ethereum/beacon/emulator/config/node/KeyData.java b/start/config/src/main/java/org/ethereum/beacon/emulator/config/node/KeyData.java new file mode 100644 index 000000000..e5d19da15 --- /dev/null +++ b/start/config/src/main/java/org/ethereum/beacon/emulator/config/node/KeyData.java @@ -0,0 +1,25 @@ +package org.ethereum.beacon.emulator.config.node; + +import com.fasterxml.jackson.annotation.JsonProperty; + +public class KeyData { + private String privkey; + @JsonProperty + private String pubkey; + + public String getPrivkey() { + return privkey; + } + + public void setPrivkey(String privkey) { + this.privkey = privkey; + } + + public String getPubkey() { + return pubkey; + } + + public void setPubkey(String pubkey) { + this.pubkey = pubkey; + } +} diff --git a/start/config/src/main/java/org/ethereum/beacon/emulator/config/node/KeyDataReader.java b/start/config/src/main/java/org/ethereum/beacon/emulator/config/node/KeyDataReader.java new file mode 100644 index 000000000..5e5e7ea0d --- /dev/null +++ b/start/config/src/main/java/org/ethereum/beacon/emulator/config/node/KeyDataReader.java @@ -0,0 +1,28 @@ +package org.ethereum.beacon.emulator.config.node; + +import com.fasterxml.jackson.core.type.TypeReference; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.dataformat.yaml.YAMLFactory; + +import java.io.File; +import java.io.IOException; +import java.util.List; + +/** Reads YAML file with keys {@link KeyData}*/ +public class KeyDataReader { + private final File file; + private ObjectMapper mapper; + + public KeyDataReader(File file) { + this.file = file; + this.mapper = new ObjectMapper(new YAMLFactory()); + } + + public List readKeys() { + try { + return mapper.readValue(file, new TypeReference>(){}); + } catch (IOException e) { + throw new RuntimeException(String.format("Error thrown when reading file %s", file), e); + } + } +} diff --git a/start/node/src/main/java/org/ethereum/beacon/node/Node.java b/start/node/src/main/java/org/ethereum/beacon/node/Node.java index 777d87b87..5e6c6f0cf 100644 --- a/start/node/src/main/java/org/ethereum/beacon/node/Node.java +++ b/start/node/src/main/java/org/ethereum/beacon/node/Node.java @@ -46,6 +46,13 @@ public class Node implements Runnable { ) private Integer listenPort; + @CommandLine.Option( + names = {"--node-key"}, + paramLabel = "key", + description = "Private key of p2p node (32 bytes)" + ) + private String nodeKey; + @CommandLine.Option( names = {"--connect"}, paramLabel = "URL", @@ -71,6 +78,17 @@ public class Node implements Runnable { ) private List validators; + @CommandLine.Option( + names = {"--validators-file"}, + paramLabel = "key", + split = ",", + description = { + "Validator registry as yaml file. Entries are pairs of privkey and pubkey", + "Example: --validators-file=keygen_10_validators.yaml" + } + ) + private String validatorsFile; + @CommandLine.Option( names = {"--name"}, paramLabel = "node-name", @@ -184,6 +202,10 @@ public Integer getListenPort() { return listenPort; } + public String getNodeKey() { + return nodeKey; + } + public List getActivePeers() { return activePeers; } @@ -192,6 +214,10 @@ public List getValidators() { return validators; } + public String getValidatorsFile() { + return validatorsFile; + } + public String getGenesisTime() { return genesisTime; } diff --git a/start/node/src/main/java/org/ethereum/beacon/node/NodeCommandLauncher.java b/start/node/src/main/java/org/ethereum/beacon/node/NodeCommandLauncher.java index 4806d8c55..9083d0fab 100644 --- a/start/node/src/main/java/org/ethereum/beacon/node/NodeCommandLauncher.java +++ b/start/node/src/main/java/org/ethereum/beacon/node/NodeCommandLauncher.java @@ -1,20 +1,5 @@ package org.ethereum.beacon.node; -import java.io.File; -import java.io.IOException; -import java.lang.reflect.InvocationTargetException; -import java.nio.file.Files; -import java.nio.file.Paths; -import java.text.ParseException; -import java.text.SimpleDateFormat; -import java.util.ArrayList; -import java.util.Collections; -import java.util.Date; -import java.util.List; -import java.util.Random; -import java.util.TimeZone; -import java.util.concurrent.ThreadFactory; -import java.util.stream.IntStream; import org.apache.logging.log4j.Level; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; @@ -53,6 +38,8 @@ import org.ethereum.beacon.emulator.config.main.network.Libp2pNetwork; import org.ethereum.beacon.emulator.config.main.network.NettyNetwork; import org.ethereum.beacon.emulator.config.main.network.Network; +import org.ethereum.beacon.emulator.config.node.KeyData; +import org.ethereum.beacon.emulator.config.node.KeyDataReader; import org.ethereum.beacon.node.metrics.Metrics; import org.ethereum.beacon.pow.DepositContract; import org.ethereum.beacon.schedulers.DefaultSchedulers; @@ -69,10 +56,27 @@ import tech.pegasys.artemis.ethereum.core.Hash32; import tech.pegasys.artemis.util.bytes.BytesValue; +import java.io.File; +import java.io.IOException; +import java.lang.reflect.InvocationTargetException; +import java.nio.file.Files; +import java.nio.file.Paths; +import java.text.ParseException; +import java.text.SimpleDateFormat; +import java.util.ArrayList; +import java.util.Collections; +import java.util.Date; +import java.util.List; +import java.util.Random; +import java.util.TimeZone; +import java.util.concurrent.ThreadFactory; +import java.util.stream.Collectors; +import java.util.stream.IntStream; + public class NodeCommandLauncher implements Runnable { private static final Logger logger = LogManager.getLogger("node"); - private final static long DB_BUFFER_SIZE = 64L << 20; // 64Mb + private static final long DB_BUFFER_SIZE = 64L << 20; // 64Mb private final MainConfig config; private final SpecConstants specConstants; @@ -95,10 +99,7 @@ public class NodeCommandLauncher implements Runnable { * @param logLevel Log level, Apache log4j type. */ public NodeCommandLauncher( - MainConfig config, - SpecBuilder specBuilder, - Node cliOptions, - Level logLevel) { + MainConfig config, SpecBuilder specBuilder, Node cliOptions, Level logLevel) { this.config = config; this.specBuilder = specBuilder; this.specConstants = specBuilder.buildSpecConstants(); @@ -112,8 +113,7 @@ public NodeCommandLauncher( private void setupLogging() { // set logLevel if (logLevel != null) { - LoggerContext context = - (LoggerContext) LogManager.getContext(false); + LoggerContext context = (LoggerContext) LogManager.getContext(false); Configuration config = context.getConfiguration(); LoggerConfig loggerConfig = config.getLoggerConfig("node"); loggerConfig.setLevel(logLevel); @@ -139,7 +139,8 @@ public void run() { @Override protected ThreadFactory createThreadFactory(String namePattern) { ThreadFactory factory = - createThreadFactoryBuilder((nodeName == null ? "" : nodeName + "-") + namePattern).build(); + createThreadFactoryBuilder((nodeName == null ? "" : nodeName + "-") + namePattern) + .build(); if (nodeName == null) { return factory; } else { @@ -158,10 +159,11 @@ protected ThreadFactory createThreadFactory(String namePattern) { BeaconStateEx initialState; SerializerFactory serializerFactory = SerializerFactory.createSSZ(specConstants); if (initialStateFile == null) { - chainStart = ConfigUtils.createChainStart( - config.getConfig().getValidator().getContract(), - spec, - config.getChainSpec().getSpecHelpersOptions().isBlsVerifyProofOfPossession()); + chainStart = + ConfigUtils.createChainStart( + config.getConfig().getValidator().getContract(), + spec, + config.getChainSpec().getSpecHelpersOptions().isBlsVerifyProofOfPossession()); initialState = new InitialStateTransition(chainStart, spec).apply(spec.get_empty_block()); } else { byte[] fileData; @@ -172,7 +174,9 @@ protected ThreadFactory createThreadFactory(String namePattern) { } initialState = new BeaconStateExImpl( - serializerFactory.getDeserializer(BeaconStateImpl.class).apply(BytesValue.wrap(fileData)), + serializerFactory + .getDeserializer(BeaconStateImpl.class) + .apply(BytesValue.wrap(fileData)), TransitionType.INITIAL); chainStart = new ChainStart( @@ -180,9 +184,10 @@ protected ThreadFactory createThreadFactory(String namePattern) { } DepositContract depositContract = new SimpleDepositContract(chainStart, schedulers); - List credentials = ConfigUtils.createCredentials( - config.getConfig().getValidator().getSigner(), - config.getChainSpec().getSpecHelpersOptions().isBlsSign()); + List credentials = + ConfigUtils.createCredentials( + config.getConfig().getValidator().getSigner(), + config.getChainSpec().getSpecHelpersOptions().isBlsSign()); if (config.getConfig().getNetworks().size() != 1) { throw new IllegalArgumentException("1 network should be specified in config"); @@ -260,8 +265,7 @@ protected ThreadFactory createThreadFactory(String namePattern) { Metrics.startMetricsServer(metricsHost, metricsPort); SSZBeaconChainStorageFactory storageFactory = - new SSZBeaconChainStorageFactory( - spec.getObjectHasher(), serializerFactory); + new SSZBeaconChainStorageFactory(spec.getObjectHasher(), serializerFactory); String dbPrefix = config.getConfig().getDb(); String startMode; @@ -305,8 +309,9 @@ protected ThreadFactory createThreadFactory(String namePattern) { break; case "initial": if (!emptyStorage && !forceDBClean) { - throw new IllegalArgumentException("Cannot initialize non-empty storage." - + " Use --force-db-clean to clean automatically"); + throw new IllegalArgumentException( + "Cannot initialize non-empty storage." + + " Use --force-db-clean to clean automatically"); } doInitialize = true; break; @@ -316,7 +321,7 @@ protected ThreadFactory createThreadFactory(String namePattern) { if (doInitialize && !emptyStorage && forceDBClean) { db.close(); - try{ + try { dbFactory.removeDatabase(genesisTime, depositRoot); } catch (RuntimeException e) { throw new IllegalStateException("Cannot clean DB, remove files manually", e); @@ -329,15 +334,16 @@ protected ThreadFactory createThreadFactory(String namePattern) { StorageUtils.initializeStorage(beaconChainStorage, spec, initialState); } - NodeLauncher node = new NodeLauncher( - specBuilder.buildSpec(), - depositContract, - credentials, - libp2pLauncher, //connectionManager, - db, - beaconChainStorage, - schedulers, - true); + NodeLauncher node = + new NodeLauncher( + specBuilder.buildSpec(), + depositContract, + credentials, + libp2pLauncher, // connectionManager, + db, + beaconChainStorage, + schedulers, + true); if (cliOptions.isDumpTuples()) { BeaconTupleDetailsDumper dumper = @@ -385,11 +391,38 @@ public static class Builder { public Builder() {} + @NotNull + private static SpecConstantsData mergeSpecConstantsData( + SpecConstantsData specConsts, SpecConstantsDataMerged specConstsYaml) { + if (specConsts == null) { + return specConstsYaml; + } else { + try { + return Objects.copyProperties(specConsts, specConstsYaml); + } catch (IllegalAccessException | InvocationTargetException e) { + throw new RuntimeException( + String.format("Failed to merge config %s into main config", specConsts), e); + } + } + } + + private static SpecConstantsDataMerged loadSpecConstantsDataMerged(String specConstants) { + ConfigBuilder specConstsBuilder = + new ConfigBuilder<>(SpecConstantsDataMerged.class); + if ("minimal".equals(specConstants)) { + specConstsBuilder.addYamlConfigFromResources("/spec/" + specConstants + ".yaml"); + } else { + specConstsBuilder.addYamlConfig(Paths.get(specConstants).toFile()); + } + return specConstsBuilder.build(); + } + public NodeCommandLauncher build() { assert config != null; ConfigBuilder specConfigBuilder = - new ConfigBuilder<>(SpecData.class).addYamlConfigFromResources("/config/spec-constants.yml"); + new ConfigBuilder<>(SpecData.class) + .addYamlConfigFromResources("/config/spec-constants.yml"); if (cliOptions != null && cliOptions.getSpecConstantsFile() != null @@ -416,7 +449,8 @@ public NodeCommandLauncher build() { if (cliOptions.getInitialDepositCount() != null) { if (config.getConfig().getValidator().getContract() instanceof EmulatorContract) { - EmulatorContract contract = (EmulatorContract) config.getConfig().getValidator().getContract(); + EmulatorContract contract = + (EmulatorContract) config.getConfig().getValidator().getContract(); ValidatorKeys.InteropKeys keys = new ValidatorKeys.InteropKeys(); keys.setCount(cliOptions.getInitialDepositCount()); contract.setKeys(Collections.singletonList(keys)); @@ -424,12 +458,12 @@ public NodeCommandLauncher build() { } if (cliOptions.getListenPort() != null || cliOptions.getActivePeers() != null) { - Libp2pNetwork network = (Libp2pNetwork) config - .getConfig() - .getNetworks() - .stream() - .filter(n -> n instanceof Libp2pNetwork) - .findFirst().orElse(null); + Libp2pNetwork network = + (Libp2pNetwork) + config.getConfig().getNetworks().stream() + .filter(n -> n instanceof Libp2pNetwork) + .findFirst() + .orElse(null); if (network == null) { network = new Libp2pNetwork(); @@ -445,6 +479,26 @@ public NodeCommandLauncher build() { } } + if (cliOptions.getNodeKey() != null) { + config.getConfig().getNetworks().stream() + .filter(n -> n instanceof Libp2pNetwork) + .forEach(n -> ((Libp2pNetwork) n).setPrivateKey(cliOptions.getNodeKey())); + } + + if (cliOptions.getValidatorsFile() != null) { + if (cliOptions.getValidators() != null) { + throw new RuntimeException("Only one validators option could be used at time!"); + } + List keysData = + new KeyDataReader(new File(cliOptions.getValidatorsFile())).readKeys(); + Insecure signer = new Insecure(); + signer.setKeys( + Collections.singletonList( + new Private( + keysData.stream().map(KeyData::getPrivkey).collect(Collectors.toList())))); + config.getConfig().getValidator().setSigner(signer); + } + if (cliOptions.getValidators() != null) { List validatorKeys = new ArrayList<>(); List depositKeypairs = null; @@ -453,8 +507,10 @@ public NodeCommandLauncher build() { validatorKeys.add(key); } else { if (depositKeypairs == null) { - depositKeypairs = ConfigUtils - .createKeyPairs(((EmulatorContract)config.getConfig().getValidator().getContract()).getKeys()); + depositKeypairs = + ConfigUtils.createKeyPairs( + ((EmulatorContract) config.getConfig().getValidator().getContract()) + .getKeys()); } List finalDepositKeypairs = depositKeypairs; @@ -468,7 +524,8 @@ public NodeCommandLauncher build() { indices = IntStream.of(Integer.parseInt(key)); } indices - .mapToObj(i -> finalDepositKeypairs.get(i).getPrivate().getEncodedBytes().toString()) + .mapToObj( + i -> finalDepositKeypairs.get(i).getPrivate().getEncodedBytes().toString()) .forEach(validatorKeys::add); } } @@ -478,9 +535,10 @@ public NodeCommandLauncher build() { } if (cliOptions.getGenesisTime() != null) { - SimpleDateFormat[] supportedFormats = new SimpleDateFormat[] { - new SimpleDateFormat("yyyy-MM-dd HH:mm"), - new SimpleDateFormat("HH:mm")}; + SimpleDateFormat[] supportedFormats = + new SimpleDateFormat[] { + new SimpleDateFormat("yyyy-MM-dd HH:mm"), new SimpleDateFormat("HH:mm") + }; Date time = null; for (SimpleDateFormat format : supportedFormats) { @@ -518,17 +576,18 @@ public NodeCommandLauncher build() { if (!(config.getConfig().getValidator().getContract() instanceof EmulatorContract)) { throw new ConfigException("Genesis time can only be set for 'emulator' contract type"); } - EmulatorContract contract = (EmulatorContract) config.getConfig().getValidator().getContract(); + EmulatorContract contract = + (EmulatorContract) config.getConfig().getValidator().getContract(); contract.setGenesisTime(time); SimpleDateFormat format = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss"); format.setTimeZone(TimeZone.getTimeZone("GMT")); - logger.info("Genesis time from cli option: " - + format.format(time) + " GMT"); + logger.info("Genesis time from cli option: " + format.format(time) + " GMT"); } if (config.getConfig().getValidator().getContract() instanceof EmulatorContract) { - EmulatorContract contract = (EmulatorContract) config.getConfig().getValidator().getContract(); + EmulatorContract contract = + (EmulatorContract) config.getConfig().getValidator().getContract(); if (contract.getGenesisTime() == null) { Date defaultTime = new Date(); defaultTime.setMinutes(0); @@ -538,8 +597,10 @@ public NodeCommandLauncher build() { SimpleDateFormat format = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss"); format.setTimeZone(TimeZone.getTimeZone("GMT")); - logger.warn("Genesis time not specified. Default genesisTime was generated: " - + format.format(defaultTime) + " GMT"); + logger.warn( + "Genesis time not specified. Default genesisTime was generated: " + + format.format(defaultTime) + + " GMT"); } } @@ -551,36 +612,7 @@ public NodeCommandLauncher build() { config.getConfig().setDb(cliOptions.getDbPrefix()); } - return new NodeCommandLauncher( - config, - specBuilder, - cliOptions, - logLevel); - } - - @NotNull - private static SpecConstantsData mergeSpecConstantsData(SpecConstantsData specConsts, SpecConstantsDataMerged specConstsYaml) { - if (specConsts == null) { - return specConstsYaml; - } else { - try { - return Objects.copyProperties(specConsts, specConstsYaml); - } catch (IllegalAccessException| InvocationTargetException e) { - throw new RuntimeException( - String.format("Failed to merge config %s into main config", specConsts), e); - } - } - } - - private static SpecConstantsDataMerged loadSpecConstantsDataMerged(String specConstants) { - ConfigBuilder specConstsBuilder = - new ConfigBuilder<>(SpecConstantsDataMerged.class); - if ("minimal".equals(specConstants)) { - specConstsBuilder.addYamlConfigFromResources("/spec/" + specConstants + ".yaml"); - } else { - specConstsBuilder.addYamlConfig(Paths.get(specConstants).toFile()); - } - return specConstsBuilder.build(); + return new NodeCommandLauncher(config, specBuilder, cliOptions, logLevel); } public Builder withConfigFromFile(File file) { @@ -590,9 +622,7 @@ public Builder withConfigFromFile(File file) { public Builder withConfigFromResource(String resourceName) { this.config = - new ConfigBuilder<>(MainConfig.class) - .addYamlConfigFromResources(resourceName) - .build(); + new ConfigBuilder<>(MainConfig.class).addYamlConfigFromResources(resourceName).build(); return this; }