From 30a7218cc8fba2336382908ddaaca544f2404761 Mon Sep 17 00:00:00 2001 From: Erik Krogen Date: Thu, 25 Jul 2019 16:08:52 -0700 Subject: [PATCH] Closes #99. Document supported versions in the README, and improve the support for the 2.8 line as a host cluster. --- .travis.yml | 4 +-- README.md | 15 ++++++++ build.gradle | 2 +- .../linkedin/dynamometer/DynoInfraUtils.java | 35 ++++++++++++++++++- .../SimulatedMultiStorageFSDataset.java | 13 +++++-- 5 files changed, 62 insertions(+), 7 deletions(-) diff --git a/.travis.yml b/.travis.yml index 492106cb48..51873ee71d 100644 --- a/.travis.yml +++ b/.travis.yml @@ -12,10 +12,10 @@ jobs: script: "./gradlew build -x test" - stage: test script: "./gradlew test --stacktrace --info -Ddyno.hadoop.bin.version=${HADOOP_VERSION}" - env: HADOOP_VERSION=2.7.6 + env: HADOOP_VERSION=2.7.7 - stage: test script: "./gradlew test --stacktrace --info -Ddyno.hadoop.bin.version=${HADOOP_VERSION}" - env: HADOOP_VERSION=2.8.4 + env: HADOOP_VERSION=2.8.5 - stage: deploy script: "./gradlew build -s && ./gradlew ciPerformRelease" diff --git a/README.md b/README.md index 77d398acfd..245c8e6029 100644 --- a/README.md +++ b/README.md @@ -1,5 +1,11 @@ # Dynamometer [![Build Status](https://travis-ci.org/linkedin/dynamometer.svg?branch=master)](https://travis-ci.org/linkedin/dynamometer) +## Dynamometer in Hadoop + +Please be aware that Dynamometer has now been committed into Hadoop itself in the JIRA ticket [HDFS-12345](https://issues.apache.org/jira/browse/HDFS-12345). It is located under the `hadoop-tools/hadoop-dynamometer` submodule. This GitHub project will continue to be maintained for testing +against the `2.x` release line of Hadoop, but all versions of Dynamometer which work with Hadoop 3 will +only appear in Hadoop, and future development will primarily occur there. + ## Overview Dynamometer is a tool to performance test Hadoop's HDFS NameNode. The intent is to provide a @@ -26,6 +32,15 @@ top of which Dynamometer is run. Dynamometer is based around YARN applications, so an existing YARN cluster will be required for execution. It also requires an accompanying HDFS instance to store some temporary files for communication. +Please be aware that Dynamometer makes certain assumptions about HDFS, and thus only works with certain +versions. As discussed at the start of this README, this project only works with Hadoop 2; support for +Hadoop 3 is introduced in the version of Dynamometer within the Hadoop repository. Below is a list of known +supported versions of Hadoop which are compatible with Dynamometer: +* Hadoop 2.7 starting at 2.7.4 +* Hadoop 2.8 starting at 2.8.4 + +Hadoop 2.8.2 and 2.8.3 are compatible as a cluster version on which to run Dynamometer, but are not supported as a version-under-test. + ## Building Dynamometer consists of three main components: diff --git a/build.gradle b/build.gradle index 8497c46c19..ec91253ec7 100644 --- a/build.gradle +++ b/build.gradle @@ -18,7 +18,7 @@ plugins { id "org.shipkit.java" version "2.1.3" } -def hadoopVersion = '2.7.5' +def hadoopVersion = '2.7.7' ext.deps = [ hadoop: [ hdfs: "org.apache.hadoop:hadoop-hdfs:${hadoopVersion}", diff --git a/dynamometer-infra/src/main/java/com/linkedin/dynamometer/DynoInfraUtils.java b/dynamometer-infra/src/main/java/com/linkedin/dynamometer/DynoInfraUtils.java index e473996fe2..42c3db94a4 100644 --- a/dynamometer-infra/src/main/java/com/linkedin/dynamometer/DynoInfraUtils.java +++ b/dynamometer-infra/src/main/java/com/linkedin/dynamometer/DynoInfraUtils.java @@ -11,6 +11,8 @@ import java.io.FileNotFoundException; import java.io.IOException; import java.io.InputStream; +import java.lang.reflect.InvocationTargetException; +import java.lang.reflect.Method; import java.net.HttpURLConnection; import java.net.InetSocketAddress; import java.net.MalformedURLException; @@ -22,6 +24,7 @@ import java.util.Set; import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicBoolean; +import javax.net.SocketFactory; import org.apache.commons.io.FileUtils; import org.apache.commons.io.IOUtils; import org.apache.commons.logging.Log; @@ -317,13 +320,43 @@ public void run() { private static void triggerDataNodeBlockReport(Configuration conf, String dataNodeTarget) throws IOException { InetSocketAddress datanodeAddr = NetUtils.createSocketAddr(dataNodeTarget); - ClientDatanodeProtocol dnProtocol = DFSUtil.createClientDatanodeProtocolProxy( + ClientDatanodeProtocol dnProtocol = createClientDatanodeProtocolProxy( datanodeAddr, UserGroupInformation.getCurrentUser(), conf, NetUtils.getSocketFactory(conf, ClientDatanodeProtocol.class)); dnProtocol.triggerBlockReport(new BlockReportOptions.Factory().build()); } + /** + * Between Hadoop 2.7 and 2.8, createClientDatanodeProtocolProxy moved from DFSUtil to DFSUtilClient. + * This provides a shim which will find the method in either class and invoke it. + */ + private static ClientDatanodeProtocol createClientDatanodeProtocolProxy(InetSocketAddress datanodeAddr, + UserGroupInformation ugi, Configuration conf, SocketFactory socketFactory) { + final String methodName = "createClientDatanodeProtocolProxy"; + Method createProxyMethod; + try { + // In versions 2.7 and below, it is located within DFSUtil ... + createProxyMethod = DFSUtil.class.getMethod(methodName, InetSocketAddress.class, UserGroupInformation.class, + Configuration.class, SocketFactory.class); + } catch (NoSuchMethodException nsme) { + // ... but in later versions, it is located within DFSUtilClient + try { + // DFSUtilClient doesn't yet exist in Hadoop 2.7 so it has to be loaded via reflection + Class dfsUtilClient = Class.forName(DFSUtil.class.getName() + "Client"); + createProxyMethod = dfsUtilClient.getMethod(methodName, InetSocketAddress.class, + UserGroupInformation.class, Configuration.class, SocketFactory.class); + } catch (ClassNotFoundException | NoSuchMethodException e) { + throw new RuntimeException("Unable to load " + methodName + "; looked in DFSUtil and DFSUtilClient", e); + } + } + try { + return (ClientDatanodeProtocol) createProxyMethod.invoke(null, datanodeAddr, ugi, conf, socketFactory); + } catch (IllegalAccessException | InvocationTargetException e) { + throw new RuntimeException("Unable to call " + methodName, e); + } + } + /** * Poll the launched NameNode's JMX for a specific value, waiting for it to cross some threshold. Continues until * the threshold has been crossed or {@code shouldExit} returns true. Periodically logs the current value. diff --git a/dynamometer-infra/src/main/java/com/linkedin/dynamometer/SimulatedMultiStorageFSDataset.java b/dynamometer-infra/src/main/java/com/linkedin/dynamometer/SimulatedMultiStorageFSDataset.java index 54cc38bf86..3a87bf7958 100644 --- a/dynamometer-infra/src/main/java/com/linkedin/dynamometer/SimulatedMultiStorageFSDataset.java +++ b/dynamometer-infra/src/main/java/com/linkedin/dynamometer/SimulatedMultiStorageFSDataset.java @@ -325,6 +325,10 @@ public ChunkChecksum getLastChecksumAndDataLen() { public boolean isOnTransientStorage() { return false; } + + public OutputStream createRestartMetaStream() throws IOException { + return null; + } } /** @@ -509,7 +513,6 @@ public boolean isTransientStorage() { return false; } - @Override public void reserveSpaceForRbw(long bytesToReserve) { } @@ -538,6 +541,12 @@ public byte[] loadLastPartialChunkChecksum( File blockFile, File metaFile) throws IOException { return null; } + + public void reserveSpaceForReplica(long l) { + } + + public void releaseLockedMemory(long l) { + } } private final List storages; @@ -1308,7 +1317,6 @@ public void checkAndUpdate(String bpid, long blockId, File diskFile, throw new UnsupportedOperationException(); } - @Override public List getVolumes() { throw new UnsupportedOperationException(); } @@ -1344,7 +1352,6 @@ public List getFinalizedBlocks(String bpid) { throw new UnsupportedOperationException(); } - @Override public List getFinalizedBlocksOnPersistentStorage(String bpid) { throw new UnsupportedOperationException(); }