Closes #99. Document supported versions and improve compatibility with 2.8

.travis.yml

@@ -12,10 +12,10 @@ jobs:
       script: "./gradlew build -x test"
     - stage: test
       script: "./gradlew test --stacktrace --info -Ddyno.hadoop.bin.version=${HADOOP_VERSION}"
-      env: HADOOP_VERSION=2.7.6
+      env: HADOOP_VERSION=2.7.7
     - stage: test
       script: "./gradlew test --stacktrace --info -Ddyno.hadoop.bin.version=${HADOOP_VERSION}"
-      env: HADOOP_VERSION=2.8.4
+      env: HADOOP_VERSION=2.8.5
     - stage: deploy
       script: "./gradlew build -s && ./gradlew ciPerformRelease"
 

README.md

@@ -1,5 +1,11 @@
 # Dynamometer [![Build Status](https://travis-ci.org/linkedin/dynamometer.svg?branch=master)](https://travis-ci.org/linkedin/dynamometer)
 
+## Dynamometer in Hadoop
+
+Please be aware that Dynamometer has now been committed into Hadoop itself in the JIRA ticket [HDFS-12345](https://issues.apache.org/jira/browse/HDFS-12345). It is located under the `hadoop-tools/hadoop-dynamometer` submodule. This GitHub project will continue to be maintained for testing
+against the `2.x` release line of Hadoop, but all versions of Dynamometer which work with Hadoop 3 will
+only appear in Hadoop, and future development will primarily occur there.
+
 ## Overview
 
 Dynamometer is a tool to performance test Hadoop's HDFS NameNode. The intent is to provide a
@@ -26,6 +32,15 @@ top of which Dynamometer is run.
 Dynamometer is based around YARN applications, so an existing YARN cluster will be required for execution.
 It also requires an accompanying HDFS instance to store some temporary files for communication.
 
+Please be aware that Dynamometer makes certain assumptions about HDFS, and thus only works with certain
+versions. As discussed at the start of this README, this project only works with Hadoop 2; support for
+Hadoop 3 is introduced in the version of Dynamometer within the Hadoop repository. Below is a list of known
+supported versions of Hadoop which are compatible with Dynamometer:
+* Hadoop 2.7 starting at 2.7.4
+* Hadoop 2.8 starting at 2.8.4
+
+Hadoop 2.8.2 and 2.8.3 are compatible as a cluster version on which to run Dynamometer, but are not supported as a version-under-test.
+
 ## Building
 
 Dynamometer consists of three main components:

build.gradle

@@ -18,7 +18,7 @@ plugins {
   id "org.shipkit.java" version "2.1.3"
 }
 
-def hadoopVersion = '2.7.5'
+def hadoopVersion = '2.7.7'
 ext.deps = [
   hadoop: [
     hdfs: "org.apache.hadoop:hadoop-hdfs:${hadoopVersion}",

dynamometer-infra/src/main/java/com/linkedin/dynamometer/DynoInfraUtils.java

@@ -11,6 +11,8 @@
 import java.io.FileNotFoundException;
 import java.io.IOException;
 import java.io.InputStream;
+import java.lang.reflect.InvocationTargetException;
+import java.lang.reflect.Method;
 import java.net.HttpURLConnection;
 import java.net.InetSocketAddress;
 import java.net.MalformedURLException;
@@ -22,6 +24,7 @@
 import java.util.Set;
 import java.util.concurrent.TimeUnit;
 import java.util.concurrent.atomic.AtomicBoolean;
+import javax.net.SocketFactory;
 import org.apache.commons.io.FileUtils;
 import org.apache.commons.io.IOUtils;
 import org.apache.commons.logging.Log;
@@ -317,13 +320,43 @@ public void run() {
   private static void triggerDataNodeBlockReport(Configuration conf, String dataNodeTarget) throws IOException {
     InetSocketAddress datanodeAddr = NetUtils.createSocketAddr(dataNodeTarget);
 
-    ClientDatanodeProtocol dnProtocol = DFSUtil.createClientDatanodeProtocolProxy(
+    ClientDatanodeProtocol dnProtocol = createClientDatanodeProtocolProxy(
         datanodeAddr, UserGroupInformation.getCurrentUser(), conf,
         NetUtils.getSocketFactory(conf, ClientDatanodeProtocol.class));
 
     dnProtocol.triggerBlockReport(new BlockReportOptions.Factory().build());
   }
 
+  /**
+   * Between Hadoop 2.7 and 2.8, createClientDatanodeProtocolProxy moved from DFSUtil to DFSUtilClient.
+   * This provides a shim which will find the method in either class and invoke it.
+   */
+  private static ClientDatanodeProtocol createClientDatanodeProtocolProxy(InetSocketAddress datanodeAddr,
+      UserGroupInformation ugi, Configuration conf, SocketFactory socketFactory) {
+    final String methodName = "createClientDatanodeProtocolProxy";
+    Method createProxyMethod;
+    try {
+      // In versions 2.7 and below, it is located within DFSUtil ...
+      createProxyMethod = DFSUtil.class.getMethod(methodName, InetSocketAddress.class, UserGroupInformation.class,
+          Configuration.class, SocketFactory.class);
+    } catch (NoSuchMethodException nsme) {
+      // ... but in later versions, it is located within DFSUtilClient
+      try {
+        // DFSUtilClient doesn't yet exist in Hadoop 2.7 so it has to be loaded via reflection
+        Class<?> dfsUtilClient = Class.forName(DFSUtil.class.getName() + "Client");
+        createProxyMethod = dfsUtilClient.getMethod(methodName, InetSocketAddress.class,
+            UserGroupInformation.class, Configuration.class, SocketFactory.class);
+      } catch (ClassNotFoundException | NoSuchMethodException e) {
+        throw new RuntimeException("Unable to load " + methodName + "; looked in DFSUtil and DFSUtilClient", e);
+      }
+    }
+    try {
+      return (ClientDatanodeProtocol) createProxyMethod.invoke(null, datanodeAddr, ugi, conf, socketFactory);
+    } catch (IllegalAccessException | InvocationTargetException e) {
+      throw new RuntimeException("Unable to call " + methodName, e);
+    }
+  }
+
   /**
    * Poll the launched NameNode's JMX for a specific value, waiting for it to cross some threshold. Continues until
    * the threshold has been crossed or {@code shouldExit} returns true. Periodically logs the current value.

dynamometer-infra/src/main/java/com/linkedin/dynamometer/SimulatedMultiStorageFSDataset.java

@@ -325,6 +325,10 @@ public ChunkChecksum getLastChecksumAndDataLen() {
     public boolean isOnTransientStorage() {
       return false;
     }
+
+    public OutputStream createRestartMetaStream() throws IOException {
+      return null;
+    }
   }
 
   /**
@@ -509,7 +513,6 @@ public boolean isTransientStorage() {
       return false;
     }
 
-    @Override
     public void reserveSpaceForRbw(long bytesToReserve) {
     }
 
@@ -538,6 +541,12 @@ public byte[] loadLastPartialChunkChecksum(
         File blockFile, File metaFile) throws IOException {
       return null;
     }
+
+    public void reserveSpaceForReplica(long l) {
+    }
+
+    public void releaseLockedMemory(long l) {
+    }
   }
 
   private final List<SimulatedStorage> storages;
@@ -1308,7 +1317,6 @@ public void checkAndUpdate(String bpid, long blockId, File diskFile,
     throw new UnsupportedOperationException();
   }
 
-  @Override
   public List<FsVolumeSpi> getVolumes() {
     throw new UnsupportedOperationException();
   }
@@ -1344,7 +1352,6 @@ public List<FinalizedReplica> getFinalizedBlocks(String bpid) {
     throw new UnsupportedOperationException();
   }
 
-  @Override
   public List<FinalizedReplica> getFinalizedBlocksOnPersistentStorage(String bpid) {
     throw new UnsupportedOperationException();
   }