From 21fd9b036a20d2a282fd142d36f666283fd99294 Mon Sep 17 00:00:00 2001
From: Erik Krogen <ekrogen@linkedin.com>
Date: Thu, 10 Jan 2019 13:34:02 -0800
Subject: [PATCH 1/2] Avoid needing to use fixed ports for the MiniYARN cluster
 by using classpath setup as in TestDistributedShell

---
 .../dynamometer/TestDynamometerInfra.java     | 19 +++++++++++++++++--
 .../src/test/resources/yarn-site.xml          | 12 ++++++++++++
 2 files changed, 29 insertions(+), 2 deletions(-)
 create mode 100644 dynamometer-infra/src/test/resources/yarn-site.xml

diff --git a/dynamometer-infra/src/test/java/com/linkedin/dynamometer/TestDynamometerInfra.java b/dynamometer-infra/src/test/java/com/linkedin/dynamometer/TestDynamometerInfra.java
index d36474d8ec..e18ad04f4f 100644
--- a/dynamometer-infra/src/test/java/com/linkedin/dynamometer/TestDynamometerInfra.java
+++ b/dynamometer-infra/src/test/java/com/linkedin/dynamometer/TestDynamometerInfra.java
@@ -9,12 +9,15 @@
 import com.google.common.collect.Sets;
 import com.linkedin.dynamometer.workloadgenerator.audit.AuditLogDirectParser;
 import com.linkedin.dynamometer.workloadgenerator.audit.AuditReplayMapper;
+import java.io.ByteArrayOutputStream;
 import java.io.File;
 import java.io.FileOutputStream;
 import java.io.IOException;
 import java.io.InputStream;
+import java.io.OutputStream;
 import java.net.URI;
 import java.net.URISyntaxException;
+import java.net.URL;
 import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
@@ -152,8 +155,6 @@ public static void setupClass() throws Exception {
       fail("Unable to execute tar to expand Hadoop binary");
     }
 
-    conf.setBoolean(YarnConfiguration.YARN_MINICLUSTER_FIXED_PORTS, true);
-    conf.setBoolean(YarnConfiguration.YARN_MINICLUSTER_USE_RPC, true);
     conf.setInt(YarnConfiguration.RM_SCHEDULER_MINIMUM_ALLOCATION_MB, 128);
     conf.setBoolean(YarnConfiguration.NODE_LABELS_ENABLED, true);
     for (String q : new String[] { "root", "root.default" } ) {
@@ -182,6 +183,20 @@ public static void setupClass() throws Exception {
     FileSystem.setDefaultUri(yarnConf, miniDFSCluster.getURI());
     fs = miniDFSCluster.getFileSystem();
 
+    URL url = Thread.currentThread().getContextClassLoader().getResource("yarn-site.xml");
+    if (url == null) {
+      throw new RuntimeException("Could not find 'yarn-site.xml' dummy file in classpath");
+    }
+    yarnConf.set(YarnConfiguration.YARN_APPLICATION_CLASSPATH, new File(url.getPath()).getParent());
+    // Write the XML to a buffer before writing to the file, since the XML dump can cause the
+    // yarn-site.xml file to be read
+    try (ByteArrayOutputStream bytesOut = new ByteArrayOutputStream()) {
+      yarnConf.writeXml(bytesOut);
+      try (OutputStream fileOut = new FileOutputStream(new File(url.getPath()))) {
+        fileOut.write(bytesOut.toByteArray());
+      }
+    }
+
     yarnClient = YarnClient.createYarnClient();
     yarnClient.init(new Configuration(yarnConf));
     yarnClient.start();
diff --git a/dynamometer-infra/src/test/resources/yarn-site.xml b/dynamometer-infra/src/test/resources/yarn-site.xml
new file mode 100644
index 0000000000..4adae53303
--- /dev/null
+++ b/dynamometer-infra/src/test/resources/yarn-site.xml
@@ -0,0 +1,12 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
+<!--
+
+    Copyright 2019 LinkedIn Corporation. All rights reserved. Licensed under the BSD-2 Clause license.
+    See LICENSE in the project root for license information.
+
+-->
+
+<configuration>
+  <!-- Dummy (invalid) config file to be overwritten by tests with MiniCluster configuration. -->
+</configuration>

From 22e847adaf93af36f3becabd92fe990c5da16502 Mon Sep 17 00:00:00 2001
From: Erik Krogen <ekrogen@linkedin.com>
Date: Fri, 11 Jan 2019 10:00:15 -0800
Subject: [PATCH 2/2] Try to make the comment about the buffer for yarn-site
 more clear

---
 .../java/com/linkedin/dynamometer/TestDynamometerInfra.java  | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/dynamometer-infra/src/test/java/com/linkedin/dynamometer/TestDynamometerInfra.java b/dynamometer-infra/src/test/java/com/linkedin/dynamometer/TestDynamometerInfra.java
index e18ad04f4f..44a108dfb9 100644
--- a/dynamometer-infra/src/test/java/com/linkedin/dynamometer/TestDynamometerInfra.java
+++ b/dynamometer-infra/src/test/java/com/linkedin/dynamometer/TestDynamometerInfra.java
@@ -188,8 +188,9 @@ public static void setupClass() throws Exception {
       throw new RuntimeException("Could not find 'yarn-site.xml' dummy file in classpath");
     }
     yarnConf.set(YarnConfiguration.YARN_APPLICATION_CLASSPATH, new File(url.getPath()).getParent());
-    // Write the XML to a buffer before writing to the file, since the XML dump can cause the
-    // yarn-site.xml file to be read
+    // Write the XML to a buffer before writing to the file. writeXml() can trigger a read of the existing
+    // yarn-site.xml, so writing directly could trigger a read of the file while it is in an inconsistent state
+    // (partially written)
     try (ByteArrayOutputStream bytesOut = new ByteArrayOutputStream()) {
       yarnConf.writeXml(bytesOut);
       try (OutputStream fileOut = new FileOutputStream(new File(url.getPath()))) {