diff --git a/dynamometer-workload/build.gradle b/dynamometer-workload/build.gradle index 52e639b645..2d48a043a8 100644 --- a/dynamometer-workload/build.gradle +++ b/dynamometer-workload/build.gradle @@ -4,6 +4,7 @@ */ dependencies { compile deps.hadoop.common + compile deps.hadoop.hdfs compile deps.hadoop.'mapreduce-client-core' testCompile deps.hadoop.minicluster diff --git a/dynamometer-workload/src/main/java/com/linkedin/dynamometer/workloadgenerator/audit/AuditReplayThread.java b/dynamometer-workload/src/main/java/com/linkedin/dynamometer/workloadgenerator/audit/AuditReplayThread.java index 0dd5efc58e..726e3c98f7 100644 --- a/dynamometer-workload/src/main/java/com/linkedin/dynamometer/workloadgenerator/audit/AuditReplayThread.java +++ b/dynamometer-workload/src/main/java/com/linkedin/dynamometer/workloadgenerator/audit/AuditReplayThread.java @@ -21,6 +21,10 @@ import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; import org.apache.hadoop.fs.permission.FsPermission; +import org.apache.hadoop.hdfs.DFSClient; +import org.apache.hadoop.hdfs.DynoDFSUtil; +import org.apache.hadoop.hdfs.DistributedFileSystem; +import org.apache.hadoop.hdfs.protocol.HdfsFileStatus; import org.apache.hadoop.mapreduce.Counter; import org.apache.hadoop.mapreduce.Mapper; import org.apache.hadoop.mapreduce.counters.GenericCounter; @@ -50,6 +54,7 @@ public class AuditReplayThread extends Thread { private Exception exception; private long startTimestampMs; private FileSystem fs; + private DFSClient dfsClient; private boolean createBlocks; // Counters are not thread-safe so we store a local mapping in our thread @@ -65,6 +70,7 @@ public class AuditReplayThread extends Thread { createBlocks = mapperConf.getBoolean(AuditReplayMapper.CREATE_BLOCKS_KEY, AuditReplayMapper.CREATE_BLOCKS_DEFAULT); fs = FileSystem.get(URI.create(namenodeURI), mapperConf); + dfsClient = DynoDFSUtil.getDFSClient((DistributedFileSystem) fs); LOG.info("Start timestamp: " + startTimestampMs); for (REPLAYCOUNTERS rc : REPLAYCOUNTERS.values()) { replayCountersMap.put(rc, new GenericCounter()); @@ -185,7 +191,7 @@ private boolean replayLog(String command, String src, String dst) { break; case LISTSTATUS: - fs.listStatus(new Path(src)); + dfsClient.listPaths(src, HdfsFileStatus.EMPTY_NAME); break; case APPEND: @@ -193,7 +199,7 @@ private boolean replayLog(String command, String src, String dst) { return true; case DELETE: - fs.delete(new Path(src), false); + fs.delete(new Path(src), true); break; case OPEN: diff --git a/dynamometer-workload/src/main/java/org/apache/hadoop/hdfs/DynoDFSUtil.java b/dynamometer-workload/src/main/java/org/apache/hadoop/hdfs/DynoDFSUtil.java new file mode 100644 index 0000000000..741919e28a --- /dev/null +++ b/dynamometer-workload/src/main/java/org/apache/hadoop/hdfs/DynoDFSUtil.java @@ -0,0 +1,13 @@ +/** + * Copyright 2017 LinkedIn Corporation. All rights reserved. Licensed under the BSD-2 Clause license. + * See LICENSE in the project root for license information. + */ +package org.apache.hadoop.hdfs; + +public class DynoDFSUtil { + + public static DFSClient getDFSClient(DistributedFileSystem dfs) { + return dfs.dfs; + } + +}