From 52ffbbc7eadc4eab855c05b9e6d1453f4a268d5d Mon Sep 17 00:00:00 2001 From: Christopher Gregorian Date: Tue, 26 Feb 2019 15:19:50 -0800 Subject: [PATCH 1/3] Add travis build step and fix javadoc formatting --- .travis.yml | 18 +++++++++++++++--- .../workloadgenerator/TimedInputFormat.java | 8 ++++---- .../audit/AuditLogHiveTableParser.java | 2 +- .../audit/UserCommandKey.java | 2 +- 4 files changed, 21 insertions(+), 9 deletions(-) diff --git a/.travis.yml b/.travis.yml index 4b9bb96785..ceda02d265 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,9 +1,21 @@ language: java -git: - depth: 3 jdk: - oraclejdk8 env: - HADOOP_VERSION=2.7.6 - HADOOP_VERSION=2.8.4 -script: "./gradlew test --stacktrace --info -Ddyno.hadoop.bin.version=${HADOOP_VERSION}" +git: + depth: 3 + +jobs: + include: + - stage: test + name: "Build" + script: "./gradlew build" + install: true + - name: "Unit tests" + script: "./gradlew test --stacktrace --info -Ddyno.hadoop.bin.version=${HADOOP_VERSION}" + +stages: + - test + diff --git a/dynamometer-workload/src/main/java/com/linkedin/dynamometer/workloadgenerator/TimedInputFormat.java b/dynamometer-workload/src/main/java/com/linkedin/dynamometer/workloadgenerator/TimedInputFormat.java index 147c57e5f7..6880d87fcc 100644 --- a/dynamometer-workload/src/main/java/com/linkedin/dynamometer/workloadgenerator/TimedInputFormat.java +++ b/dynamometer-workload/src/main/java/com/linkedin/dynamometer/workloadgenerator/TimedInputFormat.java @@ -20,11 +20,11 @@ /** * An {@link InputFormat} which is time-based. Starts at some timestamp (specified by the - * {@value WorkloadDriver#START_TIMESTAMP_MS configuration) and runs for a time specified by the + * {@value WorkloadDriver#START_TIMESTAMP_MS} configuration) and runs for a time specified by the * {@value DURATION_KEY} configuration. Spawns {@value NUM_MAPPERS_KEY} mappers. Both {@value DURATION_KEY} * and {@value NUM_MAPPERS_KEY} are required. * - *

The values returned as the key by this InputFormat are just a sequential counter. + *

The values returned as the key by this InputFormat are just a sequential counter. */ public class TimedInputFormat extends InputFormat { @@ -48,14 +48,14 @@ public List getSplits(JobContext job) { public RecordReader createRecordReader(InputSplit split, TaskAttemptContext context) { return new TimedRecordReader(); } - + public static List getConfigDescriptions() { return Lists.newArrayList( NUM_MAPPERS_KEY + " (required): Number of mappers to launch.", DURATION_KEY + " (required): Number of minutes to induce workload for." ); } - + public static boolean verifyConfigurations(Configuration conf) { return conf.getInt(NUM_MAPPERS_KEY, -1) != -1 && conf.getTimeDuration(DURATION_KEY, -1, TimeUnit.MILLISECONDS) != -1; diff --git a/dynamometer-workload/src/main/java/com/linkedin/dynamometer/workloadgenerator/audit/AuditLogHiveTableParser.java b/dynamometer-workload/src/main/java/com/linkedin/dynamometer/workloadgenerator/audit/AuditLogHiveTableParser.java index 7651fd71a5..67cd7a603b 100644 --- a/dynamometer-workload/src/main/java/com/linkedin/dynamometer/workloadgenerator/audit/AuditLogHiveTableParser.java +++ b/dynamometer-workload/src/main/java/com/linkedin/dynamometer/workloadgenerator/audit/AuditLogHiveTableParser.java @@ -25,7 +25,7 @@ * INSERT OVERWRITE DIRECTORY '${outputPath}' * SELECT (timestamp - ${startTimestamp} AS relativeTimestamp, ugi, cmd, src, dst, ip * FROM '${auditLogTableLocation}' - * WHERE timestamp >= ${startTimestamp} AND timestamp < ${endTimestamp} + * WHERE timestamp >= ${startTimestamp} AND timestamp < ${endTimestamp} * DISTRIBUTE BY src * SORT BY relativeTimestamp ASC; * diff --git a/dynamometer-workload/src/main/java/com/linkedin/dynamometer/workloadgenerator/audit/UserCommandKey.java b/dynamometer-workload/src/main/java/com/linkedin/dynamometer/workloadgenerator/audit/UserCommandKey.java index 7126bd83f4..2c4c922263 100644 --- a/dynamometer-workload/src/main/java/com/linkedin/dynamometer/workloadgenerator/audit/UserCommandKey.java +++ b/dynamometer-workload/src/main/java/com/linkedin/dynamometer/workloadgenerator/audit/UserCommandKey.java @@ -13,7 +13,7 @@ import org.apache.hadoop.io.WritableComparable; /** - * UserCommandKey is a {@link Writable} used as a composite key combining the user id and + * UserCommandKey is a {@link WritableComparable} used as a composite key combining the user id and * type of a replayed command. It is used as the output key for AuditReplayMapper and the * keys for AuditReplayReducer. */ From a94aff664359f25f938ea12f1633be945040bd2f Mon Sep 17 00:00:00 2001 From: Christopher Gregorian Date: Wed, 27 Feb 2019 09:17:36 -0800 Subject: [PATCH 2/3] Change to single job with both Hadoop versions --- .travis.yml | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/.travis.yml b/.travis.yml index ceda02d265..bbde0af5b0 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,21 +1,21 @@ language: java jdk: - oraclejdk8 -env: - - HADOOP_VERSION=2.7.6 - - HADOOP_VERSION=2.8.4 git: depth: 3 +install: true jobs: include: - stage: test name: "Build" - script: "./gradlew build" - install: true - - name: "Unit tests" + script: "./gradlew build -x test" + - stage: test + script: "./gradlew test --stacktrace --info -Ddyno.hadoop.bin.version=${HADOOP_VERSION}" + env: HADOOP_VERSION=2.7.6 + - stage: test script: "./gradlew test --stacktrace --info -Ddyno.hadoop.bin.version=${HADOOP_VERSION}" + env: HADOOP_VERSION=2.8.4 stages: - test - From 2588d2654937699e449daa40c9b7ceffe71bd896 Mon Sep 17 00:00:00 2001 From: Christopher Gregorian Date: Wed, 27 Feb 2019 09:32:24 -0800 Subject: [PATCH 3/3] Prettier javadoc --- .../workloadgenerator/audit/AuditLogHiveTableParser.java | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/dynamometer-workload/src/main/java/com/linkedin/dynamometer/workloadgenerator/audit/AuditLogHiveTableParser.java b/dynamometer-workload/src/main/java/com/linkedin/dynamometer/workloadgenerator/audit/AuditLogHiveTableParser.java index 67cd7a603b..ab7a51a1e7 100644 --- a/dynamometer-workload/src/main/java/com/linkedin/dynamometer/workloadgenerator/audit/AuditLogHiveTableParser.java +++ b/dynamometer-workload/src/main/java/com/linkedin/dynamometer/workloadgenerator/audit/AuditLogHiveTableParser.java @@ -21,14 +21,14 @@ * Where relativeTimestampMs represents the time elapsed between the start of * the audit log and the occurrence of the audit event. Assuming your audit * logs are available in Hive, this can be generated with a query looking like: - * 

+ * 
{@code
  *   INSERT OVERWRITE DIRECTORY '${outputPath}'
  *   SELECT (timestamp - ${startTimestamp} AS relativeTimestamp, ugi, cmd, src, dst, ip
  *   FROM '${auditLogTableLocation}'
- *   WHERE timestamp >= ${startTimestamp} AND timestamp < ${endTimestamp}
+ *   WHERE timestamp >= ${startTimestamp} AND timestamp < ${endTimestamp}
  *   DISTRIBUTE BY src
  *   SORT BY relativeTimestamp ASC;
- * 

+ * }
* Note that the sorting step is important; events in each distinct file must be in * time-ascending order. */