Not run/Skipped test reporting

codice-itest-api/src/main/java/org/codice/itest/api/TestResultFactory.java

@@ -40,4 +40,10 @@ public interface TestResultFactory {
      * @return an appropriate TestResult object.
      */
     TestResult error(String testName, Throwable throwable, Instant startTime, Instant endTime);
+
+    /**
+     * @param testName - the name of the test that did not run.
+     * @return an appropriate TestResult object.
+     */
+    TestResult notExecuted(String testName);
 }

codice-itest-api/src/main/java/org/codice/itest/api/TestStatus.java

@@ -15,7 +15,7 @@
  */
 
 public enum TestStatus {
-    PASS(0), FAIL(1), ERROR(2);
+    PASS(0), FAIL(1), ERROR(2), NOT_EXECUTED(3);
 
     private int returnCode;
     TestStatus(int returnCode) {

codice-itest-impl/src/main/java/org/codice/itest/ITestExecutorService.java

@@ -0,0 +1,60 @@
+/**
+ * Copyright (c) Codice Foundation
+
+ * This is free software: you can redistribute it and/or modify it under the terms of the GNU Lesser General Public License
+ * as published by the Free Software Foundation, either version 3 of the License, or any later version.
+
+ * This program is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
+ * See the GNU Lesser General Public License for more details. A copy of the GNU Lesser General Public License is distributed along with this program and can be found at
+ * http://www.gnu.org/licenses/lgpl.html.
+ */
+package org.codice.itest;
+
+import org.codice.itest.api.TestResult;
+import org.codice.itest.api.TestResultFactory;
+import org.codice.itest.config.ITestConfigurationProperties;
+import org.codice.itest.reporter.RemainingTestsReporter;
+import org.springframework.beans.factory.annotation.Value;
+import org.springframework.stereotype.Component;
+
+import java.util.List;
+import java.util.concurrent.BlockingQueue;
+import java.util.concurrent.ThreadPoolExecutor;
+import java.util.concurrent.TimeUnit;
+import java.util.function.Consumer;
+
+@Component("ITestExecutorService")
+public class ITestExecutorService extends ThreadPoolExecutor {
+
+    @Value("${itest.tests:#{null}}")
+    private String tests;
+
+    private List<Consumer<TestResult>> testResultListenerList;
+    private final ITestConfigurationProperties iTestConfigurationProperties;
+
+    private List<String> testNameList;
+
+    private RemainingTestsReporter remainingTestsReporter;
+
+    private TestResultFactory testResultFactory;
+
+    public ITestExecutorService(int corePoolSize, int maximumPoolSize, long keepAliveTime, TimeUnit unit,
+                                BlockingQueue<Runnable> workQueue, List<Consumer<TestResult>> testResultListenerList, ITestConfigurationProperties iTestConfigurationProperties, RemainingTestsReporter remainingTestsReporter, TestResultFactory testResultFactory) {
+        super(corePoolSize, maximumPoolSize, keepAliveTime, unit, workQueue);
+        this.testResultListenerList = testResultListenerList;
+        this.iTestConfigurationProperties = iTestConfigurationProperties;
+        this.testNameList = iTestConfigurationProperties.tests();
+        this.remainingTestsReporter = remainingTestsReporter;
+        this.testResultFactory = testResultFactory;
+    }
+
+    @Override
+    public List<Runnable> shutdownNow() {
+        remainingTestsReporter.getRemainingList().forEach(testName -> {
+            TestResult testResult = testResultFactory.notExecuted(testName);
+            testResultListenerList.forEach(listener -> listener.accept(testResult));
+        });
+
+        return super.shutdownNow();
+    }
+}

codice-itest-impl/src/main/java/org/codice/itest/IntegrationTestService.java

@@ -14,12 +14,10 @@
 import org.codice.itest.api.TestResult;
 import org.codice.itest.api.TestResultFactory;
 import org.codice.itest.config.ITestConfigurationProperties;
-import org.springframework.beans.factory.annotation.Value;
 import org.springframework.boot.CommandLineRunner;
 import org.springframework.stereotype.Component;
 
 import java.util.List;
-import java.util.concurrent.ExecutorService;
 import java.util.concurrent.TimeUnit;
 import java.util.function.Consumer;
 import java.util.stream.Stream;
@@ -41,26 +39,24 @@ final class IntegrationTestService implements CommandLineRunner {
 
     private List<Consumer<TestResult>> testResultListenerList;
 
-    private ExecutorService executorService;
+    private ITestExecutorService iTestExecutorService;
 
     private TestResultFactory testResultFactory;
 
     private ITestConfigurationProperties iTestConfigurationProperties;
     /**
-     *
-     * @param tests - The set of all DiagnosticTest objects found in the Spring application context.
-     * @param testResultListenerList - A list of listeners to be notified upon test completion.
-     * @param executorService - Used to allow tests to be executed in parallel.
+     * @param tests                    - The set of all DiagnosticTest objects found in the Spring application context.
+     * @param testResultListenerList   - A list of listeners to be notified upon test completion.
+     * @param iTestExecutorService - Used to allow tests to be executed in parallel while checking for not executed test results
      */
     public IntegrationTestService(Stream<IntegrationTest> tests,
                                   List<Consumer<TestResult>> testResultListenerList,
-                                  TestResultFactory testResultFactory,
-                                  ExecutorService executorService,
+                                  ITestExecutorService iTestExecutorService, TestResultFactory testResultFactory,
                                   ITestConfigurationProperties iTestConfigurationProperties) {
         this.tests = tests;
         this.testResultListenerList = testResultListenerList;
+        this.iTestExecutorService = iTestExecutorService;
         this.testResultFactory = testResultFactory;
-        this.executorService = executorService;
         this.iTestConfigurationProperties = iTestConfigurationProperties;
     }
 
@@ -71,9 +67,9 @@ public IntegrationTestService(Stream<IntegrationTest> tests,
      * @throws InterruptedException - When interrupted while waiting for a test to complete.
      */
     public void run(String[] args) throws InterruptedException {
-        this.tests.forEach(test -> executorService.execute(new TestExecutorTask(test,
+        this.tests.forEach(test -> iTestExecutorService.execute(new TestExecutorTask(test,
                 testResultListenerList, testResultFactory)));
-        executorService.shutdown();
-        executorService.awaitTermination(iTestConfigurationProperties.maxExecutionMinutes(), TimeUnit.MINUTES);
+        iTestExecutorService.shutdown();
+        iTestExecutorService.awaitTermination(iTestConfigurationProperties.maxExecutionMinutes(), TimeUnit.MINUTES);
     }
 }

codice-itest-impl/src/main/java/org/codice/itest/reporter/LoggingDiagnosticTestReporter.java

@@ -35,7 +35,7 @@ public void accept(TestResult testResult) {
                 break;
             case FAIL: logger.warn(formattedResult);
                 break;
-            case ERROR: logger.error(formattedResult);
+            case ERROR, NOT_EXECUTED: logger.error(formattedResult);
         }
     }
 }

codice-itest-impl/src/main/java/org/codice/itest/reporter/RemainingTestsReporter.java

@@ -0,0 +1,36 @@
+/**
+ * Copyright (c) Codice Foundation
+
+ * This is free software: you can redistribute it and/or modify it under the terms of the GNU Lesser General Public License
+ * as published by the Free Software Foundation, either version 3 of the License, or any later version.
+
+ * This program is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
+ * See the GNU Lesser General Public License for more details. A copy of the GNU Lesser General Public License is distributed along with this program and can be found at
+ * http://www.gnu.org/licenses/lgpl.html.
+ */
+package org.codice.itest.reporter;
+
+import org.codice.itest.api.TestResult;
+
+import java.util.List;
+import java.util.function.Consumer;
+
+public class RemainingTestsReporter implements Consumer<TestResult> {
+    private List<String> testNames;
+    private List<String> remainingTestNames;
+
+    public RemainingTestsReporter(List<String> testNames){
+        this.testNames = testNames;
+        this.remainingTestNames = testNames;
+    }
+
+    @Override
+    public void accept(TestResult testResult) {
+        if(testNames.contains(testResult.getTestName())){
+            remainingTestNames.remove(testResult.getTestName());
+        }
+    }
+
+    public List<String> getRemainingList(){return remainingTestNames;}
+
+}

codice-itest-impl/src/main/java/org/codice/itest/reporter/TestReporterConfiguration.java

@@ -19,14 +19,17 @@
 import org.springframework.context.annotation.Bean;
 import org.springframework.context.annotation.Configuration;
 
+import java.util.List;
 import java.util.function.Consumer;
 
 @Configuration
 public class TestReporterConfiguration {
     private final ITestConfigurationProperties iTestConfigurationProperties;
+    private List<String> testNameList;
 
     public TestReporterConfiguration(ITestConfigurationProperties iTestConfigurationProperties) {
         this.iTestConfigurationProperties = iTestConfigurationProperties;
+        this.testNameList = iTestConfigurationProperties.tests();
     }
 
     @Bean
@@ -52,4 +55,9 @@ public Consumer<TestResult> defaultLoggingDiagnosticTestReporter() {
     public Consumer<TestResult> exitCodeReporterConsumer(ExitCodeReporter exitCodeReporter) {
         return (tr) -> exitCodeReporter.register(tr.getTestStatus());
     }
+
+    @Bean("remainingTestsConsumer")
+    public Consumer<TestResult> remainingTestsReporterConsumer() {
+        return new RemainingTestsReporter(testNameList);
+    }
 }

codice-itest-impl/src/main/java/org/codice/itest/result/NotExecutedTestResultImpl.java

@@ -0,0 +1,63 @@
+/**
+ * Copyright (c) Codice Foundation
+
+ * This is free software: you can redistribute it and/or modify it under the terms of the GNU Lesser General Public License
+ * as published by the Free Software Foundation, either version 3 of the License, or any later version.
+
+ * This program is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.
+ * See the GNU Lesser General Public License for more details. A copy of the GNU Lesser General Public License is distributed along with this program and can be found at
+ * http://www.gnu.org/licenses/lgpl.html.
+ */
+package org.codice.itest.result;
+
+import org.apache.commons.lang3.builder.EqualsBuilder;
+import org.apache.commons.lang3.builder.HashCodeBuilder;
+
+import java.time.Instant;
+import java.util.UUID;
+
+import static org.codice.itest.api.TestStatus.NOT_EXECUTED;
+
+final class NotExecutedTestResultImpl extends BaseTestResult{
+
+    NotExecutedTestResultImpl(UUID runId, String testName) {
+        super(runId, testName, NOT_EXECUTED, Instant.now(), Instant.now());
+    }
+
+    @Override
+    public String toString() {
+        return toStringCreator.append(super.getRunId())
+                .append(super.getTestStatus())
+                .append(super.getTestName())
+                .append(NOT_EXECUTED)
+                .toString();
+    }
+
+    @Override
+    public boolean equals(Object other) {
+        if (other == null) {
+            return false;
+        }
+
+        if (other == this) {
+            return true;
+        }
+
+        if (!(other instanceof NotExecutedTestResultImpl testResult)) {
+            return false;
+        }
+
+        return new EqualsBuilder().append(super.getRunId(), testResult.getRunId())
+                .append(super.getTestName(), testResult.getTestName())
+                .append(super.getTestStatus(), testResult.getTestStatus())
+                .isEquals();
+    }
+
+    @Override
+    public int hashCode() {
+        return new HashCodeBuilder(1313, 4137).append(super.getRunId())
+                .append(super.getTestName())
+                .append(super.getTestStatus())
+                .build();
+    }
+}

codice-itest-impl/src/main/java/org/codice/itest/result/TestResultFactoryImpl.java

@@ -65,6 +65,12 @@ public TestResult error(String testName, Throwable throwable, Instant startTime,
         return new ErrorTestResultImpl(runId, testName, throwable, startTime, endTime);
     }
 
+    @Override
+    public TestResult notExecuted(String testName) {
+        MDC.put(TEST_STATUS, TestStatus.NOT_EXECUTED.name());
+        return new NotExecutedTestResultImpl(runId, testName);
+    }
+
     private void logTestResultCommonFields(String testName, Instant startTime, Instant endTime) {
         MDC.clear();
         MDC.put(TEST_NAME, testName);