Add Trace Time Interval Warning Retention Policy (#1342)

* Update documentation

* Add tests for trace time warning retention policy

* Add trace time warning retention policy

Author: pithumke

common/persistence/src/main/java/app/coronawarn/server/common/persistence/repository/TraceTimeIntervalWarningRepository.java

@@ -21,4 +21,22 @@ boolean saveDoNothingOnConflict(@Param("trace_location_id") byte[] traceLocation
       @Param("start_interval_number") Integer startIntervalNumber, @Param("period") Integer period,
       @Param("transmission_risk_level") Integer transmissionRiskLevel,
       @Param("submission_timestamp") Integer submissionTimestamp);
+
+  /**
+   * Counts all entries that have a submission timestamp older than the specified one.
+   *
+   * @param submissionTimestamp The submission timestamp up to which entries will be expired.
+   * @return The number of expired trace time warnings.
+   */
+  @Query("SELECT COUNT(*) FROM trace_time_interval_warning WHERE submission_timestamp<:threshold")
+  int countOlderThan(@Param("threshold") long submissionTimestamp);
+
+  /**
+   * Deletes all entries that have a submission timestamp older than the specified one.
+   *
+   * @param submissionTimestamp The submission timestamp up to which entries will be deleted.
+   */
+  @Modifying
+  @Query("DELETE FROM trace_time_interval_warning WHERE submission_timestamp<:threshold")
+  void deleteOlderThan(@Param("threshold") long submissionTimestamp);
 }

common/persistence/src/main/java/app/coronawarn/server/common/persistence/service/TraceTimeIntervalWarningService.java

@@ -1,14 +1,17 @@
 package app.coronawarn.server.common.persistence.service;
 
+import static app.coronawarn.server.common.persistence.domain.validation.ValidSubmissionTimestampValidator.SECONDS_PER_HOUR;
+import static java.time.ZoneOffset.UTC;
+
 import app.coronawarn.server.common.persistence.domain.TraceTimeIntervalWarning;
 import app.coronawarn.server.common.persistence.repository.TraceTimeIntervalWarningRepository;
-import app.coronawarn.server.common.persistence.service.utils.checkins.CheckinsDateSpecification;
 import app.coronawarn.server.common.persistence.service.utils.checkins.FakeCheckinsGenerator;
 import app.coronawarn.server.common.protocols.internal.pt.CheckIn;
 import com.google.protobuf.ByteString;
 import java.security.MessageDigest;
 import java.security.NoSuchAlgorithmException;
 import java.time.Instant;
+import java.time.LocalDateTime;
 import java.util.ArrayList;
 import java.util.Collection;
 import java.util.List;
@@ -28,15 +31,16 @@ public class TraceTimeIntervalWarningService {
   private static final Logger logger =
       LoggerFactory.getLogger(TraceTimeIntervalWarningService.class);
 
-
   private final TraceTimeIntervalWarningRepository traceTimeIntervalWarningRepo;
   private final FakeCheckinsGenerator fakeCheckinsGenerator;
   private final MessageDigest hashAlgorithm;
 
   /**
    * Constructs the service instance.
+   *
    * @param traceTimeIntervalWarningRepo Repository for {@link TraceTimeIntervalWarning} entities.
-   * @param fakeCheckinsGenerator Generator of fake data that gets stored side by side with the real checkin data.
+   * @param fakeCheckinsGenerator        Generator of fake data that gets stored side by side with the real checkin
+   *                                     data.
    * @throws NoSuchAlgorithmException In case the MessageDigest used in hashing can not be instantiated.
    */
   public TraceTimeIntervalWarningService(
@@ -48,16 +52,17 @@ public TraceTimeIntervalWarningService(
   }
 
   /**
-   * Store the given checkin data as {@link TraceTimeIntervalWarning} entities. Returns the number
-   * of inserted entities, which is useful for the case where there might be conflicts with the
-   * table constraints during the db save operations.
+   * Store the given checkin data as {@link TraceTimeIntervalWarning} entities. Returns the number of inserted entities,
+   * which is useful for the case where there might be conflicts with the table constraints during the db save
+   * operations.
    */
   @Transactional
-  public int saveCheckins(List<CheckIn> checkins) {
-    return saveCheckins(checkins, this::hashLocationId);
+  public int saveCheckins(List<CheckIn> checkins, int submissionTimestamp) {
+    return saveCheckins(checkins, this::hashLocationId, submissionTimestamp);
   }
 
-  private int saveCheckins(List<CheckIn> checkins, Function<ByteString, byte[]> idHashGenerator) {
+  private int saveCheckins(List<CheckIn> checkins, Function<ByteString, byte[]> idHashGenerator,
+      int submissionTimestamp) {
     int numberOfInsertedTraceWarnings = 0;
 
     for (CheckIn checkin : checkins) {
@@ -66,8 +71,7 @@ private int saveCheckins(List<CheckIn> checkins, Function<ByteString, byte[]> id
           .saveDoNothingOnConflict(hashId, checkin.getStartIntervalNumber(),
               checkin.getEndIntervalNumber() - checkin.getStartIntervalNumber(),
               checkin.getTransmissionRiskLevel(),
-              CheckinsDateSpecification.HOUR_SINCE_EPOCH_DERIVATION
-                  .apply(Instant.now().getEpochSecond()));
+              submissionTimestamp);
 
       if (traceWarningInsertedSuccessfully) {
         numberOfInsertedTraceWarnings++;
@@ -78,7 +82,7 @@ private int saveCheckins(List<CheckIn> checkins, Function<ByteString, byte[]> id
     if (conflictingTraceWarnings > 0) {
       logger.warn(
           "{} out of {} TraceTimeIntervalWarnings conflicted with existing "
-          + "database entries or had errors while storing "
+              + "database entries or had errors while storing "
               + "and were ignored.",
           conflictingTraceWarnings, checkins.size());
     }
@@ -87,18 +91,17 @@ private int saveCheckins(List<CheckIn> checkins, Function<ByteString, byte[]> id
   }
 
   /**
-   * For each checkin in the given list, generate other fake checkin data based on the passed in
-   * number and store everything as {@link TraceTimeIntervalWarning} entities. Returns the number of
-   * inserted entities which is useful for the case where there might be conflicts with the table
-   * constraints during the db save operations.
+   * For each checkin in the given list, generate other fake checkin data based on the passed in number and store
+   * everything as {@link TraceTimeIntervalWarning} entities. Returns the number of inserted entities which is useful
+   * for the case where there might be conflicts with the table constraints during the db save operations.
    */
   @Transactional
   public int saveCheckinsWithFakeData(List<CheckIn> originalCheckins, int numberOfFakesToCreate,
-      byte[] pepper) {
+      byte[] pepper, int submissionTimestamp) {
     List<CheckIn> allCheckins = new ArrayList<>(originalCheckins);
     allCheckins.addAll(fakeCheckinsGenerator.generateFakeCheckins(originalCheckins,
         numberOfFakesToCreate, pepper));
-    return saveCheckins(allCheckins, this::hashLocationId);
+    return saveCheckins(allCheckins, this::hashLocationId, submissionTimestamp);
   }
 
   /**
@@ -114,4 +117,27 @@ public Collection<TraceTimeIntervalWarning> getTraceTimeIntervalWarnings() {
   private byte[] hashLocationId(ByteString locationId) {
     return hashAlgorithm.digest(locationId.toByteArray());
   }
+
+  /**
+   * Deletes all trace time warning entries which have a submission timestamp that is older than the specified number
+   * of days.
+   *
+   * @param daysToRetain the number of days until which trace time warnings will be retained.
+   * @throws IllegalArgumentException if {@code daysToRetain} is negative.
+   */
+  @Transactional
+  public void applyRetentionPolicy(int daysToRetain) {
+    if (daysToRetain < 0) {
+      throw new IllegalArgumentException("Number of days to retain must be greater or equal to 0.");
+    }
+
+    long threshold = LocalDateTime
+        .ofInstant(Instant.now(), UTC)
+        .minusDays(daysToRetain)
+        .toEpochSecond(UTC) / SECONDS_PER_HOUR;
+    int numberOfDeletions = traceTimeIntervalWarningRepo.countOlderThan(threshold);
+    logger.info("Deleting {} trace time warning(s) with a submission timestamp older than {} day(s) ago.",
+        numberOfDeletions, daysToRetain);
+    traceTimeIntervalWarningRepo.deleteOlderThan(threshold);
+  }
 }

common/persistence/src/test/java/app/coronawarn/server/common/persistence/service/TraceTimeIntervalWarningServiceTest.java

@@ -1,27 +1,35 @@
 package app.coronawarn.server.common.persistence.service;
 
+import static org.assertj.core.api.Assertions.assertThat;
+import static org.assertj.core.api.Assertions.assertThatCode;
+import static org.assertj.core.api.Assertions.catchThrowable;
+import static org.junit.Assert.assertArrayEquals;
+import static org.junit.Assert.assertEquals;
+
 import app.coronawarn.server.common.persistence.domain.TraceTimeIntervalWarning;
 import app.coronawarn.server.common.persistence.repository.TraceTimeIntervalWarningRepository;
 import app.coronawarn.server.common.persistence.service.utils.checkins.CheckinsDateSpecification;
 import app.coronawarn.server.common.protocols.internal.pt.CheckIn;
 import com.google.protobuf.ByteString;
-import org.junit.jupiter.api.BeforeEach;
-import org.junit.jupiter.api.Test;
-import org.springframework.beans.factory.annotation.Autowired;
-import org.springframework.boot.test.autoconfigure.data.jdbc.DataJdbcTest;
-import org.testcontainers.shaded.org.bouncycastle.util.encoders.Hex;
 import java.security.MessageDigest;
 import java.security.NoSuchAlgorithmException;
 import java.security.SecureRandom;
 import java.time.Instant;
-import java.util.ArrayList;
+import java.util.Collection;
 import java.util.Collections;
 import java.util.Comparator;
 import java.util.List;
+import java.util.concurrent.TimeUnit;
 import java.util.stream.Collectors;
 import java.util.stream.StreamSupport;
-
-import static org.junit.Assert.*;
+import org.junit.jupiter.api.BeforeEach;
+import org.junit.jupiter.api.DisplayName;
+import org.junit.jupiter.api.Test;
+import org.junit.jupiter.params.ParameterizedTest;
+import org.junit.jupiter.params.provider.ValueSource;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.boot.test.autoconfigure.data.jdbc.DataJdbcTest;
+import org.testcontainers.shaded.org.bouncycastle.util.encoders.Hex;
 
 @DataJdbcTest
 class TraceTimeIntervalWarningServiceTest {
@@ -32,70 +40,35 @@ class TraceTimeIntervalWarningServiceTest {
   @Autowired
   TraceTimeIntervalWarningRepository traceWarningsRepository;
 
+  private int currentTimestamp;
+
   @BeforeEach
   public void setup() {
     traceWarningsRepository.deleteAll();
+    currentTimestamp = CheckinsDateSpecification.HOUR_SINCE_EPOCH_DERIVATION.apply(Instant.now().getEpochSecond());
   }
 
   @Test
   void testStorage() {
     List<CheckIn> checkins = getRandomTestData();
-    traceWarningsService.saveCheckins(checkins);
+    traceWarningsService.saveCheckins(checkins, currentTimestamp);
 
     List<TraceTimeIntervalWarning> actualTraceWarningsStored =
         StreamSupport.stream(traceWarningsRepository.findAll().spliterator(), false)
             .collect(Collectors.toList());
 
-    assertCheckinsAndWarningsAreEqual(new ArrayList<>(checkins), actualTraceWarningsStored);
+    assertCheckinsAndWarningsAreEqual(checkins, actualTraceWarningsStored);
   }
 
   @Test
   void testStorageWithRandomPadding() {
     List<CheckIn> checkins = getRandomTestData();
-    traceWarningsService.saveCheckinsWithFakeData(checkins, 2, randomHashPepper());
+    traceWarningsService.saveCheckinsWithFakeData(checkins, 2, randomHashPepper(), currentTimestamp);
 
     List<TraceTimeIntervalWarning> actualTraceWarningsStored =
         StreamSupport.stream(traceWarningsRepository.findAll().spliterator(), false)
             .collect(Collectors.toList());
-    assertTrue(actualTraceWarningsStored.size() == checkins.size() + checkins.size() * 2);
-  }
-
-  private List<CheckIn> getRandomTestData() {
-    List<CheckIn> checkins = List.of(
-        CheckIn.newBuilder().setStartIntervalNumber(0).setEndIntervalNumber(1)
-            .setTransmissionRiskLevel(1)
-            .setLocationId(ByteString.copyFromUtf8("uuid1"))
-            .build(),
-        CheckIn.newBuilder().setStartIntervalNumber(23).setEndIntervalNumber(30)
-            .setTransmissionRiskLevel(2)
-            .setLocationId(ByteString.copyFromUtf8("uuid1"))
-            .build(),
-        CheckIn.newBuilder().setStartIntervalNumber(40).setEndIntervalNumber(50)
-            .setTransmissionRiskLevel(3)
-            .setLocationId(ByteString.copyFromUtf8("uuid1"))
-            .build());
-    return checkins;
-  }
-
-  private void assertCheckinsAndWarningsAreEqual(List<CheckIn> checkins,
-      List<TraceTimeIntervalWarning> actualTraceWarningsStored) {
-
-    assertEquals(checkins.size(), actualTraceWarningsStored.size());
-
-    Collections.sort(checkins, Comparator.comparing(CheckIn::getTransmissionRiskLevel));
-    Collections.sort(actualTraceWarningsStored,
-        Comparator.comparing(TraceTimeIntervalWarning::getTransmissionRiskLevel));
-
-    for (int i = 0; i < checkins.size(); i++) {
-      CheckIn checkin = checkins.get(i);
-      TraceTimeIntervalWarning warning = actualTraceWarningsStored.get(i);
-      assertEquals(checkin.getTransmissionRiskLevel(),
-          warning.getTransmissionRiskLevel().intValue());
-      assertEquals(checkin.getStartIntervalNumber(), warning.getStartIntervalNumber().intValue());
-      assertEquals(checkin.getEndIntervalNumber() - checkin.getStartIntervalNumber(), warning.getPeriod().intValue());
-      assertArrayEquals(hashLocationId(checkin.getLocationId()),
-          warning.getTraceLocationId());
-    }
+    assertEquals(actualTraceWarningsStored.size(), checkins.size() + checkins.size() * 2);
   }
 
   @Test
@@ -115,7 +88,7 @@ void testSortedRetrievalResult() {
             CheckinsDateSpecification.HOUR_SINCE_EPOCH_DERIVATION
                 .apply(Instant.now().getEpochSecond()) - 10);
 
-    List<CheckIn> checkins = new ArrayList<>(List.of(
+    List<CheckIn> checkins = new java.util.ArrayList<>(List.of(
         CheckIn.newBuilder().setStartIntervalNumber(56).setEndIntervalNumber(66)
             .setTransmissionRiskLevel(3)
             .setLocationId(ByteString.copyFromUtf8("sorted-uuid2"))
@@ -128,8 +101,7 @@ void testSortedRetrievalResult() {
     // Reverse as we tempered with submission timestamp
     Collections.reverse(checkins);
 
-    List<TraceTimeIntervalWarning> checkinsFromDB = new ArrayList<>(
-        traceWarningsService.getTraceTimeIntervalWarnings());
+    var checkinsFromDB = traceWarningsService.getTraceTimeIntervalWarnings();
 
     assertCheckinsAndWarningsAreEqual(checkins, checkinsFromDB);
   }
@@ -149,11 +121,95 @@ public void testHashingOfTraceLocationId() {
     assertEquals("0f37dac11d1b8118ea0b44303400faa5e3b876da9d758058b5ff7dc2e5da8230", s);
   }
 
+  @DisplayName("Assert a positive retention period is accepted.")
+  @ValueSource(ints = {0, 1, Integer.MAX_VALUE})
+  @ParameterizedTest
+  void testApplyRetentionPolicyForValidNumberOfDays(int daysToRetain) {
+    assertThatCode(() -> traceWarningsService.applyRetentionPolicy(daysToRetain))
+        .doesNotThrowAnyException();
+  }
+
+  @DisplayName("Assert a negative retention period is rejected.")
+  @ValueSource(ints = {Integer.MIN_VALUE, -1})
+  @ParameterizedTest
+  void testApplyRetentionPolicyForNegativeNumberOfDays(int daysToRetain) {
+    assertThat(catchThrowable(() -> traceWarningsService.applyRetentionPolicy(daysToRetain)))
+        .isInstanceOf(IllegalArgumentException.class);
+  }
+
+  @Test
+  void testApplyRetentionPolicyForEmptyDb() {
+    traceWarningsService.applyRetentionPolicy(1);
+    var actKeys = traceWarningsService.getTraceTimeIntervalWarnings();
+    assertThat(actKeys).isEmpty();
+  }
+
+  @Test
+  void testApplyRetentionPolicyForNotApplicableEntries() {
+    var expKeys = getRandomTestData();
+
+    traceWarningsService.saveCheckins(expKeys, currentTimestamp);
+    traceWarningsService.applyRetentionPolicy(1);
+    var actKeys = traceWarningsService.getTraceTimeIntervalWarnings();
+
+    assertCheckinsAndWarningsAreEqual(expKeys, actKeys);
+  }
+
+  @Test
+  void testApplyRetentionPolicyForOneApplicableEntry() {
+    var keys = getRandomTestData();
+
+    traceWarningsService.saveCheckins(keys, currentTimestamp - (int) TimeUnit.DAYS.toHours(1) - 1);
+    traceWarningsService.applyRetentionPolicy(1);
+    var actKeys = traceWarningsService.getTraceTimeIntervalWarnings();
+
+    assertThat(actKeys).isEmpty();
+  }
+
+  private List<CheckIn> getRandomTestData() {
+    return List.of(
+        CheckIn.newBuilder().setStartIntervalNumber(0).setEndIntervalNumber(1)
+            .setTransmissionRiskLevel(1)
+            .setLocationId(ByteString.copyFromUtf8("uuid1"))
+            .build(),
+        CheckIn.newBuilder().setStartIntervalNumber(23).setEndIntervalNumber(30)
+            .setTransmissionRiskLevel(2)
+            .setLocationId(ByteString.copyFromUtf8("uuid1"))
+            .build(),
+        CheckIn.newBuilder().setStartIntervalNumber(40).setEndIntervalNumber(50)
+            .setTransmissionRiskLevel(3)
+            .setLocationId(ByteString.copyFromUtf8("uuid1"))
+            .build());
+  }
+
+  private void assertCheckinsAndWarningsAreEqual(Collection<CheckIn> checkins,
+      Collection<TraceTimeIntervalWarning> actualTraceWarningsStored) {
+
+    assertEquals(checkins.size(), actualTraceWarningsStored.size());
+
+    var sortedCheckins = checkins.stream()
+        .sorted(Comparator.comparing(CheckIn::getTransmissionRiskLevel))
+        .collect(Collectors.toList());
+    var sortedTraceTimeWarnings = actualTraceWarningsStored.stream()
+        .sorted(Comparator.comparing(TraceTimeIntervalWarning::getTransmissionRiskLevel))
+        .collect(Collectors.toList());
+
+    for (int i = 0; i < checkins.size(); i++) {
+      CheckIn checkin = sortedCheckins.get(i);
+      TraceTimeIntervalWarning warning = sortedTraceTimeWarnings.get(i);
+      assertEquals(checkin.getTransmissionRiskLevel(),
+          warning.getTransmissionRiskLevel().intValue());
+      assertEquals(checkin.getStartIntervalNumber(), warning.getStartIntervalNumber().intValue());
+      assertEquals(checkin.getEndIntervalNumber() - checkin.getStartIntervalNumber(), warning.getPeriod().intValue());
+      assertArrayEquals(hashLocationId(checkin.getLocationId()),
+          warning.getTraceLocationId());
+    }
+  }
 
   private byte[] hashLocationId(ByteString locationId) {
     try {
       return MessageDigest.getInstance("SHA-256").digest(locationId.toByteArray());
-    } catch (NoSuchAlgorithmException e) {
+    } catch (NoSuchAlgorithmException ignored) {
     }
     return new byte[0];
   }