fix(striped_backups): changed logic to get all striped backups

src/BackupChain.cs

@@ -4,6 +4,7 @@ namespace AgDatabaseMove
   using System.Collections.Generic;
   using System.IO;
   using System.Linq;
+  using Exceptions;
   using SmoFacade;
 
 
@@ -19,26 +20,28 @@ public class BackupChain : IBackupChain
   {
     private readonly IList<BackupMetadata> _orderedBackups;
 
+    // This also handles any striped backups
     private BackupChain(IList<BackupMetadata> recentBackups)
     {
+      if (recentBackups == null || recentBackups.Count == 0) {
+        throw new BackupChainException("There are no recent backups to form a chain");
+      }
+
       var backups = recentBackups.Distinct(new BackupMetadataEqualityComparer())
-        .Where(b => IsValidFilePath(b)) // A third party application caused invalid path strings to be inserted into backupmediafamily
+        .Where(IsValidFilePath) // A third party application caused invalid path strings to be inserted into backupmediafamily
         .ToList();
 
-      var mostRecentFullBackup = MostRecentFullBackup(recentBackups);
-      _orderedBackups = new List<BackupMetadata> { mostRecentFullBackup };
-
-      var differentialBackup = MostRecentDifferentialBackup(backups, mostRecentFullBackup);
-      if(differentialBackup != null)
-        _orderedBackups.Add(differentialBackup);
+      var orderedBackups = MostRecentFullBackup(backups).ToList();
+      orderedBackups.AddRange(MostRecentDiffBackup(backups, orderedBackups.First()));
 
-      var mostRecentBackup = _orderedBackups.Last();
-      while(mostRecentBackup != null) {
-        mostRecentBackup = NextLogBackup(backups, mostRecentBackup);
-
-        if(mostRecentBackup != null)
-          _orderedBackups.Add(mostRecentBackup);
+      var prevBackup = orderedBackups.Last();
+      IEnumerable<BackupMetadata> nextLogBackups;
+      while((nextLogBackups = NextLogBackup(backups, prevBackup)).Any()) {
+        orderedBackups.AddRange(nextLogBackups);
+        prevBackup = orderedBackups.Last();
       }
+
+      _orderedBackups = orderedBackups;
     }
 
     /// <summary>
@@ -56,26 +59,44 @@ public BackupChain(Database database) : this(database.RecentBackups()) { }
     /// </summary>
     public IEnumerable<BackupMetadata> OrderedBackups => _orderedBackups;
 
-    private BackupMetadata MostRecentFullBackup(IList<BackupMetadata> backups)
+    private static IEnumerable<BackupMetadata> MostRecentFullBackup(IEnumerable<BackupMetadata> backups)
     {
-      return backups.Where(b => b.BackupType == BackupFileTools.BackupType.Full).OrderByDescending(d => d.CheckpointLsn)
-        .First();
+      var fullBackupsOrdered = backups
+        .Where(b => b.BackupType == BackupFileTools.BackupType.Full)
+        .OrderByDescending(d => d.CheckpointLsn).ToList();
+
+      if(!fullBackupsOrdered.Any()) {
+        throw new BackupChainException("Could not find any full backups");
+      }
+
+      var targetCheckpointLsn = fullBackupsOrdered.First().CheckpointLsn;
+      // get all the stripes of this backup
+      return fullBackupsOrdered.Where(fullBackup => fullBackup.CheckpointLsn == targetCheckpointLsn); 
     }
 
-    private BackupMetadata MostRecentDifferentialBackup(IList<BackupMetadata> backups, BackupMetadata lastFullBackup)
+    private static IEnumerable<BackupMetadata> MostRecentDiffBackup(IEnumerable<BackupMetadata> backups, BackupMetadata lastFullBackup)
     {
-      return backups.Where(b => b.BackupType == BackupFileTools.BackupType.Diff &&
-                                b.DatabaseBackupLsn == lastFullBackup.CheckpointLsn)
-        .OrderByDescending(b => b.LastLsn).FirstOrDefault();
+      var diffBackupsOrdered = backups
+        .Where(b => b.BackupType == BackupFileTools.BackupType.Diff &&
+                    b.DatabaseBackupLsn == lastFullBackup.CheckpointLsn)
+        .OrderByDescending(b => b.LastLsn).ToList();
+
+      if (!diffBackupsOrdered.Any()) {
+        return new List<BackupMetadata>();
+      }
+      var targetLastLsn = diffBackupsOrdered.First().LastLsn;
+      // get all the stripes of this backup
+      return diffBackupsOrdered.Where(diffBackup => diffBackup.LastLsn == targetLastLsn); 
     }
 
-    private BackupMetadata NextLogBackup(IList<BackupMetadata> backups, BackupMetadata prevBackup)
+    private static IEnumerable<BackupMetadata> NextLogBackup(IEnumerable<BackupMetadata> backups, BackupMetadata prevBackup)
     {
-      return backups.Where(b => b.BackupType == BackupFileTools.BackupType.Log)
-        .SingleOrDefault(d => prevBackup.LastLsn >= d.FirstLsn && prevBackup.LastLsn + 1 < d.LastLsn);
+      // also gets all the stripes of the next backup
+      return backups.Where(b => b.BackupType == BackupFileTools.BackupType.Log && 
+                                prevBackup.LastLsn >= b.FirstLsn && prevBackup.LastLsn + 1 < b.LastLsn);
     }
 
-    private bool IsValidFilePath(BackupMetadata meta)
+    private static bool IsValidFilePath(BackupMetadata meta)
     {
       if(BackupFileTools.IsUrl(meta.PhysicalDeviceName))
         return true;

src/BackupMetadata.cs

@@ -17,13 +17,15 @@ public bool Equals(BackupMetadata x, BackupMetadata y)
       return x.LastLsn == y.LastLsn &&
              x.FirstLsn == y.FirstLsn &&
              x.BackupType == y.BackupType &&
-             x.DatabaseName == y.DatabaseName;
+             x.DatabaseName == y.DatabaseName &&
+             x.PhysicalDeviceName == y.PhysicalDeviceName;
     }
 
     public int GetHashCode(BackupMetadata obj)
     {
       var hashCode = -1277603921;
       hashCode = hashCode * -1521134295 + EqualityComparer<string>.Default.GetHashCode(obj.DatabaseName);
+      hashCode = hashCode * -1521134295 + EqualityComparer<string>.Default.GetHashCode(obj.PhysicalDeviceName);
       hashCode = hashCode * -1521134295 +
                  EqualityComparer<BackupFileTools.BackupType>.Default.GetHashCode(obj.BackupType);
       hashCode = hashCode * -1521134295 + obj.FirstLsn.GetHashCode();
@@ -35,7 +37,7 @@ public int GetHashCode(BackupMetadata obj)
   /// <summary>
   ///   Metadata about backups from msdb.dbo.backupset and msdb.dbo.backupmediafamily
   /// </summary>
-  public class BackupMetadata
+  public class BackupMetadata : ICloneable
   {
     public decimal CheckpointLsn { get; set; }
     public decimal DatabaseBackupLsn { get; set; }
@@ -52,5 +54,11 @@ public class BackupMetadata
     ///   https://docs.microsoft.com/en-us/sql/relational-databases/system-tables/backupset-transact-sql?view=sql-server-2017
     /// </summary>
     public BackupFileTools.BackupType BackupType { get; set; }
+
+    // used during testing
+    public object Clone()
+    {
+      return MemberwiseClone();
+    }
   }
 }

tests/AgDatabaseMove.Unit/BackupOrder.cs

@@ -4,32 +4,48 @@ namespace AgDatabaseMove.Unit
   using System.Collections;
   using System.Collections.Generic;
   using System.Linq;
+  using Exceptions;
   using Moq;
   using SmoFacade;
   using Xunit;
 
 
   public class BackupOrder
   {
-    private readonly List<BackupMetadata> _listBackups;
 
-    public BackupOrder()
+    private static IEnumerable<BackupMetadata> CloneBackupMetaDataList(List<BackupMetadata> list)
     {
-      _listBackups = ListBackups();
+      var result = new List<BackupMetadata>();
+      list.ForEach(b => {
+        result.Add((BackupMetadata)b.Clone());
+      });
+      result.Reverse();
+      return result;
     }
 
-    private static List<BackupMetadata> ListBackups()
+    private static List<BackupMetadata> GetBackupList()
     {
       return new List<BackupMetadata> {
         new BackupMetadata {
           BackupType = BackupFileTools.BackupType.Log,
           DatabaseBackupLsn = 126000000943800037,
           CheckpointLsn = 126000000953600034,
-          FirstLsn = 126000000955500001,
-          LastLsn = 126000000955800001,
+          FirstLsn = 126000000955200001,
+          LastLsn = 126000000955500001,
+          DatabaseName = "TestDb",
+          ServerName = "ServerA",
+          PhysicalDeviceName = @"\\DFS\BACKUP\ServerA\testDb\Testdb_backup_2018_10_29_020007_343.trn",
+          StartTime = DateTime.Parse("2018-10-29 02:00:07.000")
+        },
+        new BackupMetadata {
+          BackupType = BackupFileTools.BackupType.Log,
+          DatabaseBackupLsn = 126000000943800037,
+          CheckpointLsn = 126000000953600034,
+          FirstLsn = 126000000955800001,
+          LastLsn = 126000000965800001,
           DatabaseName = "TestDb",
           ServerName = "ServerB",
-          PhysicalDeviceName = @"\\DFS\BACKUP\ServerB\testDb\Testdb_backup_2018_10_29_030006_660.trn",
+          PhysicalDeviceName = @"\\DFS\BACKUP\ServerB\testDb\Testdb_backup_2018_10_29_040005_900.trn",
           StartTime = DateTime.Parse("2018-10-29 03:00:06.000")
         },
         new BackupMetadata {
@@ -44,51 +60,155 @@ private static List<BackupMetadata> ListBackups()
           StartTime = DateTime.Parse("2018-10-28 00:02:28.000")
         },
         new BackupMetadata {
-          BackupType = BackupFileTools.BackupType.Diff,
+          BackupType = BackupFileTools.BackupType.Log,
           DatabaseBackupLsn = 126000000943800037,
           CheckpointLsn = 126000000953600034,
-          FirstLsn = 126000000943800037,
-          LastLsn = 126000000955200001,
+          FirstLsn = 126000000955500001,
+          LastLsn = 126000000955800001,
           DatabaseName = "TestDb",
-          ServerName = "ServerA",
-          PhysicalDeviceName = @"\\DFS\BACKUP\ServerA\testDb\Testdb_backup_2018_10_29_000339_780.diff",
-          StartTime = DateTime.Parse("2018-10-29 00:03:39.000")
+          ServerName = "ServerB",
+          PhysicalDeviceName = @"\\DFS\BACKUP\ServerB\testDb\Testdb_backup_2018_10_29_030006_660.trn",
+          StartTime = DateTime.Parse("2018-10-29 03:00:06.000")
         },
         new BackupMetadata {
-          BackupType = BackupFileTools.BackupType.Log,
-          DatabaseBackupLsn = 126000000882000037,
+          BackupType = BackupFileTools.BackupType.Diff,
+          DatabaseBackupLsn = 126000000943800037,
           CheckpointLsn = 126000000953600034,
-          FirstLsn = 126000000955200001,
-          LastLsn = 126000000955500001,
+          FirstLsn = 126000000945600000,
+          LastLsn = 126000000955200001,
           DatabaseName = "TestDb",
           ServerName = "ServerA",
-          PhysicalDeviceName = @"\\DFS\BACKUP\ServerA\testDb\Testdb_backup_2018_10_29_020007_343.trn",
-          StartTime = DateTime.Parse("2018-10-29 02:00:07.000")
+          PhysicalDeviceName = @"\\DFS\BACKUP\ServerA\testDb\Testdb_backup_2018_10_29_000339_780.diff",
+          StartTime = DateTime.Parse("2018-10-29 00:03:39.000")
         }
       };
     }
 
-    [Fact]
-    public void BackupChainOrdered()
+    private static List<BackupMetadata> GetBackupListWithoutLogs()
+    {
+      var list = GetBackupList();
+      list.RemoveAll(b => b.BackupType == BackupFileTools.BackupType.Log);
+      return list;
+    }
+
+    private static List<BackupMetadata> GetBackupListWithoutDiff()
+    {
+      var list = GetBackupList();
+      list.RemoveAll(b => b.BackupType == BackupFileTools.BackupType.Diff);
+      return list;
+    }
+
+    private static List<BackupMetadata> GetBackupListWithStripes()
+    {
+      var list = GetBackupList();
+      var listWithStripes = CloneBackupMetaDataList(list).ToList();
+      listWithStripes.ForEach(b => {
+        var path = b.PhysicalDeviceName.Split('.');
+        b.PhysicalDeviceName = $"{path[0]}_striped.{path[1]}";
+      });
+      list.AddRange(listWithStripes);
+      return list;
+    }
+
+    private static List<BackupMetadata> GetBackupListWithStripesAndDuplicates()
+    {
+      var listWithStripes = GetBackupListWithStripes();
+      var duplicate = CloneBackupMetaDataList(listWithStripes);
+      listWithStripes.AddRange(duplicate);
+      return listWithStripes;
+    }
+
+    private static List<BackupMetadata> GetBackupListWithoutFull()
+    {
+      var list = GetBackupList();
+      list.RemoveAll(b => b.BackupType == BackupFileTools.BackupType.Full);
+      return list;
+    }
+
+    private static void VerifyListIsAValidBackupChain(List<BackupMetadata> backupChain)
+    {
+      bool foundFull, foundDiff, foundLog;
+      foundFull = foundDiff = foundLog = false;
+      BackupMetadata full = null;
+      BackupMetadata lastBackup = null;
+
+      BackupMetadata currentBackup;
+      while((currentBackup = backupChain.FirstOrDefault()) != null) {
+
+        if(currentBackup.BackupType == BackupFileTools.BackupType.Full) {
+          Assert.True(!foundFull && !foundDiff && !foundLog);
+          foundFull = true;
+          full = currentBackup;
+        }
+        else if(currentBackup.BackupType == BackupFileTools.BackupType.Diff) {
+          Assert.True(foundFull && !foundDiff && !foundLog);
+          Assert.Equal(full.CheckpointLsn, currentBackup.DatabaseBackupLsn);
+          Assert.True(currentBackup.FirstLsn >= lastBackup.LastLsn);
+          foundDiff = true;
+        }
+        else if(currentBackup.BackupType == BackupFileTools.BackupType.Log) {
+          Assert.True(foundFull);
+          Assert.True(currentBackup.FirstLsn >= lastBackup.LastLsn);
+          foundLog = true;
+        }
+
+        lastBackup = currentBackup;
+        backupChain.RemoveAll(b => b.LastLsn == currentBackup.LastLsn);
+      }
+    }
+
+    public static IEnumerable<object[]> PositiveTestData => new List<object[]> {
+      new object[] { GetBackupList() },
+      new object[] { GetBackupListWithStripes() },
+      new object[] { GetBackupListWithoutDiff() },
+      new object[] { GetBackupListWithoutLogs() }
+    };
+
+    [Theory]
+    [MemberData(nameof(PositiveTestData))]
+    public void BackupChainIsCorrect(List<BackupMetadata> backupList)
     {
       var agDatabase = new Mock<IAgDatabase>();
-      agDatabase.Setup(agd => agd.RecentBackups()).Returns(_listBackups);
+      agDatabase.Setup(agd => agd.RecentBackups()).Returns(backupList);
       var backupChain = new BackupChain(agDatabase.Object);
+      VerifyListIsAValidBackupChain(backupChain.OrderedBackups.ToList());
+    }
+
 
-      var expected = _listBackups.OrderBy(bu => bu.FirstLsn);
+    public static IEnumerable<object[]> NegativeTestData => new List<object[]> {
+      new object[] { GetBackupListWithoutFull() },
+      new object[] { new List<BackupMetadata>() }
+    };
 
-      Assert.Equal<IEnumerable>(backupChain.OrderedBackups, expected);
+    [Theory]
+    [MemberData(nameof(NegativeTestData))]
+    public void CanDetectBackupChainIsWrong(List<BackupMetadata> backupList)
+    {
+      var agDatabase = new Mock<IAgDatabase>();
+      agDatabase.Setup(agd => agd.RecentBackups()).Returns(backupList);
+      Assert.Throws<BackupChainException>(() => new BackupChain(agDatabase.Object));
     }
 
     [Fact]
     public void MissingLink()
     {
-      var backups = ListBackups().Where(b => b.FirstLsn != 126000000955200001).ToList();
+      var backups = GetBackupList().Where(b => b.FirstLsn != 126000000955200001).ToList();
       var agDatabase = new Mock<IAgDatabase>();
       agDatabase.Setup(agd => agd.RecentBackups()).Returns(backups);
 
-      var chain = new BackupChain(agDatabase.Object).OrderedBackups;
-      Assert.NotEqual(chain.Last().LastLsn, ListBackups().Max(b => b.LastLsn));
+      var chain = new BackupChain(agDatabase.Object).OrderedBackups.ToList();
+      Assert.NotEqual(chain.Last().LastLsn, GetBackupList().Max(b => b.LastLsn));
+      VerifyListIsAValidBackupChain(chain);
+    }
+
+    [Fact]
+    public void DuplicateFiles()
+    {
+      var backups = GetBackupListWithStripesAndDuplicates();
+      var agDatabase = new Mock<IAgDatabase>();
+      agDatabase.Setup(agd => agd.RecentBackups()).Returns(backups);
+      var chain = new BackupChain(agDatabase.Object).OrderedBackups.ToList();
+      Assert.Equal(backups.GroupBy(b => b.PhysicalDeviceName).Count(), chain.Count);
     }
 
     // TODO: test skipping of logs if diff last LSN and log last LSN matches