diff --git a/src/BackupChain.cs b/src/BackupChain.cs index 95096ea..4d85f29 100755 --- a/src/BackupChain.cs +++ b/src/BackupChain.cs @@ -4,6 +4,7 @@ namespace AgDatabaseMove using System.Collections.Generic; using System.IO; using System.Linq; + using Exceptions; using SmoFacade; @@ -19,26 +20,28 @@ public class BackupChain : IBackupChain { private readonly IList _orderedBackups; + // This also handles any striped backups private BackupChain(IList recentBackups) { + if (recentBackups == null || recentBackups.Count == 0) { + throw new BackupChainException("There are no recent backups to form a chain"); + } + var backups = recentBackups.Distinct(new BackupMetadataEqualityComparer()) - .Where(b => IsValidFilePath(b)) // A third party application caused invalid path strings to be inserted into backupmediafamily + .Where(IsValidFilePath) // A third party application caused invalid path strings to be inserted into backupmediafamily .ToList(); - var mostRecentFullBackup = MostRecentFullBackup(recentBackups); - _orderedBackups = new List { mostRecentFullBackup }; - - var differentialBackup = MostRecentDifferentialBackup(backups, mostRecentFullBackup); - if(differentialBackup != null) - _orderedBackups.Add(differentialBackup); + var orderedBackups = MostRecentFullBackup(backups).ToList(); + orderedBackups.AddRange(MostRecentDiffBackup(backups, orderedBackups.First())); - var mostRecentBackup = _orderedBackups.Last(); - while(mostRecentBackup != null) { - mostRecentBackup = NextLogBackup(backups, mostRecentBackup); - - if(mostRecentBackup != null) - _orderedBackups.Add(mostRecentBackup); + var prevBackup = orderedBackups.Last(); + IEnumerable nextLogBackups; + while((nextLogBackups = NextLogBackup(backups, prevBackup)).Any()) { + orderedBackups.AddRange(nextLogBackups); + prevBackup = orderedBackups.Last(); } + + _orderedBackups = orderedBackups; } /// @@ -56,26 +59,44 @@ public BackupChain(Database database) : this(database.RecentBackups()) { } /// public IEnumerable OrderedBackups => _orderedBackups; - private BackupMetadata MostRecentFullBackup(IList backups) + private static IEnumerable MostRecentFullBackup(IEnumerable backups) { - return backups.Where(b => b.BackupType == BackupFileTools.BackupType.Full).OrderByDescending(d => d.CheckpointLsn) - .First(); + var fullBackupsOrdered = backups + .Where(b => b.BackupType == BackupFileTools.BackupType.Full) + .OrderByDescending(d => d.CheckpointLsn).ToList(); + + if(!fullBackupsOrdered.Any()) { + throw new BackupChainException("Could not find any full backups"); + } + + var targetCheckpointLsn = fullBackupsOrdered.First().CheckpointLsn; + // get all the stripes of this backup + return fullBackupsOrdered.Where(fullBackup => fullBackup.CheckpointLsn == targetCheckpointLsn); } - private BackupMetadata MostRecentDifferentialBackup(IList backups, BackupMetadata lastFullBackup) + private static IEnumerable MostRecentDiffBackup(IEnumerable backups, BackupMetadata lastFullBackup) { - return backups.Where(b => b.BackupType == BackupFileTools.BackupType.Diff && - b.DatabaseBackupLsn == lastFullBackup.CheckpointLsn) - .OrderByDescending(b => b.LastLsn).FirstOrDefault(); + var diffBackupsOrdered = backups + .Where(b => b.BackupType == BackupFileTools.BackupType.Diff && + b.DatabaseBackupLsn == lastFullBackup.CheckpointLsn) + .OrderByDescending(b => b.LastLsn).ToList(); + + if (!diffBackupsOrdered.Any()) { + return new List(); + } + var targetLastLsn = diffBackupsOrdered.First().LastLsn; + // get all the stripes of this backup + return diffBackupsOrdered.Where(diffBackup => diffBackup.LastLsn == targetLastLsn); } - private BackupMetadata NextLogBackup(IList backups, BackupMetadata prevBackup) + private static IEnumerable NextLogBackup(IEnumerable backups, BackupMetadata prevBackup) { - return backups.Where(b => b.BackupType == BackupFileTools.BackupType.Log) - .SingleOrDefault(d => prevBackup.LastLsn >= d.FirstLsn && prevBackup.LastLsn + 1 < d.LastLsn); + // also gets all the stripes of the next backup + return backups.Where(b => b.BackupType == BackupFileTools.BackupType.Log && + prevBackup.LastLsn >= b.FirstLsn && prevBackup.LastLsn + 1 < b.LastLsn); } - private bool IsValidFilePath(BackupMetadata meta) + private static bool IsValidFilePath(BackupMetadata meta) { if(BackupFileTools.IsUrl(meta.PhysicalDeviceName)) return true; diff --git a/src/BackupMetadata.cs b/src/BackupMetadata.cs index 705c1cd..5253f57 100755 --- a/src/BackupMetadata.cs +++ b/src/BackupMetadata.cs @@ -17,13 +17,15 @@ public bool Equals(BackupMetadata x, BackupMetadata y) return x.LastLsn == y.LastLsn && x.FirstLsn == y.FirstLsn && x.BackupType == y.BackupType && - x.DatabaseName == y.DatabaseName; + x.DatabaseName == y.DatabaseName && + x.PhysicalDeviceName == y.PhysicalDeviceName; } public int GetHashCode(BackupMetadata obj) { var hashCode = -1277603921; hashCode = hashCode * -1521134295 + EqualityComparer.Default.GetHashCode(obj.DatabaseName); + hashCode = hashCode * -1521134295 + EqualityComparer.Default.GetHashCode(obj.PhysicalDeviceName); hashCode = hashCode * -1521134295 + EqualityComparer.Default.GetHashCode(obj.BackupType); hashCode = hashCode * -1521134295 + obj.FirstLsn.GetHashCode(); @@ -35,7 +37,7 @@ public int GetHashCode(BackupMetadata obj) /// /// Metadata about backups from msdb.dbo.backupset and msdb.dbo.backupmediafamily /// - public class BackupMetadata + public class BackupMetadata : ICloneable { public decimal CheckpointLsn { get; set; } public decimal DatabaseBackupLsn { get; set; } @@ -52,5 +54,11 @@ public class BackupMetadata /// https://docs.microsoft.com/en-us/sql/relational-databases/system-tables/backupset-transact-sql?view=sql-server-2017 /// public BackupFileTools.BackupType BackupType { get; set; } + + // used during testing + public object Clone() + { + return MemberwiseClone(); + } } } \ No newline at end of file diff --git a/tests/AgDatabaseMove.Unit/BackupOrder.cs b/tests/AgDatabaseMove.Unit/BackupOrder.cs index b97e321..9f8f860 100755 --- a/tests/AgDatabaseMove.Unit/BackupOrder.cs +++ b/tests/AgDatabaseMove.Unit/BackupOrder.cs @@ -4,6 +4,7 @@ namespace AgDatabaseMove.Unit using System.Collections; using System.Collections.Generic; using System.Linq; + using Exceptions; using Moq; using SmoFacade; using Xunit; @@ -11,25 +12,40 @@ namespace AgDatabaseMove.Unit public class BackupOrder { - private readonly List _listBackups; - public BackupOrder() + private static IEnumerable CloneBackupMetaDataList(List list) { - _listBackups = ListBackups(); + var result = new List(); + list.ForEach(b => { + result.Add((BackupMetadata)b.Clone()); + }); + result.Reverse(); + return result; } - private static List ListBackups() + private static List GetBackupList() { return new List { new BackupMetadata { BackupType = BackupFileTools.BackupType.Log, DatabaseBackupLsn = 126000000943800037, CheckpointLsn = 126000000953600034, - FirstLsn = 126000000955500001, - LastLsn = 126000000955800001, + FirstLsn = 126000000955200001, + LastLsn = 126000000955500001, + DatabaseName = "TestDb", + ServerName = "ServerA", + PhysicalDeviceName = @"\\DFS\BACKUP\ServerA\testDb\Testdb_backup_2018_10_29_020007_343.trn", + StartTime = DateTime.Parse("2018-10-29 02:00:07.000") + }, + new BackupMetadata { + BackupType = BackupFileTools.BackupType.Log, + DatabaseBackupLsn = 126000000943800037, + CheckpointLsn = 126000000953600034, + FirstLsn = 126000000955800001, + LastLsn = 126000000965800001, DatabaseName = "TestDb", ServerName = "ServerB", - PhysicalDeviceName = @"\\DFS\BACKUP\ServerB\testDb\Testdb_backup_2018_10_29_030006_660.trn", + PhysicalDeviceName = @"\\DFS\BACKUP\ServerB\testDb\Testdb_backup_2018_10_29_040005_900.trn", StartTime = DateTime.Parse("2018-10-29 03:00:06.000") }, new BackupMetadata { @@ -44,51 +60,155 @@ private static List ListBackups() StartTime = DateTime.Parse("2018-10-28 00:02:28.000") }, new BackupMetadata { - BackupType = BackupFileTools.BackupType.Diff, + BackupType = BackupFileTools.BackupType.Log, DatabaseBackupLsn = 126000000943800037, CheckpointLsn = 126000000953600034, - FirstLsn = 126000000943800037, - LastLsn = 126000000955200001, + FirstLsn = 126000000955500001, + LastLsn = 126000000955800001, DatabaseName = "TestDb", - ServerName = "ServerA", - PhysicalDeviceName = @"\\DFS\BACKUP\ServerA\testDb\Testdb_backup_2018_10_29_000339_780.diff", - StartTime = DateTime.Parse("2018-10-29 00:03:39.000") + ServerName = "ServerB", + PhysicalDeviceName = @"\\DFS\BACKUP\ServerB\testDb\Testdb_backup_2018_10_29_030006_660.trn", + StartTime = DateTime.Parse("2018-10-29 03:00:06.000") }, new BackupMetadata { - BackupType = BackupFileTools.BackupType.Log, - DatabaseBackupLsn = 126000000882000037, + BackupType = BackupFileTools.BackupType.Diff, + DatabaseBackupLsn = 126000000943800037, CheckpointLsn = 126000000953600034, - FirstLsn = 126000000955200001, - LastLsn = 126000000955500001, + FirstLsn = 126000000945600000, + LastLsn = 126000000955200001, DatabaseName = "TestDb", ServerName = "ServerA", - PhysicalDeviceName = @"\\DFS\BACKUP\ServerA\testDb\Testdb_backup_2018_10_29_020007_343.trn", - StartTime = DateTime.Parse("2018-10-29 02:00:07.000") + PhysicalDeviceName = @"\\DFS\BACKUP\ServerA\testDb\Testdb_backup_2018_10_29_000339_780.diff", + StartTime = DateTime.Parse("2018-10-29 00:03:39.000") } }; } - [Fact] - public void BackupChainOrdered() + private static List GetBackupListWithoutLogs() + { + var list = GetBackupList(); + list.RemoveAll(b => b.BackupType == BackupFileTools.BackupType.Log); + return list; + } + + private static List GetBackupListWithoutDiff() + { + var list = GetBackupList(); + list.RemoveAll(b => b.BackupType == BackupFileTools.BackupType.Diff); + return list; + } + + private static List GetBackupListWithStripes() + { + var list = GetBackupList(); + var listWithStripes = CloneBackupMetaDataList(list).ToList(); + listWithStripes.ForEach(b => { + var path = b.PhysicalDeviceName.Split('.'); + b.PhysicalDeviceName = $"{path[0]}_striped.{path[1]}"; + }); + list.AddRange(listWithStripes); + return list; + } + + private static List GetBackupListWithStripesAndDuplicates() + { + var listWithStripes = GetBackupListWithStripes(); + var duplicate = CloneBackupMetaDataList(listWithStripes); + listWithStripes.AddRange(duplicate); + return listWithStripes; + } + + private static List GetBackupListWithoutFull() + { + var list = GetBackupList(); + list.RemoveAll(b => b.BackupType == BackupFileTools.BackupType.Full); + return list; + } + + private static void VerifyListIsAValidBackupChain(List backupChain) + { + bool foundFull, foundDiff, foundLog; + foundFull = foundDiff = foundLog = false; + BackupMetadata full = null; + BackupMetadata lastBackup = null; + + BackupMetadata currentBackup; + while((currentBackup = backupChain.FirstOrDefault()) != null) { + + if(currentBackup.BackupType == BackupFileTools.BackupType.Full) { + Assert.True(!foundFull && !foundDiff && !foundLog); + foundFull = true; + full = currentBackup; + } + else if(currentBackup.BackupType == BackupFileTools.BackupType.Diff) { + Assert.True(foundFull && !foundDiff && !foundLog); + Assert.Equal(full.CheckpointLsn, currentBackup.DatabaseBackupLsn); + Assert.True(currentBackup.FirstLsn >= lastBackup.LastLsn); + foundDiff = true; + } + else if(currentBackup.BackupType == BackupFileTools.BackupType.Log) { + Assert.True(foundFull); + Assert.True(currentBackup.FirstLsn >= lastBackup.LastLsn); + foundLog = true; + } + + lastBackup = currentBackup; + backupChain.RemoveAll(b => b.LastLsn == currentBackup.LastLsn); + } + } + + public static IEnumerable PositiveTestData => new List { + new object[] { GetBackupList() }, + new object[] { GetBackupListWithStripes() }, + new object[] { GetBackupListWithoutDiff() }, + new object[] { GetBackupListWithoutLogs() } + }; + + [Theory] + [MemberData(nameof(PositiveTestData))] + public void BackupChainIsCorrect(List backupList) { var agDatabase = new Mock(); - agDatabase.Setup(agd => agd.RecentBackups()).Returns(_listBackups); + agDatabase.Setup(agd => agd.RecentBackups()).Returns(backupList); var backupChain = new BackupChain(agDatabase.Object); + VerifyListIsAValidBackupChain(backupChain.OrderedBackups.ToList()); + } + - var expected = _listBackups.OrderBy(bu => bu.FirstLsn); + public static IEnumerable NegativeTestData => new List { + new object[] { GetBackupListWithoutFull() }, + new object[] { new List() } + }; - Assert.Equal(backupChain.OrderedBackups, expected); + [Theory] + [MemberData(nameof(NegativeTestData))] + public void CanDetectBackupChainIsWrong(List backupList) + { + var agDatabase = new Mock(); + agDatabase.Setup(agd => agd.RecentBackups()).Returns(backupList); + Assert.Throws(() => new BackupChain(agDatabase.Object)); } [Fact] public void MissingLink() { - var backups = ListBackups().Where(b => b.FirstLsn != 126000000955200001).ToList(); + var backups = GetBackupList().Where(b => b.FirstLsn != 126000000955200001).ToList(); var agDatabase = new Mock(); agDatabase.Setup(agd => agd.RecentBackups()).Returns(backups); - var chain = new BackupChain(agDatabase.Object).OrderedBackups; - Assert.NotEqual(chain.Last().LastLsn, ListBackups().Max(b => b.LastLsn)); + var chain = new BackupChain(agDatabase.Object).OrderedBackups.ToList(); + Assert.NotEqual(chain.Last().LastLsn, GetBackupList().Max(b => b.LastLsn)); + VerifyListIsAValidBackupChain(chain); + } + + [Fact] + public void DuplicateFiles() + { + var backups = GetBackupListWithStripesAndDuplicates(); + var agDatabase = new Mock(); + agDatabase.Setup(agd => agd.RecentBackups()).Returns(backups); + var chain = new BackupChain(agDatabase.Object).OrderedBackups.ToList(); + Assert.Equal(backups.GroupBy(b => b.PhysicalDeviceName).Count(), chain.Count); } // TODO: test skipping of logs if diff last LSN and log last LSN matches