diff --git a/GVFS/GVFS.Common/FileBasedCollection.cs b/GVFS/GVFS.Common/FileBasedCollection.cs index 3600fbbfa..325bf10d4 100644 --- a/GVFS/GVFS.Common/FileBasedCollection.cs +++ b/GVFS/GVFS.Common/FileBasedCollection.cs @@ -15,6 +15,9 @@ public abstract class FileBasedCollection : IDisposable private const string AddEntryPrefix = "A "; private const string RemoveEntryPrefix = "D "; + + // Use the same newline separator regardless of platform + private const string NewLine = "\r\n"; private const int IoFailureRetryDelayMS = 50; private const int IoFailureLoggingThreshold = 500; @@ -390,7 +393,7 @@ private void WriteToDisk(string value) throw new InvalidOperationException(nameof(this.WriteToDisk) + " requires that collectionAppendsDirectlyToFile be true"); } - byte[] bytes = Encoding.UTF8.GetBytes(value + "\r\n"); + byte[] bytes = Encoding.UTF8.GetBytes(value + NewLine); lock (this.fileLock) { this.dataFileHandle.Write(bytes, 0, bytes.Length); @@ -399,7 +402,7 @@ private void WriteToDisk(string value) } /// - /// Reads entries from dataFileHandle, removing any data after the last \r\n. Requires fileLock. + /// Reads entries from dataFileHandle, removing any data after the last NewLine ("\r\n"). Requires fileLock. /// private void RemoveLastEntryIfInvalid() { @@ -442,7 +445,7 @@ private bool TryWriteTempFile(Func> getDataLines, out Except { foreach (string line in getDataLines()) { - writer.Write(line + "\r\n"); + writer.Write(line + NewLine); } tempFile.Flush(); diff --git a/GVFS/GVFS.Common/FileSystem/IKernelDriver.cs b/GVFS/GVFS.Common/FileSystem/IKernelDriver.cs index f10aba1f7..270019da4 100644 --- a/GVFS/GVFS.Common/FileSystem/IKernelDriver.cs +++ b/GVFS/GVFS.Common/FileSystem/IKernelDriver.cs @@ -4,8 +4,8 @@ namespace GVFS.Common.FileSystem { public interface IKernelDriver { + bool EnumerationExpandsDirectories { get; } string DriverLogFolderName { get; } - bool IsSupported(string normalizedEnlistmentRootPath, out string warning, out string error); string FlushDriverLogs(); bool TryPrepareFolderForCallbacks(string folderPath, out string error); diff --git a/GVFS/GVFS.Common/PlaceholderListDatabase.cs b/GVFS/GVFS.Common/PlaceholderListDatabase.cs index 48971b239..5531fd0c0 100644 --- a/GVFS/GVFS.Common/PlaceholderListDatabase.cs +++ b/GVFS/GVFS.Common/PlaceholderListDatabase.cs @@ -8,6 +8,12 @@ namespace GVFS.Common { public class PlaceholderListDatabase : FileBasedCollection { + // Special folder values must: + // - Be 40 characters long + // - Not be a valid SHA-1 value (to avoid collisions with files) + public const string PartialFolderValue = " PARTIAL FOLDER"; + public const string ExpandedFolderValue = " EXPANDED FOLDER"; + private const char PathTerminator = '\0'; // This list holds entries that would otherwise be lost because WriteAllEntriesAndFlush has not been called, but a file @@ -51,19 +57,29 @@ public static bool TryCreate(ITracer tracer, string dataFilePath, PhysicalFileSy output = temp; return true; } - - public void AddAndFlush(string path, string sha) + + public void AddAndFlushFile(string path, string sha) + { + this.AddAndFlush(path, sha); + } + + public void AddAndFlushFolder(string path, bool isExpanded) + { + this.AddAndFlush(path, isExpanded ? ExpandedFolderValue : PartialFolderValue); + } + + public void RemoveAndFlush(string path) { try { - this.WriteAddEntry( - path + PathTerminator + sha, + this.WriteRemoveEntry( + path, () => { - this.EstimatedCount++; + this.EstimatedCount--; if (this.placeholderDataEntries != null) { - this.placeholderDataEntries.Add(new PlaceholderDataEntry(path, sha)); + this.placeholderDataEntries.Add(new PlaceholderDataEntry(path)); } }); } @@ -73,20 +89,32 @@ public void AddAndFlush(string path, string sha) } } - public void RemoveAndFlush(string path) + public List GetAllEntries() { try { - this.WriteRemoveEntry( - path, + List placeholders = new List(Math.Max(1, this.EstimatedCount)); + + string error; + if (!this.TryLoadFromDisk( + this.TryParseAddLine, + this.TryParseRemoveLine, + (key, value) => placeholders.Add(new PlaceholderData(path: key, fileShaOrFolderValue: value)), + out error, () => { - this.EstimatedCount--; if (this.placeholderDataEntries != null) { - this.placeholderDataEntries.Add(new PlaceholderDataEntry(path)); + throw new InvalidOperationException("PlaceholderListDatabase should always flush queue placeholders using WriteAllEntriesAndFlush before calling GetAllEntries again."); } - }); + + this.placeholderDataEntries = new List(); + })) + { + throw new InvalidDataException(error); + } + + return placeholders; } catch (Exception e) { @@ -94,17 +122,28 @@ public void RemoveAndFlush(string path) } } - public List GetAllEntries() + public void GetAllEntries(out List filePlaceholders, out List folderPlaceholders) { try { - List output = new List(Math.Max(1, this.EstimatedCount)); + List filePlaceholdersFromDisk = new List(Math.Max(1, this.EstimatedCount)); + List folderPlaceholdersFromDisk = new List(Math.Max(1, (int)(this.EstimatedCount * .3))); string error; if (!this.TryLoadFromDisk( this.TryParseAddLine, this.TryParseRemoveLine, - (key, value) => output.Add(new PlaceholderData(path: key, sha: value)), + (key, value) => + { + if (value == PartialFolderValue || value == ExpandedFolderValue) + { + folderPlaceholdersFromDisk.Add(new PlaceholderData(path: key, fileShaOrFolderValue: value)); + } + else + { + filePlaceholdersFromDisk.Add(new PlaceholderData(path: key, fileShaOrFolderValue: value)); + } + }, out error, () => { @@ -119,7 +158,8 @@ public List GetAllEntries() throw new InvalidDataException(error); } - return output; + filePlaceholders = filePlaceholdersFromDisk; + folderPlaceholders = folderPlaceholdersFromDisk; } catch (Exception e) { @@ -181,6 +221,27 @@ private IEnumerable GenerateDataLines(IEnumerable updat } } + private void AddAndFlush(string path, string sha) + { + try + { + this.WriteAddEntry( + path + PathTerminator + sha, + () => + { + this.EstimatedCount++; + if (this.placeholderDataEntries != null) + { + this.placeholderDataEntries.Add(new PlaceholderDataEntry(path, sha)); + } + }); + } + catch (Exception e) + { + throw new FileBasedCollectionException(e); + } + } + private bool TryParseAddLine(string line, out string key, out string value, out string error) { // Expected: \0<40-Char-SHA1> @@ -197,7 +258,7 @@ private bool TryParseAddLine(string line, out string key, out string value, out { key = null; value = null; - error = "Invalid SHA1 length: " + line; + error = $"Invalid SHA1 length {line.Length - idx - 1}: " + line; return false; } @@ -218,10 +279,10 @@ private bool TryParseRemoveLine(string line, out string key, out string error) public class PlaceholderData { - public PlaceholderData(string path, string sha) + public PlaceholderData(string path, string fileShaOrFolderValue) { this.Path = path; - this.Sha = sha; + this.Sha = fileShaOrFolderValue; } public string Path { get; } @@ -229,7 +290,18 @@ public PlaceholderData(string path, string sha) public bool IsFolder { - get { return this.Sha == GVFSConstants.AllZeroSha; } + get + { + return this.Sha == PartialFolderValue || this.IsExpandedFolder; + } + } + + public bool IsExpandedFolder + { + get + { + return this.Sha == ExpandedFolderValue; + } } } diff --git a/GVFS/GVFS.Common/RepoMetadata.cs b/GVFS/GVFS.Common/RepoMetadata.cs index db28c223d..eeefb78dc 100644 --- a/GVFS/GVFS.Common/RepoMetadata.cs +++ b/GVFS/GVFS.Common/RepoMetadata.cs @@ -316,7 +316,7 @@ public static class DiskLayoutVersion // The major version should be bumped whenever there is an on-disk format change that requires a one-way upgrade. // Increasing this version will make older versions of GVFS unable to mount a repo that has been mounted by a newer // version of GVFS. - public const int CurrentMajorVersion = 16; + public const int CurrentMajorVersion = 17; // The minor version should be bumped whenever there is an upgrade that can be safely ignored by older versions of GVFS. // For example, this allows an upgrade step that sets a default value for some new config setting. diff --git a/GVFS/GVFS.FunctionalTests.Windows/Windows/Tests/DiskLayoutUpgradeTests.cs b/GVFS/GVFS.FunctionalTests.Windows/Windows/Tests/DiskLayoutUpgradeTests.cs index 6495a9cd6..1abbbaafb 100644 --- a/GVFS/GVFS.FunctionalTests.Windows/Windows/Tests/DiskLayoutUpgradeTests.cs +++ b/GVFS/GVFS.FunctionalTests.Windows/Windows/Tests/DiskLayoutUpgradeTests.cs @@ -17,7 +17,7 @@ namespace GVFS.FunctionalTests.Windows.Tests [Category(Categories.WindowsOnly)] public class DiskLayoutUpgradeTests : TestsWithEnlistmentPerTestCase { - public const int CurrentDiskLayoutMajorVersion = 16; + public const int CurrentDiskLayoutMajorVersion = 17; public const int CurrentDiskLayoutMinorVersion = 0; public const string BlobSizesCacheName = "blobSizes"; @@ -153,34 +153,34 @@ public void MountSucceedsIfMinorVersionHasAdvancedButNotMajorVersion() [TestCase] public void MountWritesFolderPlaceholdersToPlaceholderDatabase() { - // Create some placeholder data - this.fileSystem.ReadAllText(Path.Combine(this.Enlistment.RepoRoot, "Readme.md")); - this.fileSystem.ReadAllText(Path.Combine(this.Enlistment.RepoRoot, "Scripts\\RunUnitTests.bat")); - this.fileSystem.ReadAllText(Path.Combine(this.Enlistment.RepoRoot, "GVFS\\GVFS.Common\\Git\\GitRefs.cs")); + this.PerformIOBeforePlaceholderDatabaseUpgradeTest(); - // Create a full folder - this.fileSystem.CreateDirectory(Path.Combine(this.Enlistment.RepoRoot, "GVFS\\FullFolder")); - this.fileSystem.WriteAllText(Path.Combine(this.Enlistment.RepoRoot, "GVFS\\FullFolder\\test.txt"), "Test contents"); + this.Enlistment.UnmountGVFS(); - // Create a tombstone - this.fileSystem.DeleteDirectory(Path.Combine(this.Enlistment.RepoRoot, "GVFS\\GVFS.Tests\\Properties")); + // Delete the existing folder placeholder data + string placeholderDatabasePath = Path.Combine(this.Enlistment.DotGVFSRoot, GVFSHelpers.PlaceholderListFile); + string[] lines = this.GetPlaceholderDatabaseLinesBeforeUpgrade(placeholderDatabasePath); - string junctionTarget = Path.Combine(this.Enlistment.EnlistmentRoot, "DirJunction"); - string symlinkTarget = Path.Combine(this.Enlistment.EnlistmentRoot, "DirSymlink"); - Directory.CreateDirectory(junctionTarget); - Directory.CreateDirectory(symlinkTarget); + // Placeholder database file should only have file placeholders + this.fileSystem.WriteAllText( + placeholderDatabasePath, + string.Join(Environment.NewLine, lines.Where(x => !x.EndsWith(TestConstants.PartialFolderPlaceholderDatabaseValue))) + Environment.NewLine); - string junctionLink = Path.Combine(this.Enlistment.RepoRoot, "DirJunction"); - string symlink = Path.Combine(this.Enlistment.RepoRoot, "DirLink"); - ProcessHelper.Run("CMD.exe", "/C mklink /J " + junctionLink + " " + junctionTarget); - ProcessHelper.Run("CMD.exe", "/C mklink /D " + symlink + " " + symlinkTarget); + GVFSHelpers.SaveDiskLayoutVersion(this.Enlistment.DotGVFSRoot, "12", "1"); - string target = Path.Combine(this.Enlistment.EnlistmentRoot, "GVFS", "GVFS", "GVFS.UnitTests"); - string link = Path.Combine(this.Enlistment.RepoRoot, "UnitTests"); - ProcessHelper.Run("CMD.exe", "/C mklink /J " + link + " " + target); - target = Path.Combine(this.Enlistment.EnlistmentRoot, "GVFS", "GVFS", "GVFS.Installer"); - link = Path.Combine(this.Enlistment.RepoRoot, "Installer"); - ProcessHelper.Run("CMD.exe", "/C mklink /D " + link + " " + target); + this.Enlistment.MountGVFS(); + this.Enlistment.UnmountGVFS(); + + // Validate the folder placeholders are in the placeholder database now + this.GetPlaceholderDatabaseLinesAfterUpgradeFrom12_1(placeholderDatabasePath); + + this.ValidatePersistedVersionMatchesCurrentVersion(); + } + + [TestCase] + public void MountUpdatesAllZeroShaFolderPlaceholderEntriesToPartialFolderSpecialValue() + { + this.PerformIOBeforePlaceholderDatabaseUpgradeTest(); this.Enlistment.UnmountGVFS(); @@ -188,16 +188,22 @@ public void MountWritesFolderPlaceholdersToPlaceholderDatabase() string placeholderDatabasePath = Path.Combine(this.Enlistment.DotGVFSRoot, GVFSHelpers.PlaceholderListFile); string[] lines = this.GetPlaceholderDatabaseLinesBeforeUpgrade(placeholderDatabasePath); - // Placeholder database file should only have file placeholders - this.fileSystem.WriteAllText(placeholderDatabasePath, string.Join(Environment.NewLine, lines.Where(x => !x.EndsWith(TestConstants.AllZeroSha))) + Environment.NewLine); + // Update the placeholder file so that folders have an all zero SHA + this.fileSystem.WriteAllText( + placeholderDatabasePath, + string.Join( + Environment.NewLine, + lines.Select(x => x.Replace(TestConstants.PartialFolderPlaceholderDatabaseValue, TestConstants.AllZeroSha))) + Environment.NewLine); - GVFSHelpers.SaveDiskLayoutVersion(this.Enlistment.DotGVFSRoot, "12", "1"); + GVFSHelpers.SaveDiskLayoutVersion(this.Enlistment.DotGVFSRoot, "16", "0"); this.Enlistment.MountGVFS(); this.Enlistment.UnmountGVFS(); - // Validate the folder placeholders are in the placeholder database now - this.GetPlaceholderDatabaseLinesAfterUpgrade(placeholderDatabasePath); + // Validate the folder placeholders in the database have PartialFolderPlaceholderDatabaseValue values + this.GetPlaceholderDatabaseLinesAfterUpgradeFrom16(placeholderDatabasePath); + + this.ValidatePersistedVersionMatchesCurrentVersion(); } [TestCase] @@ -232,17 +238,7 @@ public void MountUpgradesPreSharedCacheLocalSizes() this.Enlistment.MountGVFS(); - string majorVersion; - string minorVersion; - GVFSHelpers.GetPersistedDiskLayoutVersion(this.Enlistment.DotGVFSRoot, out majorVersion, out minorVersion); - - majorVersion - .ShouldBeAnInt("Disk layout version should always be an int") - .ShouldEqual(CurrentDiskLayoutMajorVersion, "Disk layout version should be upgraded to the latest"); - - minorVersion - .ShouldBeAnInt("Disk layout version should always be an int") - .ShouldEqual(CurrentDiskLayoutMinorVersion, "Disk layout version should be upgraded to the latest"); + this.ValidatePersistedVersionMatchesCurrentVersion(); GVFSHelpers.GetPersistedLocalCacheRoot(this.Enlistment.DotGVFSRoot) .ShouldEqual(string.Empty, "LocalCacheRoot should be an empty string when upgrading from a version prior to 12"); @@ -335,6 +331,53 @@ A tools/perllib/MS/Somefile.txt "; modifiedPathsDatabasePath.ShouldBeAFile(this.fileSystem).WithContents(expectedModifiedPaths); + + this.ValidatePersistedVersionMatchesCurrentVersion(); + } + + private void PerformIOBeforePlaceholderDatabaseUpgradeTest() + { + // Create some placeholder data + this.fileSystem.ReadAllText(Path.Combine(this.Enlistment.RepoRoot, "Readme.md")); + this.fileSystem.ReadAllText(Path.Combine(this.Enlistment.RepoRoot, "Scripts\\RunUnitTests.bat")); + this.fileSystem.ReadAllText(Path.Combine(this.Enlistment.RepoRoot, "GVFS\\GVFS.Common\\Git\\GitRefs.cs")); + + // Create a full folder + this.fileSystem.CreateDirectory(Path.Combine(this.Enlistment.RepoRoot, "GVFS\\FullFolder")); + this.fileSystem.WriteAllText(Path.Combine(this.Enlistment.RepoRoot, "GVFS\\FullFolder\\test.txt"), "Test contents"); + + // Create a tombstone + this.fileSystem.DeleteDirectory(Path.Combine(this.Enlistment.RepoRoot, "GVFS\\GVFS.Tests\\Properties")); + + string junctionTarget = Path.Combine(this.Enlistment.EnlistmentRoot, "DirJunction"); + string symlinkTarget = Path.Combine(this.Enlistment.EnlistmentRoot, "DirSymlink"); + Directory.CreateDirectory(junctionTarget); + Directory.CreateDirectory(symlinkTarget); + + string junctionLink = Path.Combine(this.Enlistment.RepoRoot, "DirJunction"); + string symlink = Path.Combine(this.Enlistment.RepoRoot, "DirLink"); + ProcessHelper.Run("CMD.exe", "/C mklink /J " + junctionLink + " " + junctionTarget); + ProcessHelper.Run("CMD.exe", "/C mklink /D " + symlink + " " + symlinkTarget); + + string target = Path.Combine(this.Enlistment.EnlistmentRoot, "GVFS", "GVFS", "GVFS.UnitTests"); + string link = Path.Combine(this.Enlistment.RepoRoot, "UnitTests"); + ProcessHelper.Run("CMD.exe", "/C mklink /J " + link + " " + target); + target = Path.Combine(this.Enlistment.EnlistmentRoot, "GVFS", "GVFS", "GVFS.Installer"); + link = Path.Combine(this.Enlistment.RepoRoot, "Installer"); + ProcessHelper.Run("CMD.exe", "/C mklink /D " + link + " " + target); + } + + private void PlaceholderDatabaseShouldIncludeCommonLinesForUpgradeTestIO(string[] placeholderLines) + { + placeholderLines.ShouldContain(x => x.Contains("A Readme.md")); + placeholderLines.ShouldContain(x => x.Contains("A Scripts\\RunUnitTests.bat")); + placeholderLines.ShouldContain(x => x.Contains("A GVFS\\GVFS.Common\\Git\\GitRefs.cs")); + placeholderLines.ShouldContain(x => x.Contains("A .gitignore")); + placeholderLines.ShouldContain(x => x == "A Scripts\0" + TestConstants.PartialFolderPlaceholderDatabaseValue); + placeholderLines.ShouldContain(x => x == "A GVFS\0" + TestConstants.PartialFolderPlaceholderDatabaseValue); + placeholderLines.ShouldContain(x => x == "A GVFS\\GVFS.Common\0" + TestConstants.PartialFolderPlaceholderDatabaseValue); + placeholderLines.ShouldContain(x => x == "A GVFS\\GVFS.Common\\Git\0" + TestConstants.PartialFolderPlaceholderDatabaseValue); + placeholderLines.ShouldContain(x => x == "A GVFS\\GVFS.Tests\0" + TestConstants.PartialFolderPlaceholderDatabaseValue); } private string[] GetPlaceholderDatabaseLinesBeforeUpgrade(string placeholderDatabasePath) @@ -342,35 +385,29 @@ private string[] GetPlaceholderDatabaseLinesBeforeUpgrade(string placeholderData placeholderDatabasePath.ShouldBeAFile(this.fileSystem); string[] lines = this.fileSystem.ReadAllText(placeholderDatabasePath).Split(new char[] { '\r', '\n' }, StringSplitOptions.RemoveEmptyEntries); lines.Length.ShouldEqual(12); - lines.ShouldContain(x => x.Contains("Readme.md")); - lines.ShouldContain(x => x.Contains("Scripts\\RunUnitTests.bat")); - lines.ShouldContain(x => x.Contains("GVFS\\GVFS.Common\\Git\\GitRefs.cs")); + this.PlaceholderDatabaseShouldIncludeCommonLinesForUpgradeTestIO(lines); lines.ShouldContain(x => x.Contains("A GVFS\\GVFS.Tests\\Properties\\AssemblyInfo.cs")); - lines.ShouldContain(x => x.Contains("A .gitignore")); lines.ShouldContain(x => x == "D GVFS\\GVFS.Tests\\Properties\\AssemblyInfo.cs"); - lines.ShouldContain(x => x == "A Scripts\0" + TestConstants.AllZeroSha); - lines.ShouldContain(x => x == "A GVFS\0" + TestConstants.AllZeroSha); - lines.ShouldContain(x => x == "A GVFS\\GVFS.Common\0" + TestConstants.AllZeroSha); - lines.ShouldContain(x => x == "A GVFS\\GVFS.Common\\Git\0" + TestConstants.AllZeroSha); - lines.ShouldContain(x => x == "A GVFS\\GVFS.Tests\0" + TestConstants.AllZeroSha); - lines.ShouldContain(x => x == "A GVFS\\GVFS.Tests\\Properties\0" + TestConstants.AllZeroSha); + lines.ShouldContain(x => x == "A GVFS\\GVFS.Tests\\Properties\0" + TestConstants.PartialFolderPlaceholderDatabaseValue); return lines; } - private string[] GetPlaceholderDatabaseLinesAfterUpgrade(string placeholderDatabasePath) + private string[] GetPlaceholderDatabaseLinesAfterUpgradeFrom12_1(string placeholderDatabasePath) { placeholderDatabasePath.ShouldBeAFile(this.fileSystem); string[] lines = this.fileSystem.ReadAllText(placeholderDatabasePath).Split(new char[] { '\r', '\n' }, StringSplitOptions.RemoveEmptyEntries); lines.Length.ShouldEqual(9); - lines.ShouldContain(x => x.Contains("Readme.md")); - lines.ShouldContain(x => x.Contains("Scripts\\RunUnitTests.bat")); - lines.ShouldContain(x => x.Contains("GVFS\\GVFS.Common\\Git\\GitRefs.cs")); - lines.ShouldContain(x => x.Contains("A .gitignore")); - lines.ShouldContain(x => x == "A Scripts\0" + TestConstants.AllZeroSha); - lines.ShouldContain(x => x == "A GVFS\0" + TestConstants.AllZeroSha); - lines.ShouldContain(x => x == "A GVFS\\GVFS.Common\0" + TestConstants.AllZeroSha); - lines.ShouldContain(x => x == "A GVFS\\GVFS.Common\\Git\0" + TestConstants.AllZeroSha); - lines.ShouldContain(x => x == "A GVFS\\GVFS.Tests\0" + TestConstants.AllZeroSha); + this.PlaceholderDatabaseShouldIncludeCommonLinesForUpgradeTestIO(lines); + return lines; + } + + private string[] GetPlaceholderDatabaseLinesAfterUpgradeFrom16(string placeholderDatabasePath) + { + placeholderDatabasePath.ShouldBeAFile(this.fileSystem); + string[] lines = this.fileSystem.ReadAllText(placeholderDatabasePath).Split(new char[] { '\r', '\n' }, StringSplitOptions.RemoveEmptyEntries); + lines.Length.ShouldEqual(10); + this.PlaceholderDatabaseShouldIncludeCommonLinesForUpgradeTestIO(lines); + lines.ShouldContain(x => x == "A GVFS\\GVFS.Tests\\Properties\0" + TestConstants.PartialFolderPlaceholderDatabaseValue); return lines; } diff --git a/GVFS/GVFS.FunctionalTests/Categories.cs b/GVFS/GVFS.FunctionalTests/Categories.cs index fe837c42b..181d671ad 100644 --- a/GVFS/GVFS.FunctionalTests/Categories.cs +++ b/GVFS/GVFS.FunctionalTests/Categories.cs @@ -14,8 +14,8 @@ public static class MacTODO // The FailsOnBuildAgent category is for tests that pass on dev // machines but not on the build agents public const string FailsOnBuildAgent = "FailsOnBuildAgent"; - - public const string NeedsLockHolder = "NeedsDotCoreLockHolder"; + public const string NeedsLockHolder = "NeedsDotCoreLockHolder"; + public const string NeedsCachePoisonFix = "NeedsCachePoisonFix"; public const string M2 = "M2_StaticViewGitCommands"; public const string M3 = "M3_AllGitCommands"; public const string M4 = "M4_All"; diff --git a/GVFS/GVFS.FunctionalTests/Program.cs b/GVFS/GVFS.FunctionalTests/Program.cs index b467b28d5..a9cf972c0 100644 --- a/GVFS/GVFS.FunctionalTests/Program.cs +++ b/GVFS/GVFS.FunctionalTests/Program.cs @@ -67,6 +67,7 @@ public static void Main(string[] args) { excludeCategories.Add(Categories.MacTODO.NeedsLockHolder); excludeCategories.Add(Categories.MacTODO.FailsOnBuildAgent); + excludeCategories.Add(Categories.MacTODO.NeedsCachePoisonFix); excludeCategories.Add(Categories.MacTODO.M2); excludeCategories.Add(Categories.MacTODO.M3); excludeCategories.Add(Categories.MacTODO.M4); diff --git a/GVFS/GVFS.FunctionalTests/Tests/EnlistmentPerFixture/MultithreadedReadWriteTests.cs b/GVFS/GVFS.FunctionalTests/Tests/EnlistmentPerFixture/MultithreadedReadWriteTests.cs index 21924e706..9846642af 100644 --- a/GVFS/GVFS.FunctionalTests/Tests/EnlistmentPerFixture/MultithreadedReadWriteTests.cs +++ b/GVFS/GVFS.FunctionalTests/Tests/EnlistmentPerFixture/MultithreadedReadWriteTests.cs @@ -14,7 +14,7 @@ namespace GVFS.FunctionalTests.Tests.EnlistmentPerFixture public class MultithreadedReadWriteTests : TestsWithEnlistmentPerFixture { [TestCase, Order(1)] - [Category(Categories.WindowsOnly)] + [Category(Categories.MacTODO.NeedsCachePoisonFix)] public void CanReadVirtualFileInParallel() { // Note: This test MUST go first, or else it needs to ensure that it is reading a unique path compared to the diff --git a/GVFS/GVFS.FunctionalTests/Tests/EnlistmentPerFixture/PrefetchVerbTests.cs b/GVFS/GVFS.FunctionalTests/Tests/EnlistmentPerFixture/PrefetchVerbTests.cs index 3500e0977..cb0a2a4a6 100644 --- a/GVFS/GVFS.FunctionalTests/Tests/EnlistmentPerFixture/PrefetchVerbTests.cs +++ b/GVFS/GVFS.FunctionalTests/Tests/EnlistmentPerFixture/PrefetchVerbTests.cs @@ -9,6 +9,7 @@ namespace GVFS.FunctionalTests.Tests.EnlistmentPerFixture { [TestFixture] + [NonParallelizable] public class PrefetchVerbTests : TestsWithEnlistmentPerFixture { private const string PrefetchCommitsAndTreesLock = "prefetch-commits-trees.lock"; diff --git a/GVFS/GVFS.FunctionalTests/Tests/EnlistmentPerFixture/PrefetchVerbWithoutSharedCacheTests.cs b/GVFS/GVFS.FunctionalTests/Tests/EnlistmentPerFixture/PrefetchVerbWithoutSharedCacheTests.cs index a64bae950..e5419e315 100644 --- a/GVFS/GVFS.FunctionalTests/Tests/EnlistmentPerFixture/PrefetchVerbWithoutSharedCacheTests.cs +++ b/GVFS/GVFS.FunctionalTests/Tests/EnlistmentPerFixture/PrefetchVerbWithoutSharedCacheTests.cs @@ -9,6 +9,8 @@ namespace GVFS.FunctionalTests.Tests.EnlistmentPerFixture { + // TODO(Mac): Before these tests can be enabled PostFetchJobShouldComplete needs + // to work on Mac (where post-fetch.lock is not removed from disk) [TestFixture] [Category(Categories.FullSuiteOnly)] [Category(Categories.MacTODO.M4)] diff --git a/GVFS/GVFS.FunctionalTests/Tests/GitCommands/CheckoutTests.cs b/GVFS/GVFS.FunctionalTests/Tests/GitCommands/CheckoutTests.cs index ed4066332..0fb311fc6 100644 --- a/GVFS/GVFS.FunctionalTests/Tests/GitCommands/CheckoutTests.cs +++ b/GVFS/GVFS.FunctionalTests/Tests/GitCommands/CheckoutTests.cs @@ -88,7 +88,23 @@ private enum NativeFileAccess : uint } [TestCase] - [Category(Categories.MacTODO.M3)] + [Category(Categories.MacTODO.NeedsCachePoisonFix)] + public void ReadDeepFilesAfterCheckout() + { + // In commit 8df701986dea0a5e78b742d2eaf9348825b14d35 the CheckoutNewBranchFromStartingPointTest files were not present + this.ValidateGitCommand("checkout 8df701986dea0a5e78b742d2eaf9348825b14d35"); + + // In commit cd5c55fea4d58252bb38058dd3818da75aff6685 the CheckoutNewBranchFromStartingPointTest files were present + this.ValidateGitCommand("checkout cd5c55fea4d58252bb38058dd3818da75aff6685"); + + this.FileShouldHaveContents("TestFile1 \r\n", "GitCommandsTests", "CheckoutNewBranchFromStartingPointTest", "test1.txt"); + this.FileShouldHaveContents("TestFile2 \r\n", "GitCommandsTests", "CheckoutNewBranchFromStartingPointTest", "test2.txt"); + + this.ValidateGitCommand("status"); + } + + [TestCase] + [Category(Categories.MacTODO.NeedsCachePoisonFix)] public void CheckoutNewBranchFromStartingPointTest() { // In commit 8df701986dea0a5e78b742d2eaf9348825b14d35 the CheckoutNewBranchFromStartingPointTest files were not present @@ -105,7 +121,7 @@ public void CheckoutNewBranchFromStartingPointTest() } [TestCase] - [Category(Categories.MacTODO.M3)] + [Category(Categories.MacTODO.NeedsCachePoisonFix)] public void CheckoutOrhpanBranchFromStartingPointTest() { // In commit 8df701986dea0a5e78b742d2eaf9348825b14d35 the CheckoutOrhpanBranchFromStartingPointTest files were not present @@ -122,13 +138,12 @@ public void CheckoutOrhpanBranchFromStartingPointTest() } [TestCase] - [Category(Categories.MacTODO.M3)] public void MoveFileFromDotGitFolderToWorkingDirectoryAndAddAndCheckout() { string testFileContents = "Test file contents for MoveFileFromDotGitFolderToWorkingDirectoryAndAddAndCheckout"; string filename = "AddedBySource.txt"; - string dotGitFilePath = @".git\" + filename; - string targetPath = @"Test_ConflictTests\AddedFiles\" + filename; + string dotGitFilePath = Path.Combine(".git", filename); + string targetPath = Path.Combine("Test_ConflictTests", "AddedFiles", filename); // In commit db95d631e379d366d26d899523f8136a77441914 Test_ConflictTests\AddedFiles\AddedBySource.txt does not exist this.ControlGitRepo.Fetch(GitRepoTests.ConflictSourceBranch); @@ -155,7 +170,6 @@ public void MoveFileFromDotGitFolderToWorkingDirectoryAndAddAndCheckout() } [TestCase] - [Category(Categories.MacTODO.M3)] public void CheckoutBranchNoCrashOnStatus() { this.ControlGitRepo.Fetch("FunctionalTests/20170331_git_crash"); @@ -164,7 +178,6 @@ public void CheckoutBranchNoCrashOnStatus() } [TestCase] - [Category(Categories.MacTODO.M3)] public void CheckoutCommitWhereFileContentsChangeAfterRead() { this.ControlGitRepo.Fetch(GitRepoTests.ConflictSourceBranch); @@ -184,7 +197,6 @@ public void CheckoutCommitWhereFileContentsChangeAfterRead() } [TestCase] - [Category(Categories.MacTODO.M3)] public void CheckoutCommitWhereFileDeletedAfterRead() { this.ControlGitRepo.Fetch(GitRepoTests.ConflictSourceBranch); @@ -205,7 +217,6 @@ public void CheckoutCommitWhereFileDeletedAfterRead() } [TestCase] - [Category(Categories.MacTODO.M3)] public void CheckoutBranchAfterReadingFileAndVerifyContentsCorrect() { this.ControlGitRepo.Fetch(GitRepoTests.ConflictSourceBranch); @@ -218,11 +229,10 @@ public void CheckoutBranchAfterReadingFileAndVerifyContentsCorrect() this.FilesShouldMatchCheckoutOfSourceBranch(); // Verify modified paths contents - GVFSHelpers.ModifiedPathsContentsShouldEqual(this.FileSystem, this.Enlistment.DotGVFSRoot, "A .gitattributes" + Environment.NewLine); + GVFSHelpers.ModifiedPathsContentsShouldEqual(this.FileSystem, this.Enlistment.DotGVFSRoot, "A .gitattributes" + GVFSHelpers.ModifiedPathsNewLine); } [TestCase] - [Category(Categories.MacTODO.M3)] public void CheckoutBranchAfterReadingAllFilesAndVerifyContentsCorrect() { this.ControlGitRepo.Fetch(GitRepoTests.ConflictSourceBranch); @@ -237,7 +247,7 @@ public void CheckoutBranchAfterReadingAllFilesAndVerifyContentsCorrect() .WithDeepStructure(this.FileSystem, this.ControlGitRepo.RootPath, compareContent: true); // Verify modified paths contents - GVFSHelpers.ModifiedPathsContentsShouldEqual(this.FileSystem, this.Enlistment.DotGVFSRoot, "A .gitattributes" + Environment.NewLine); + GVFSHelpers.ModifiedPathsContentsShouldEqual(this.FileSystem, this.Enlistment.DotGVFSRoot, "A .gitattributes" + GVFSHelpers.ModifiedPathsNewLine); } [TestCase] @@ -268,7 +278,6 @@ public void CheckoutBranchThatHasFolderShouldGetDeleted() } [TestCase] - [Category(Categories.MacTODO.M3)] public void CheckoutBranchThatDoesNotHaveFolderShouldNotHaveFolder() { // this.ControlGitRepo.Commitish should not have the folder Test_ConflictTests\AddedFiles @@ -294,7 +303,6 @@ public void CheckoutBranchThatDoesNotHaveFolderShouldNotHaveFolder() } [TestCase] - [Category(Categories.MacTODO.M3)] public void EditFileReadFileAndCheckoutConflict() { // editFilePath was changed on ConflictTargetBranch @@ -329,7 +337,6 @@ public void EditFileReadFileAndCheckoutConflict() } [TestCase] - [Category(Categories.MacTODO.M3)] public void MarkFileAsReadOnlyAndCheckoutCommitWhereFileIsDifferent() { string filePath = Path.Combine("Test_ConflictTests", "ModifiedFiles", "ConflictingChange.txt"); @@ -345,7 +352,6 @@ public void MarkFileAsReadOnlyAndCheckoutCommitWhereFileIsDifferent() } [TestCase] - [Category(Categories.MacTODO.M3)] public void MarkFileAsReadOnlyAndCheckoutCommitWhereFileIsDeleted() { string filePath = Path.Combine("Test_ConflictTests", "AddedFiles", "AddedBySource.txt"); @@ -361,7 +367,6 @@ public void MarkFileAsReadOnlyAndCheckoutCommitWhereFileIsDeleted() } [TestCase] - [Category(Categories.MacTODO.M3)] public void ModifyAndCheckoutFirstOfSeveralFilesWhoseNamesAppearBeforeDot() { // Commit cb2d05febf64e3b0df50bd8d3fe8f05c0e2caa47 has the files (a).txt and (z).txt @@ -446,7 +451,6 @@ public void ReadFileAfterTryingToReadFileAtCommitWhereFileDoesNotExist() } [TestCase] - [Category(Categories.MacTODO.M3)] public void CheckoutBranchWithOpenHandleBlockingRepoMetdataUpdate() { this.ControlGitRepo.Fetch(GitRepoTests.ConflictSourceBranch); @@ -552,7 +556,6 @@ public void CheckoutBranchWithOpenHandleBlockingProjectionDeleteAndRepoMetdataUp } [TestCase] - [Category(Categories.MacTODO.M3)] public void CheckoutBranchWithStaleRepoMetadataTmpFileOnDisk() { this.ControlGitRepo.Fetch(GitRepoTests.ConflictSourceBranch); @@ -711,7 +714,6 @@ public void ResetMixedTwiceThenCheckoutWithRemovedFiles() } [TestCase] - [Category(Categories.MacTODO.M3)] public void DeleteFolderAndChangeBranchToFolderWithDifferentCase() { // 692765 - Recursive modified paths entries for folders should be case insensitive when @@ -732,7 +734,7 @@ public void DeleteFolderAndChangeBranchToFolderWithDifferentCase() } [TestCase] - [Category(Categories.MacTODO.M3)] + [Category(Categories.MacTODO.NeedsCachePoisonFix)] public void SuccessfullyChecksOutDirectoryToFileToDirectory() { // This test switches between two branches and verifies specific transitions occured @@ -875,7 +877,6 @@ public void CheckoutBranchWithDirectoryNameSameAsFileWithWrite() } [TestCase] - [Category(Categories.MacTODO.M3)] public void CheckoutBranchDirectoryWithOneFile() { this.SetupForFileDirectoryTest(commandBranch: GitRepoTests.DirectoryWithDifferentFileAfterBranch); @@ -883,14 +884,12 @@ public void CheckoutBranchDirectoryWithOneFile() } [TestCase] - [Category(Categories.MacTODO.M3)] public void CheckoutBranchDirectoryWithOneFileEnumerate() { this.RunFileDirectoryEnumerateTest("checkout", commandBranch: GitRepoTests.DirectoryWithDifferentFileAfterBranch); } [TestCase] - [Category(Categories.MacTODO.M3)] public void CheckoutBranchDirectoryWithOneFileRead() { this.RunFileDirectoryReadTest("checkout", commandBranch: GitRepoTests.DirectoryWithDifferentFileAfterBranch); diff --git a/GVFS/GVFS.FunctionalTests/Tests/GitCommands/DeleteEmptyFolderTests.cs b/GVFS/GVFS.FunctionalTests/Tests/GitCommands/DeleteEmptyFolderTests.cs index 7bfb99cde..9fd134313 100644 --- a/GVFS/GVFS.FunctionalTests/Tests/GitCommands/DeleteEmptyFolderTests.cs +++ b/GVFS/GVFS.FunctionalTests/Tests/GitCommands/DeleteEmptyFolderTests.cs @@ -6,7 +6,6 @@ namespace GVFS.FunctionalTests.Tests.GitCommands { [TestFixture] [Category(Categories.GitCommands)] - [Category(Categories.MacTODO.M4)] public class DeleteEmptyFolderTests : GitRepoTests { public DeleteEmptyFolderTests() : base(enlistmentPerTest: true) diff --git a/GVFS/GVFS.FunctionalTests/Tests/GitCommands/EnumerationMergeTest.cs b/GVFS/GVFS.FunctionalTests/Tests/GitCommands/EnumerationMergeTest.cs index 6b77a2fdf..4306529ab 100644 --- a/GVFS/GVFS.FunctionalTests/Tests/GitCommands/EnumerationMergeTest.cs +++ b/GVFS/GVFS.FunctionalTests/Tests/GitCommands/EnumerationMergeTest.cs @@ -4,7 +4,6 @@ namespace GVFS.FunctionalTests.Tests.GitCommands { [TestFixture] [Category(Categories.GitCommands)] - [Category(Categories.MacTODO.M3)] public class EnumerationMergeTest : GitRepoTests { // Commit that found GvFlt Bug 12258777: Entries are sometimes skipped during diff --git a/GVFS/GVFS.FunctionalTests/Tests/GitCommands/GitCommandsTests.cs b/GVFS/GVFS.FunctionalTests/Tests/GitCommands/GitCommandsTests.cs index 509cfb03f..f079e5464 100644 --- a/GVFS/GVFS.FunctionalTests/Tests/GitCommands/GitCommandsTests.cs +++ b/GVFS/GVFS.FunctionalTests/Tests/GitCommands/GitCommandsTests.cs @@ -310,7 +310,6 @@ public void DeleteFileWithNameAheadOfDotAndSwitchCommits() } [TestCase] - [Category(Categories.MacTODO.M3)] public void AddFileAndCommitOnNewBranchSwitchDeleteFolderAndSwitchBack() { // 663045 - Confirm that folder can be deleted after adding a file then changing branches @@ -710,7 +709,6 @@ public void AddFileAfterFolderRename() } [TestCase] - [Category(Categories.MacTODO.M3)] public void ResetSoft() { this.ValidateGitCommand("checkout -b tests/functional/ResetSoft"); @@ -742,7 +740,6 @@ public void ManuallyModifyHead() } [TestCase] - [Category(Categories.MacTODO.M3)] public void ResetSoftTwice() { this.ValidateGitCommand("checkout -b tests/functional/ResetSoftTwice"); @@ -754,7 +751,6 @@ public void ResetSoftTwice() } [TestCase] - [Category(Categories.MacTODO.M3)] public void ResetMixedTwice() { this.ValidateGitCommand("checkout -b tests/functional/ResetMixedTwice"); diff --git a/GVFS/GVFS.FunctionalTests/Tests/GitCommands/GitRepoTests.cs b/GVFS/GVFS.FunctionalTests/Tests/GitCommands/GitRepoTests.cs index 39301b73e..932ee1eaa 100644 --- a/GVFS/GVFS.FunctionalTests/Tests/GitCommands/GitRepoTests.cs +++ b/GVFS/GVFS.FunctionalTests/Tests/GitCommands/GitRepoTests.cs @@ -409,7 +409,7 @@ protected void RunFileDirectoryEnumerateTest(string command, string commandBranc protected void RunFileDirectoryReadTest(string command, string commandBranch = DirectoryWithFileAfterBranch) { this.SetupForFileDirectoryTest(commandBranch); - this.FileContentsShouldMatch("file.txt\\file.txt"); + this.FileContentsShouldMatch("file.txt", "file.txt"); this.ValidateFileDirectoryTest(command, commandBranch); } diff --git a/GVFS/GVFS.FunctionalTests/Tests/GitCommands/ResetHardTests.cs b/GVFS/GVFS.FunctionalTests/Tests/GitCommands/ResetHardTests.cs index 8b3972b16..17ccf772f 100644 --- a/GVFS/GVFS.FunctionalTests/Tests/GitCommands/ResetHardTests.cs +++ b/GVFS/GVFS.FunctionalTests/Tests/GitCommands/ResetHardTests.cs @@ -5,7 +5,6 @@ namespace GVFS.FunctionalTests.Tests.GitCommands { [TestFixture] [Category(Categories.GitCommands)] - [Category(Categories.MacTODO.M3)] public class ResetHardTests : GitRepoTests { private const string ResetHardCommand = "reset --hard"; diff --git a/GVFS/GVFS.FunctionalTests/Tests/GitCommands/ResetSoftTests.cs b/GVFS/GVFS.FunctionalTests/Tests/GitCommands/ResetSoftTests.cs index 01ce8336b..e79cdf99b 100644 --- a/GVFS/GVFS.FunctionalTests/Tests/GitCommands/ResetSoftTests.cs +++ b/GVFS/GVFS.FunctionalTests/Tests/GitCommands/ResetSoftTests.cs @@ -4,7 +4,6 @@ namespace GVFS.FunctionalTests.Tests.GitCommands { [TestFixture] [Category(Categories.GitCommands)] - [Category(Categories.MacTODO.M3)] public class ResetSoftTests : GitRepoTests { public ResetSoftTests() : base(enlistmentPerTest: true) @@ -51,6 +50,7 @@ public void ResetSoftThenCheckoutNoConflicts() } [TestCase] + [Category(Categories.MacTODO.M3)] public void ResetSoftThenResetHeadThenCheckoutNoConflicts() { this.ValidateGitCommand("checkout " + GitRepoTests.ConflictTargetBranch); diff --git a/GVFS/GVFS.FunctionalTests/Tests/GitCommands/UpdateRefTests.cs b/GVFS/GVFS.FunctionalTests/Tests/GitCommands/UpdateRefTests.cs index 3ff64c14e..20281ecc8 100644 --- a/GVFS/GVFS.FunctionalTests/Tests/GitCommands/UpdateRefTests.cs +++ b/GVFS/GVFS.FunctionalTests/Tests/GitCommands/UpdateRefTests.cs @@ -4,7 +4,6 @@ namespace GVFS.FunctionalTests.Tests.GitCommands { [TestFixture] [Category(Categories.GitCommands)] - [Category(Categories.MacTODO.M4)] public class UpdateRefTests : GitRepoTests { public UpdateRefTests() : base(enlistmentPerTest: true) diff --git a/GVFS/GVFS.FunctionalTests/Tests/MultiEnlistmentTests/SharedCacheTests.cs b/GVFS/GVFS.FunctionalTests/Tests/MultiEnlistmentTests/SharedCacheTests.cs index 7499293e6..de37e9070 100644 --- a/GVFS/GVFS.FunctionalTests/Tests/MultiEnlistmentTests/SharedCacheTests.cs +++ b/GVFS/GVFS.FunctionalTests/Tests/MultiEnlistmentTests/SharedCacheTests.cs @@ -7,6 +7,7 @@ using System.Collections.Generic; using System.IO; using System.Linq; +using System.Runtime.InteropServices; using System.Threading; using System.Threading.Tasks; @@ -14,7 +15,6 @@ namespace GVFS.FunctionalTests.Tests.MultiEnlistmentTests { [TestFixture] [Category(Categories.FullSuiteOnly)] - [Category(Categories.MacTODO.M3)] public class SharedCacheTests : TestsWithMultiEnlistment { private const string WellKnownFile = "Readme.md"; @@ -61,6 +61,7 @@ public void SecondCloneDoesNotDownloadAdditionalObjects() } [TestCase] + [Category(Categories.MacTODO.M4)] public void RepairFixesCorruptBlobSizesDatabase() { GVFSFunctionalTestEnlistment enlistment = this.CloneAndMountEnlistment(); @@ -80,6 +81,7 @@ public void RepairFixesCorruptBlobSizesDatabase() } [TestCase] + [Category(Categories.MacTODO.M4)] public void CloneCleansUpStaleMetadataLock() { GVFSFunctionalTestEnlistment enlistment1 = this.CloneAndMountEnlistment(); @@ -124,6 +126,7 @@ public void ParallelReadsInASharedCache() } [TestCase] + [Category(Categories.MacTODO.M3)] public void DeleteObjectsCacheAndCacheMappingBeforeMount() { GVFSFunctionalTestEnlistment enlistment1 = this.CloneAndMountEnlistment(); @@ -133,7 +136,7 @@ public void DeleteObjectsCacheAndCacheMappingBeforeMount() string objectsRoot = GVFSHelpers.GetPersistedGitObjectsRoot(enlistment1.DotGVFSRoot).ShouldNotBeNull(); objectsRoot.ShouldBeADirectory(this.fileSystem); - CmdRunner.DeleteDirectoryWithUnlimitedRetries(objectsRoot); + this.DeleteDirectoryWithUnlimitedRetries(objectsRoot); string metadataPath = Path.Combine(this.localCachePath, "mapping.dat"); metadataPath.ShouldBeAFile(this.fileSystem); @@ -156,6 +159,7 @@ public void DeleteObjectsCacheAndCacheMappingBeforeMount() } [TestCase] + [Category(Categories.MacTODO.M3)] public void DeleteCacheDuringHydrations() { GVFSFunctionalTestEnlistment enlistment1 = this.CloneAndMountEnlistment(); @@ -173,7 +177,7 @@ public void DeleteCacheDuringHydrations() try { // Delete objectsRoot rather than this.localCachePath as the blob sizes database cannot be deleted while GVFS is mounted - CmdRunner.DeleteDirectoryWithUnlimitedRetries(objectsRoot); + this.DeleteDirectoryWithUnlimitedRetries(objectsRoot); Thread.Sleep(100); } catch (IOException) @@ -213,7 +217,7 @@ public void MountReusesLocalCacheKeyWhenGitObjectsRootDeleted() mappingFileContents.Length.ShouldNotEqual(0, "mapping.dat should not be empty"); // Delete the git objects root folder, mount should re-create it and the mapping.dat file should not change - CmdRunner.DeleteDirectoryWithUnlimitedRetries(objectsRoot); + this.DeleteDirectoryWithUnlimitedRetries(objectsRoot); enlistment.MountGVFS(); @@ -240,7 +244,7 @@ public void MountUsesNewLocalCacheKeyWhenLocalCacheDeleted() mappingFileContents.Length.ShouldNotEqual(0, "mapping.dat should not be empty"); // Delete the local cache folder, mount should re-create it and generate a new mapping file and local cache key - CmdRunner.DeleteDirectoryWithUnlimitedRetries(enlistment.LocalCacheRoot); + this.DeleteDirectoryWithUnlimitedRetries(enlistment.LocalCacheRoot); enlistment.MountGVFS(); @@ -268,7 +272,7 @@ public void MountUsesNewLocalCacheKeyWhenLocalCacheDeleted() // localCacheParentPath can be deleted (as the SQLite blob sizes database cannot be deleted while GVFS is mounted) protected override void OnTearDownEnlistmentsDeleted() { - CmdRunner.DeleteDirectoryWithUnlimitedRetries(this.localCacheParentPath); + this.DeleteDirectoryWithUnlimitedRetries(this.localCacheParentPath); } private GVFSFunctionalTestEnlistment CloneAndMountEnlistment(string branch = null) @@ -303,5 +307,18 @@ private void HydrateEntireRepo(GVFSFunctionalTestEnlistment enlistment) } } } + + private void DeleteDirectoryWithUnlimitedRetries(string path) + { + if (RuntimeInformation.IsOSPlatform(OSPlatform.Windows)) + { + CmdRunner.DeleteDirectoryWithUnlimitedRetries(path); + } + else + { + // TODO(Mac): See if we can use BashRunner.DeleteDirectoryWithRetry on Windows as well + BashRunner.DeleteDirectoryWithUnlimitedRetries(path); + } + } } } diff --git a/GVFS/GVFS.FunctionalTests/Tools/GVFSHelpers.cs b/GVFS/GVFS.FunctionalTests/Tools/GVFSHelpers.cs index 17fd64369..139fb0a06 100644 --- a/GVFS/GVFS.FunctionalTests/Tools/GVFSHelpers.cs +++ b/GVFS/GVFS.FunctionalTests/Tools/GVFSHelpers.cs @@ -13,6 +13,8 @@ namespace GVFS.FunctionalTests.Tools { public static class GVFSHelpers { + public const string ModifiedPathsNewLine = "\r\n"; + public static readonly string BackgroundOpsFile = Path.Combine("databases", "BackgroundGitOperations.dat"); public static readonly string PlaceholderListFile = Path.Combine("databases", "PlaceholderList.dat"); public static readonly string RepoMetadataName = Path.Combine("databases", "RepoMetadata.dat"); @@ -21,9 +23,7 @@ public static class GVFSHelpers private const string DiskLayoutMinorVersionKey = "DiskLayoutMinorVersion"; private const string LocalCacheRootKey = "LocalCacheRoot"; private const string GitObjectsRootKey = "GitObjectsRoot"; - private const string BlobSizesRootKey = "BlobSizesRoot"; - - private const string ModifiedPathsNewLine = "\r\n"; + private const string BlobSizesRootKey = "BlobSizesRoot"; public static void SaveDiskLayoutVersion(string dotGVFSRoot, string majorVersion, string minorVersion) { diff --git a/GVFS/GVFS.FunctionalTests/Tools/TestConstants.cs b/GVFS/GVFS.FunctionalTests/Tools/TestConstants.cs index 7f04e129a..ebc6ee28f 100644 --- a/GVFS/GVFS.FunctionalTests/Tools/TestConstants.cs +++ b/GVFS/GVFS.FunctionalTests/Tools/TestConstants.cs @@ -5,6 +5,7 @@ namespace GVFS.FunctionalTests.Tools public static class TestConstants { public const string AllZeroSha = "0000000000000000000000000000000000000000"; + public const string PartialFolderPlaceholderDatabaseValue = " PARTIAL FOLDER"; public static class DotGit { diff --git a/GVFS/GVFS.Platform.Mac/MacFileSystemVirtualizer.cs b/GVFS/GVFS.Platform.Mac/MacFileSystemVirtualizer.cs index 29607f276..bc13595cd 100644 --- a/GVFS/GVFS.Platform.Mac/MacFileSystemVirtualizer.cs +++ b/GVFS/GVFS.Platform.Mac/MacFileSystemVirtualizer.cs @@ -70,6 +70,29 @@ public override void Stop() this.Context.Tracer.RelatedEvent(EventLevel.Informational, $"{nameof(this.Stop)}_StopRequested", metadata: null); } + public override FileSystemResult WritePlaceholderFile( + string relativePath, + long endOfFile, + string sha) + { + // TODO(Mac): Add functional tests that validate file mode is set correctly + ushort fileMode = this.FileSystemCallbacks.GitIndexProjection.GetFilePathMode(relativePath); + Result result = this.virtualizationInstance.WritePlaceholderFile( + relativePath, + PlaceholderVersionId, + ToVersionIdByteArray(FileSystemVirtualizer.ConvertShaToContentId(sha)), + (ulong)endOfFile, + fileMode); + + return new FileSystemResult(ResultToFSResult(result), unchecked((int)result)); + } + + public override FileSystemResult WritePlaceholderDirectory(string relativePath) + { + Result result = this.virtualizationInstance.WritePlaceholderDirectory(relativePath); + return new FileSystemResult(ResultToFSResult(result), unchecked((int)result)); + } + public override FileSystemResult UpdatePlaceholderIfNeeded( string relativePath, DateTime creationTime, @@ -435,32 +458,29 @@ private Result CreatePlaceholders(string directoryRelativePath, IEnumerable throw new NotImplementedException(); public bool IsSupported(string normalizedEnlistmentRootPath, out string warning, out string error) diff --git a/GVFS/GVFS.Platform.Windows/DiskLayoutUpgrades/DiskLayout16to17Upgrade_FolderPlaceholderValues.cs b/GVFS/GVFS.Platform.Windows/DiskLayoutUpgrades/DiskLayout16to17Upgrade_FolderPlaceholderValues.cs new file mode 100644 index 000000000..24893770a --- /dev/null +++ b/GVFS/GVFS.Platform.Windows/DiskLayoutUpgrades/DiskLayout16to17Upgrade_FolderPlaceholderValues.cs @@ -0,0 +1,75 @@ +using GVFS.Common; +using GVFS.Common.FileSystem; +using GVFS.Common.Tracing; +using GVFS.DiskLayoutUpgrades; +using System; +using System.Collections.Generic; +using System.IO; + +namespace GVFS.Platform.Windows.DiskLayoutUpgrades +{ + /// + /// Updates the values for folder placeholders from AllZeroSha to PlaceholderListDatabase.PartialFolderValue + /// + public class DiskLayout16to17Upgrade_FolderPlaceholderValues : DiskLayoutUpgrade.MajorUpgrade + { + protected override int SourceMajorVersion => 16; + + public override bool TryUpgrade(ITracer tracer, string enlistmentRoot) + { + string dotGVFSRoot = Path.Combine(enlistmentRoot, GVFSConstants.DotGVFS.Root); + try + { + string error; + PlaceholderListDatabase placeholders; + if (!PlaceholderListDatabase.TryCreate( + tracer, + Path.Combine(dotGVFSRoot, GVFSConstants.DotGVFS.Databases.PlaceholderList), + new PhysicalFileSystem(), + out placeholders, + out error)) + { + tracer.RelatedError("Failed to open placeholder database: " + error); + return false; + } + + using (placeholders) + { + List oldPlaceholderEntries = placeholders.GetAllEntries(); + List newPlaceholderEntries = new List(); + + foreach (PlaceholderListDatabase.PlaceholderData entry in oldPlaceholderEntries) + { + if (entry.Sha == GVFSConstants.AllZeroSha) + { + newPlaceholderEntries.Add(new PlaceholderListDatabase.PlaceholderData(entry.Path, PlaceholderListDatabase.PartialFolderValue)); + } + else + { + newPlaceholderEntries.Add(entry); + } + } + + placeholders.WriteAllEntriesAndFlush(newPlaceholderEntries); + } + } + catch (IOException ex) + { + tracer.RelatedError("Could not write to placeholder database: " + ex.ToString()); + return false; + } + catch (Exception ex) + { + tracer.RelatedError("Error updating placeholder database folder entries: " + ex.ToString()); + return false; + } + + if (!this.TryIncrementMajorVersion(tracer, enlistmentRoot)) + { + return false; + } + + return true; + } + } +} diff --git a/GVFS/GVFS.Platform.Windows/DiskLayoutUpgrades/DiskLayout9to10Upgrade_BackgroundAndPlaceholderListToFileBased.cs b/GVFS/GVFS.Platform.Windows/DiskLayoutUpgrades/DiskLayout9to10Upgrade_BackgroundAndPlaceholderListToFileBased.cs index 464d2c1db..397738439 100644 --- a/GVFS/GVFS.Platform.Windows/DiskLayoutUpgrades/DiskLayout9to10Upgrade_BackgroundAndPlaceholderListToFileBased.cs +++ b/GVFS/GVFS.Platform.Windows/DiskLayoutUpgrades/DiskLayout9to10Upgrade_BackgroundAndPlaceholderListToFileBased.cs @@ -75,7 +75,7 @@ private bool UpdatePlaceholderList(ITracer tracer, string dotGVFSRoot) foreach (KeyValuePair kvp in oldPlaceholders) { tracer.RelatedInfo("Copying ESENT entry: {0} = {1}", kvp.Key, kvp.Value); - data.Add(new PlaceholderListDatabase.PlaceholderData(path: kvp.Key, sha: kvp.Value)); + data.Add(new PlaceholderListDatabase.PlaceholderData(path: kvp.Key, fileShaOrFolderValue: kvp.Value)); } newPlaceholders.WriteAllEntriesAndFlush(data); diff --git a/GVFS/GVFS.Platform.Windows/DiskLayoutUpgrades/WindowsDiskLayoutUpgradeData.cs b/GVFS/GVFS.Platform.Windows/DiskLayoutUpgrades/WindowsDiskLayoutUpgradeData.cs index 36b6bafc4..bd656bae3 100644 --- a/GVFS/GVFS.Platform.Windows/DiskLayoutUpgrades/WindowsDiskLayoutUpgradeData.cs +++ b/GVFS/GVFS.Platform.Windows/DiskLayoutUpgrades/WindowsDiskLayoutUpgradeData.cs @@ -27,6 +27,7 @@ public DiskLayoutUpgrade[] Upgrades new DiskLayout13to14Upgrade_BlobSizes(), new DiskLayout14to15Upgrade_ModifiedPaths(), new DiskLayout15to16Upgrade_GitStatusCache(), + new DiskLayout16to17Upgrade_FolderPlaceholderValues() }; } } diff --git a/GVFS/GVFS.Platform.Windows/GVFS.Platform.Windows.csproj b/GVFS/GVFS.Platform.Windows/GVFS.Platform.Windows.csproj index e3deda70c..c461a584b 100644 --- a/GVFS/GVFS.Platform.Windows/GVFS.Platform.Windows.csproj +++ b/GVFS/GVFS.Platform.Windows/GVFS.Platform.Windows.csproj @@ -79,6 +79,7 @@ + diff --git a/GVFS/GVFS.Platform.Windows/ProjFSFilter.cs b/GVFS/GVFS.Platform.Windows/ProjFSFilter.cs index 6b9f20493..544b5678a 100644 --- a/GVFS/GVFS.Platform.Windows/ProjFSFilter.cs +++ b/GVFS/GVFS.Platform.Windows/ProjFSFilter.cs @@ -34,6 +34,7 @@ public class ProjFSFilter : IKernelDriver private const uint OkResult = 0; private const uint NameCollisionErrorResult = 0x801F0012; + public bool EnumerationExpandsDirectories { get; } = false; public string DriverLogFolderName { get; } = ProjFSFilter.ServiceName; public static bool TryAttach(ITracer tracer, string enlistmentRoot, out string errorMessage) diff --git a/GVFS/GVFS.Platform.Windows/WindowsFileSystemVirtualizer.cs b/GVFS/GVFS.Platform.Windows/WindowsFileSystemVirtualizer.cs index acd2addea..6418fda14 100644 --- a/GVFS/GVFS.Platform.Windows/WindowsFileSystemVirtualizer.cs +++ b/GVFS/GVFS.Platform.Windows/WindowsFileSystemVirtualizer.cs @@ -115,6 +115,45 @@ public override FileSystemResult DeleteFile(string relativePath, UpdatePlacehold return new FileSystemResult(HResultToFSResult(result), unchecked((int)result)); } + public override FileSystemResult WritePlaceholderFile( + string relativePath, + long endOfFile, + string sha) + { + FileProperties properties = this.FileSystemCallbacks.GetLogsHeadFileProperties(); + HResult result = this.virtualizationInstance.WritePlaceholderInformation( + relativePath, + properties.CreationTimeUTC, + properties.LastAccessTimeUTC, + properties.LastWriteTimeUTC, + changeTime: properties.LastWriteTimeUTC, + fileAttributes: (uint)NativeMethods.FileAttributes.FILE_ATTRIBUTE_ARCHIVE, + endOfFile: endOfFile, + isDirectory: false, + contentId: FileSystemVirtualizer.ConvertShaToContentId(sha), + providerId: PlaceholderVersionId); + + return new FileSystemResult(HResultToFSResult(result), unchecked((int)result)); + } + + public override FileSystemResult WritePlaceholderDirectory(string relativePath) + { + FileProperties properties = this.FileSystemCallbacks.GetLogsHeadFileProperties(); + HResult result = this.virtualizationInstance.WritePlaceholderInformation( + relativePath, + properties.CreationTimeUTC, + properties.LastAccessTimeUTC, + properties.LastWriteTimeUTC, + changeTime: properties.LastWriteTimeUTC, + fileAttributes: (uint)NativeMethods.FileAttributes.FILE_ATTRIBUTE_DIRECTORY, + endOfFile: 0, + isDirectory: true, + contentId: FolderContentId, + providerId: PlaceholderVersionId); + + return new FileSystemResult(HResultToFSResult(result), unchecked((int)result)); + } + public override FileSystemResult UpdatePlaceholderIfNeeded( string relativePath, DateTime creationTime, @@ -740,31 +779,20 @@ private void GetPlaceholderInformationAsyncHandler( // with proper case. string gitCaseVirtualPath = Path.Combine(parentFolderPath, fileInfo.Name); - string sha = string.Empty; - uint fileAttributes; + string sha; + FileSystemResult fileSystemResult; if (fileInfo.IsFolder) { - fileAttributes = (uint)NativeMethods.FileAttributes.FILE_ATTRIBUTE_DIRECTORY; + sha = string.Empty; + fileSystemResult = this.WritePlaceholderDirectory(gitCaseVirtualPath); } else { sha = fileInfo.Sha.ToString(); - fileAttributes = (uint)NativeMethods.FileAttributes.FILE_ATTRIBUTE_ARCHIVE; + fileSystemResult = this.WritePlaceholderFile(gitCaseVirtualPath, fileInfo.Size, sha); } - FileProperties properties = this.FileSystemCallbacks.GetLogsHeadFileProperties(); - result = this.virtualizationInstance.WritePlaceholderInformation( - gitCaseVirtualPath, - properties.CreationTimeUTC, - properties.LastAccessTimeUTC, - properties.LastWriteTimeUTC, - changeTime: properties.LastWriteTimeUTC, - fileAttributes: fileAttributes, - endOfFile: fileInfo.Size, - isDirectory: fileInfo.IsFolder, - contentId: FileSystemVirtualizer.ConvertShaToContentId(sha), - providerId: PlaceholderVersionId); - + result = (HResult)fileSystemResult.RawResult; if (result != HResult.Ok) { EventMetadata metadata = this.CreateEventMetadata(virtualPath); @@ -777,6 +805,7 @@ private void GetPlaceholderInformationAsyncHandler( metadata.Add("triggeringProcessImageFileName", triggeringProcessImageFileName); metadata.Add("FileName", fileInfo.Name); metadata.Add("IsFolder", fileInfo.IsFolder); + metadata.Add(nameof(sha), sha); metadata.Add(nameof(result), result.ToString("X") + "(" + result.ToString("G") + ")"); this.Context.Tracer.RelatedError(metadata, $"{nameof(this.GetPlaceholderInformationAsyncHandler)}: {nameof(this.virtualizationInstance.WritePlaceholderInformation)} failed"); } @@ -1102,7 +1131,7 @@ private void NotifyNewFileCreatedHandler( string directoryPath = Path.Combine(this.Context.Enlistment.WorkingDirectoryRoot, virtualPath); HResult hr = this.virtualizationInstance.ConvertDirectoryToPlaceholder( directoryPath, - ConvertShaToContentId(GVFSConstants.AllZeroSha), + FolderContentId, PlaceholderVersionId); if (hr == HResult.Ok) diff --git a/GVFS/GVFS.Tests/Should/EnumerableShouldExtensions.cs b/GVFS/GVFS.Tests/Should/EnumerableShouldExtensions.cs index 320bc2a81..913af43b8 100644 --- a/GVFS/GVFS.Tests/Should/EnumerableShouldExtensions.cs +++ b/GVFS/GVFS.Tests/Should/EnumerableShouldExtensions.cs @@ -122,6 +122,11 @@ public static IEnumerable ShouldMatchInOrder(this IEnumerable group, IE return group; } + public static IEnumerable ShouldMatchInOrder(this IEnumerable group, params T[] expectedValues) + { + return group.ShouldMatchInOrder((IEnumerable)expectedValues); + } + public static IEnumerable ShouldMatchInOrder(this IEnumerable group, IEnumerable expectedValues) { return group.ShouldMatchInOrder(expectedValues, (t1, t2) => t1.Equals(t2)); diff --git a/GVFS/GVFS.UnitTests/Common/GitStatusCacheTests.cs b/GVFS/GVFS.UnitTests/Common/GitStatusCacheTests.cs index a4ef83db6..405f22483 100644 --- a/GVFS/GVFS.UnitTests/Common/GitStatusCacheTests.cs +++ b/GVFS/GVFS.UnitTests/Common/GitStatusCacheTests.cs @@ -2,6 +2,7 @@ using GVFS.Common.Git; using GVFS.Common.NamedPipes; using GVFS.Tests.Should; +using GVFS.UnitTests.Category; using GVFS.UnitTests.Mock.Common; using GVFS.UnitTests.Mock.FileSystem; using GVFS.UnitTests.Mock.Git; @@ -108,6 +109,7 @@ public void CanInvalidateCleanCache() } [TestCase] + [Category(CategoryConstants.ExceptionExpected)] public void CacheFileErrorShouldBlock() { this.fileSystem.DeleteFileThrowsException = true; diff --git a/GVFS/GVFS.UnitTests/Common/PlaceholderDatabaseTests.cs b/GVFS/GVFS.UnitTests/Common/PlaceholderDatabaseTests.cs index ea02b8ca3..465e4b907 100644 --- a/GVFS/GVFS.UnitTests/Common/PlaceholderDatabaseTests.cs +++ b/GVFS/GVFS.UnitTests/Common/PlaceholderDatabaseTests.cs @@ -21,8 +21,9 @@ public class PlaceholderDatabaseTests private const string InputThirdFilePath = "thirdFile"; private const string InputThirdFileSHA = "ff9630E00F715315FC90D4AEC98E6A7398F8BF11"; - private const string ExpectedGitIgnoreEntry = "A " + InputGitIgnorePath + "\0" + InputGitIgnoreSHA + "\r\n"; - private const string ExpectedGitAttributesEntry = "A " + InputGitAttributesPath + "\0" + InputGitAttributesSHA + "\r\n"; + private const string PlaceholderDatabaseNewLine = "\r\n"; + private const string ExpectedGitIgnoreEntry = "A " + InputGitIgnorePath + "\0" + InputGitIgnoreSHA + PlaceholderDatabaseNewLine; + private const string ExpectedGitAttributesEntry = "A " + InputGitAttributesPath + "\0" + InputGitAttributesSHA + PlaceholderDatabaseNewLine; private const string ExpectedTwoEntries = ExpectedGitIgnoreEntry + ExpectedGitAttributesEntry; @@ -50,11 +51,11 @@ public void WritesPlaceholderAddToFile() { ConfigurableFileSystem fs = new ConfigurableFileSystem(); PlaceholderListDatabase dut = CreatePlaceholderListDatabase(fs, string.Empty); - dut.AddAndFlush(InputGitIgnorePath, InputGitIgnoreSHA); + dut.AddAndFlushFile(InputGitIgnorePath, InputGitIgnoreSHA); fs.ExpectedFiles[MockEntryFileName].ReadAsString().ShouldEqual(ExpectedGitIgnoreEntry); - dut.AddAndFlush(InputGitAttributesPath, InputGitAttributesSHA); + dut.AddAndFlushFile(InputGitAttributesPath, InputGitAttributesSHA); fs.ExpectedFiles[MockEntryFileName].ReadAsString().ShouldEqual(ExpectedTwoEntries); } @@ -65,9 +66,9 @@ public void GetAllEntriesReturnsCorrectEntries() ConfigurableFileSystem fs = new ConfigurableFileSystem(); using (PlaceholderListDatabase dut1 = CreatePlaceholderListDatabase(fs, string.Empty)) { - dut1.AddAndFlush(InputGitIgnorePath, InputGitIgnoreSHA); - dut1.AddAndFlush(InputGitAttributesPath, InputGitAttributesSHA); - dut1.AddAndFlush(InputThirdFilePath, InputThirdFileSHA); + dut1.AddAndFlushFile(InputGitIgnorePath, InputGitIgnoreSHA); + dut1.AddAndFlushFile(InputGitAttributesPath, InputGitAttributesSHA); + dut1.AddAndFlushFile(InputThirdFilePath, InputThirdFileSHA); dut1.RemoveAndFlush(InputThirdFilePath); } @@ -78,6 +79,37 @@ public void GetAllEntriesReturnsCorrectEntries() allData.Count.ShouldEqual(2); } + [TestCase] + public void GetAllEntriesSplitsFilesAndFoldersCorrectly() + { + ConfigurableFileSystem fs = new ConfigurableFileSystem(); + using (PlaceholderListDatabase dut1 = CreatePlaceholderListDatabase(fs, string.Empty)) + { + dut1.AddAndFlushFile(InputGitIgnorePath, InputGitIgnoreSHA); + dut1.AddAndFlushFolder("partialFolder", isExpanded: false); + dut1.AddAndFlushFile(InputGitAttributesPath, InputGitAttributesSHA); + dut1.AddAndFlushFolder("expandedFolder", isExpanded: true); + dut1.AddAndFlushFile(InputThirdFilePath, InputThirdFileSHA); + dut1.RemoveAndFlush(InputThirdFilePath); + } + + string error; + PlaceholderListDatabase dut2; + PlaceholderListDatabase.TryCreate(null, MockEntryFileName, fs, out dut2, out error).ShouldEqual(true, error); + List fileData; + List folderData; + dut2.GetAllEntries(out fileData, out folderData); + fileData.Count.ShouldEqual(2); + folderData.Count.ShouldEqual(2); + folderData.ShouldContain( + new[] + { + new PlaceholderListDatabase.PlaceholderData("partialFolder", PlaceholderListDatabase.PartialFolderValue), + new PlaceholderListDatabase.PlaceholderData("expandedFolder", PlaceholderListDatabase.ExpandedFolderValue) + }, + (data1, data2) => data1.Path == data2.Path && data1.Sha == data2.Sha); + } + [TestCase] public void WriteAllEntriesCorrectlyWritesFile() { @@ -106,7 +138,7 @@ public void HandlesRaceBetweenAddAndWriteAllEntries() List existingEntries = dut.GetAllEntries(); - dut.AddAndFlush(InputGitAttributesPath, InputGitAttributesSHA); + dut.AddAndFlushFile(InputGitAttributesPath, InputGitAttributesSHA); dut.WriteAllEntriesAndFlush(existingEntries); fs.ExpectedFiles[MockEntryFileName].ReadAsString().ShouldEqual(ExpectedTwoEntries); @@ -115,7 +147,7 @@ public void HandlesRaceBetweenAddAndWriteAllEntries() [TestCase] public void HandlesRaceBetweenRemoveAndWriteAllEntries() { - const string DeleteGitAttributesEntry = "D .gitattributes\r\n"; + const string DeleteGitAttributesEntry = "D .gitattributes" + PlaceholderDatabaseNewLine; ConfigurableFileSystem fs = new ConfigurableFileSystem(); fs.ExpectedFiles.Add(MockEntryFileName + ".tmp", new ReusableMemoryStream(string.Empty)); diff --git a/GVFS/GVFS.UnitTests/Mock/Virtualization/FileSystem/MockFileSystemVirtualizer.cs b/GVFS/GVFS.UnitTests/Mock/Virtualization/FileSystem/MockFileSystemVirtualizer.cs index 4972f23a2..b5223be47 100644 --- a/GVFS/GVFS.UnitTests/Mock/Virtualization/FileSystem/MockFileSystemVirtualizer.cs +++ b/GVFS/GVFS.UnitTests/Mock/Virtualization/FileSystem/MockFileSystemVirtualizer.cs @@ -27,6 +27,16 @@ public override void Stop() { } + public override FileSystemResult WritePlaceholderFile(string relativePath, long endOfFile, string sha) + { + throw new NotImplementedException(); + } + + public override FileSystemResult WritePlaceholderDirectory(string relativePath) + { + throw new NotImplementedException(); + } + public override FileSystemResult UpdatePlaceholderIfNeeded(string relativePath, DateTime creationTime, DateTime lastAccessTime, DateTime lastWriteTime, DateTime changeTime, uint fileAttributes, long endOfFile, string shaContentId, UpdatePlaceholderType updateFlags, out UpdateFailureReason failureReason) { throw new NotImplementedException(); diff --git a/GVFS/GVFS.UnitTests/Mock/Virtualization/Projection/MockGitIndexProjection.cs b/GVFS/GVFS.UnitTests/Mock/Virtualization/Projection/MockGitIndexProjection.cs index fe82cd2e3..904946610 100644 --- a/GVFS/GVFS.UnitTests/Mock/Virtualization/Projection/MockGitIndexProjection.cs +++ b/GVFS/GVFS.UnitTests/Mock/Virtualization/Projection/MockGitIndexProjection.cs @@ -35,6 +35,7 @@ public MockGitIndexProjection(IEnumerable projectedFiles) } this.PlaceholdersCreated = new ConcurrentHashSet(); + this.ExpandedFolders = new ConcurrentHashSet(); this.MockFileModes = new ConcurrentDictionary(); this.unblockGetProjectedItems = new ManualResetEvent(true); @@ -53,6 +54,8 @@ public MockGitIndexProjection(IEnumerable projectedFiles) public ConcurrentHashSet PlaceholdersCreated { get; } + public ConcurrentHashSet ExpandedFolders { get; } + public ConcurrentDictionary MockFileModes { get; } public bool ThrowOperationCanceledExceptionOnProjectionRequest { get; set; } @@ -231,6 +234,11 @@ public override ProjectedFileInfo GetProjectedFileInfo( return null; } + public override void OnPlaceholderFolderExpanded(string relativePath) + { + this.ExpandedFolders.Add(relativePath); + } + public override void OnPlaceholderFileCreated(string virtualPath, string sha) { this.PlaceholdersCreated.Add(virtualPath); diff --git a/GVFS/GVFS.UnitTests/Platform.Mac/MacFileSystemVirtualizerTests.cs b/GVFS/GVFS.UnitTests/Platform.Mac/MacFileSystemVirtualizerTests.cs index 8b430267d..675ae8a93 100644 --- a/GVFS/GVFS.UnitTests/Platform.Mac/MacFileSystemVirtualizerTests.cs +++ b/GVFS/GVFS.UnitTests/Platform.Mac/MacFileSystemVirtualizerTests.cs @@ -1,6 +1,7 @@ using GVFS.Common; using GVFS.Platform.Mac; using GVFS.Tests.Should; +using GVFS.UnitTests.Category; using GVFS.UnitTests.Mock.Git; using GVFS.UnitTests.Mock.Mac; using GVFS.UnitTests.Mock.Virtualization.Background; @@ -228,6 +229,7 @@ public void OnEnumerateDirectoryReturnsSuccessWhenResultsInMemory() mockVirtualization.OnEnumerateDirectory(1, "test", triggeringProcessId: 1, triggeringProcessName: "UnitTests").ShouldEqual(Result.Success); mockVirtualization.CreatedPlaceholders.ShouldContain( kvp => kvp.Key.Equals(Path.Combine("test", "test.txt"), StringComparison.OrdinalIgnoreCase) && kvp.Value == FileMode644); + gitIndexProjection.ExpandedFolders.ShouldMatchInOrder("test"); fileSystemCallbacks.Stop(); } } @@ -311,6 +313,7 @@ public void OnGetFileStreamReturnsSuccessWhenFileStreamAvailable() } [TestCase] + [Category(CategoryConstants.ExceptionExpected)] public void OnGetFileStreamReturnsErrorWhenWriteFileContentsFails() { using (MockBackgroundFileSystemTaskRunner backgroundTaskRunner = new MockBackgroundFileSystemTaskRunner()) diff --git a/GVFS/GVFS.Virtualization/FileSystem/FileSystemVirtualizer.cs b/GVFS/GVFS.Virtualization/FileSystem/FileSystemVirtualizer.cs index 0bfc60e0d..4f811ce76 100644 --- a/GVFS/GVFS.Virtualization/FileSystem/FileSystemVirtualizer.cs +++ b/GVFS/GVFS.Virtualization/FileSystem/FileSystemVirtualizer.cs @@ -13,6 +13,8 @@ public abstract class FileSystemVirtualizer : IDisposable { public const byte PlaceholderVersion = 1; + protected static readonly byte[] FolderContentId = Encoding.Unicode.GetBytes(GVFSConstants.AllZeroSha); + protected static readonly GitCommandLineParser.Verbs CanCreatePlaceholderVerbs = GitCommandLineParser.Verbs.AddOrStage | GitCommandLineParser.Verbs.Move | GitCommandLineParser.Verbs.Status; @@ -93,6 +95,9 @@ public void PrepareToStop() public abstract FileSystemResult DeleteFile(string relativePath, UpdatePlaceholderType updateFlags, out UpdateFailureReason failureReason); + public abstract FileSystemResult WritePlaceholderFile(string relativePath, long endOfFile, string sha); + public abstract FileSystemResult WritePlaceholderDirectory(string relativePath); + public abstract FileSystemResult UpdatePlaceholderIfNeeded( string relativePath, DateTime creationTime, diff --git a/GVFS/GVFS.Virtualization/FileSystemCallbacks.cs b/GVFS/GVFS.Virtualization/FileSystemCallbacks.cs index 28dd595f6..ce1b8afe4 100644 --- a/GVFS/GVFS.Virtualization/FileSystemCallbacks.cs +++ b/GVFS/GVFS.Virtualization/FileSystemCallbacks.cs @@ -471,6 +471,11 @@ public void OnPlaceholderFolderCreated(string relativePath) this.GitIndexProjection.OnPlaceholderFolderCreated(relativePath); } + public void OnPlaceholderFolderExpanded(string relativePath) + { + this.GitIndexProjection.OnPlaceholderFolderExpanded(relativePath); + } + public FileProperties GetLogsHeadFileProperties() { // Use a temporary FileProperties in case another thread sets this.logsHeadFileProperties before this diff --git a/GVFS/GVFS.Virtualization/Projection/GitIndexProjection.cs b/GVFS/GVFS.Virtualization/Projection/GitIndexProjection.cs index c807016a8..ba40f28b6 100644 --- a/GVFS/GVFS.Virtualization/Projection/GitIndexProjection.cs +++ b/GVFS/GVFS.Virtualization/Projection/GitIndexProjection.cs @@ -317,12 +317,17 @@ public void ClearNegativePathCacheIfPollutedByGit() public void OnPlaceholderFolderCreated(string virtualPath) { - this.placeholderList.AddAndFlush(virtualPath, GVFSConstants.AllZeroSha); + this.placeholderList.AddAndFlushFolder(virtualPath, isExpanded: false); + } + + public virtual void OnPlaceholderFolderExpanded(string relativePath) + { + this.placeholderList.AddAndFlushFolder(relativePath, isExpanded: true); } public virtual void OnPlaceholderFileCreated(string virtualPath, string sha) { - this.placeholderList.AddAndFlush(virtualPath, sha); + this.placeholderList.AddAndFlushFile(virtualPath, sha); } public virtual bool TryGetProjectedItemsFromMemory(string folderPath, out List projectedItems) @@ -894,7 +899,11 @@ private bool TryGetOrAddFolderDataFromCache( { EventMetadata metadata = CreateEventMetadata(); metadata.Add("folderPath", folderPath); - this.context.Tracer.RelatedWarning(metadata, "GitIndexProjection_TryGetOrAddFolderDataFromCacheFoundFile: Found a file when expecting a folder"); + metadata.Add(TracingConstants.MessageKey.InfoMessage, "Found file at path"); + this.context.Tracer.RelatedEvent( + EventLevel.Informational, + $"{nameof(this.TryGetOrAddFolderDataFromCache)}_FileAtPath", + metadata); folderPath = null; return false; @@ -1078,30 +1087,106 @@ private void UpdatePlaceholders() { this.ClearUpdatePlaceholderErrors(); - List placeholderListCopy = this.placeholderList.GetAllEntries(); + List placeholderFilesListCopy; + List placeholderFoldersListCopy; + this.placeholderList.GetAllEntries(out placeholderFilesListCopy, out placeholderFoldersListCopy); + EventMetadata metadata = new EventMetadata(); - metadata.Add("Count", placeholderListCopy.Count); + metadata.Add("File placeholder count", placeholderFilesListCopy.Count); + metadata.Add("Folder placeholders count", placeholderFoldersListCopy.Count); + using (ITracer activity = this.context.Tracer.StartActivity("UpdatePlaceholders", EventLevel.Informational, metadata)) { + int minItemsPerThread = 10; + int numThreads = Math.Max(8, Environment.ProcessorCount); + numThreads = Math.Min(numThreads, placeholderFilesListCopy.Count / minItemsPerThread); + numThreads = Math.Max(numThreads, 1); + ConcurrentHashSet folderPlaceholdersToKeep = new ConcurrentHashSet(); - ConcurrentBag updatedPlaceholderList = new ConcurrentBag(); + + // updatedPlaceholderDictionary and updatedPlaceholderBag are mutually exclusive. + // - On platforms that expand on enumeration: updatedPlaceholderDictionary is used (required for ReExpandFolder) + // - On platforms that do not expand on enumeration: updatedPlaceholderBag is used (for speed) + ConcurrentDictionary updatedPlaceholderDictionary; + ConcurrentBag updatedPlaceholderBag; + Action addPlaceholderToUpdatedPlaceholders; + if (GVFSPlatform.Instance.KernelDriver.EnumerationExpandsDirectories) + { + updatedPlaceholderDictionary = new ConcurrentDictionary( + concurrencyLevel: numThreads, + capacity: placeholderFilesListCopy.Count + placeholderFoldersListCopy.Count, + comparer: StringComparer.Ordinal); + updatedPlaceholderBag = null; + addPlaceholderToUpdatedPlaceholders = (data) => updatedPlaceholderDictionary.TryAdd(data.Path, data); + } + else + { + updatedPlaceholderDictionary = null; + updatedPlaceholderBag = new ConcurrentBag(); + addPlaceholderToUpdatedPlaceholders = (data) => updatedPlaceholderBag.Add(data); + } + this.ProcessListOnThreads( - placeholderListCopy.Where(x => !x.IsFolder).ToList(), + numThreads, + placeholderFilesListCopy, (placeholderBatch, start, end, blobSizesConnection, availableSizes) => this.BatchPopulateMissingSizesFromRemote(blobSizesConnection, placeholderBatch, start, end, availableSizes), (placeholder, blobSizesConnection, availableSizes) => - this.UpdateOrDeleteFilePlaceholder(blobSizesConnection, placeholder, updatedPlaceholderList, folderPlaceholdersToKeep, availableSizes)); + this.UpdateOrDeleteFilePlaceholder(blobSizesConnection, placeholder, addPlaceholderToUpdatedPlaceholders, folderPlaceholdersToKeep, availableSizes)); this.blobSizes.Flush(); - // Waiting for the UpdateOrDeleteFilePlaceholder to fill the folderPlaceholdersToKeep before trying to remove folder placeholders - // so that we don't try to delete a folder placeholder that has file placeholders and just fails - foreach (PlaceholderListDatabase.PlaceholderData folderPlaceholder in placeholderListCopy.Where(x => x.IsFolder).OrderByDescending(x => x.Path)) + using (BlobSizes.BlobSizesConnection blobSizesConnection = this.blobSizes.CreateConnection()) + { + // A hash of the folder placeholders is only required if the platform expands directories + HashSet folderPlaceholders = + GVFSPlatform.Instance.KernelDriver.EnumerationExpandsDirectories ? + new HashSet(placeholderFoldersListCopy.Select(x => x.Path), StringComparer.OrdinalIgnoreCase) : + null; + + // Order the folders in decscending order so that we walk the tree from bottom up (ensuring child folders are deleted before + // their parents) + foreach (PlaceholderListDatabase.PlaceholderData folderPlaceholder in placeholderFoldersListCopy.OrderByDescending(x => x.Path)) + { + // Remove folder placeholders before re-expansion to ensure that projection changes that convert a folder to a file work + // properly + if (!this.RemoveFolderPlaceholderIfEmpty(folderPlaceholder, addPlaceholderToUpdatedPlaceholders, folderPlaceholdersToKeep)) + { + if (GVFSPlatform.Instance.KernelDriver.EnumerationExpandsDirectories && folderPlaceholder.IsExpandedFolder) + { + if (updatedPlaceholderDictionary == null) + { + throw new InvalidOperationException( + $"{nameof(updatedPlaceholderDictionary)} must be used when enumeration expands directories"); + } + + this.ReExpandFolder(blobSizesConnection, folderPlaceholder.Path, updatedPlaceholderDictionary, folderPlaceholders); + } + } + } + } + + if (GVFSPlatform.Instance.KernelDriver.EnumerationExpandsDirectories) + { + if (updatedPlaceholderBag != null) + { + throw new InvalidOperationException( + $"{nameof(updatedPlaceholderBag)} should only be used when enumeration does not expand directories"); + } + + this.placeholderList.WriteAllEntriesAndFlush(updatedPlaceholderDictionary.Values); + } + else { - this.TryRemoveFolderPlaceholder(folderPlaceholder, updatedPlaceholderList, folderPlaceholdersToKeep); + if (updatedPlaceholderDictionary != null) + { + throw new InvalidOperationException( + $"{nameof(updatedPlaceholderDictionary)} should only be used when enumeration expands directories"); + } + + this.placeholderList.WriteAllEntriesAndFlush(updatedPlaceholderBag); } - this.placeholderList.WriteAllEntriesAndFlush(updatedPlaceholderList); this.repoMetadata.SetPlaceholdersNeedUpdate(false); TimeSpan duration = activity.Stop(null); @@ -1110,13 +1195,11 @@ private void UpdatePlaceholders() } private void ProcessListOnThreads( + int numThreads, List list, Action, int, int, BlobSizes.BlobSizesConnection, Dictionary> preProcessBatch, Action> processItem) { - int minItemsPerThread = 10; - int numThreads = Math.Max(8, Environment.ProcessorCount); - numThreads = Math.Min(numThreads, list.Count / minItemsPerThread); if (numThreads > 1) { Thread[] processThreads = new Thread[numThreads]; @@ -1257,15 +1340,106 @@ private string GetNewProjectedShaForPlaceholder(string path) return null; } - private void TryRemoveFolderPlaceholder( + private void ReExpandFolder( + BlobSizes.BlobSizesConnection blobSizesConnection, + string relativeFolderPath, + ConcurrentDictionary updatedPlaceholderList, + HashSet existingFolderPlaceholders) + { + FolderData folderData; + if (!this.TryGetOrAddFolderDataFromCache(relativeFolderPath, out folderData)) + { + // Folder is no longer in the projection + return; + } + + // TODO(Mac): Issue #255, batch file sizes up-front for the new placeholders written by ReExpandFolder + folderData.PopulateSizes( + this.context.Tracer, + this.gitObjects, + blobSizesConnection, + availableSizes: null, + cancellationToken: CancellationToken.None); + + for (int i = 0; i < folderData.ChildEntries.Count; i++) + { + FolderEntryData childEntry = folderData.ChildEntries[i]; + string childRelativePath; + if (relativeFolderPath.Length == 0) + { + childRelativePath = childEntry.Name.GetString(); + } + else + { + childRelativePath = relativeFolderPath + Path.DirectorySeparatorChar + childEntry.Name.GetString(); + } + + // TODO(Mac): Issue #245, handle failures of WritePlaceholderDirectory and WritePlaceholderFile + if (childEntry.IsFolder) + { + if (!existingFolderPlaceholders.Contains(childRelativePath)) + { + this.fileSystemVirtualizer.WritePlaceholderDirectory(childRelativePath); + updatedPlaceholderList.TryAdd( + childRelativePath, + new PlaceholderListDatabase.PlaceholderData(childRelativePath, PlaceholderListDatabase.PartialFolderValue)); + } + } + else + { + if (!updatedPlaceholderList.ContainsKey(childRelativePath)) + { + FileData childFileData = childEntry as FileData; + string sha = childFileData.Sha.ToString(); + + this.fileSystemVirtualizer.WritePlaceholderFile(childRelativePath, childFileData.Size, sha); + updatedPlaceholderList.TryAdd( + childRelativePath, + new PlaceholderListDatabase.PlaceholderData(childRelativePath, sha)); + } + } + } + } + + /// + /// Removes the folder placeholder from disk if it's empty. + /// + /// + /// trueIf the folder placeholder was deleted + /// falseIf RemoveFolderPlaceholderIfEmpty did not attempt to remove the folder placeholder + /// + /// + /// If the platform expands on enumeration the folder will only be removed if it's not in the projection + /// + private bool RemoveFolderPlaceholderIfEmpty( PlaceholderListDatabase.PlaceholderData placeholder, - ConcurrentBag updatedPlaceholderList, + Action addPlaceholderToUpdatedPlaceholders, ConcurrentHashSet folderPlaceholdersToKeep) { if (folderPlaceholdersToKeep.Contains(placeholder.Path)) { - updatedPlaceholderList.Add(placeholder); - return; + addPlaceholderToUpdatedPlaceholders(placeholder); + return false; + } + + if (GVFSPlatform.Instance.KernelDriver.EnumerationExpandsDirectories) + { + // If enumeration expands directories we should leave folder placeholders + // that are still in the projection on disk (they might still be physically empty + // on disk if they've not been expanded). + // + // If enumeration does not expand directories there is no harm in deleting empty + // folder placeholders that are in the projection as they will be re-projected during + // enumeration. Additionally, there may be folder tombstones on disk that need to be + // cleaned up (e.g. git might have deleted a folder placeholder that was not in + // ModifiedPaths.dat, resulting in a tombstone getting created). + + FolderData folderData; + if (this.TryGetOrAddFolderDataFromCache(placeholder.Path, out folderData)) + { + addPlaceholderToUpdatedPlaceholders(placeholder); + return false; + } } UpdateFailureReason failureReason = UpdateFailureReason.NoFailure; @@ -1276,7 +1450,7 @@ private void TryRemoveFolderPlaceholder( break; case FSResult.DirectoryNotEmpty: - updatedPlaceholderList.Add(new PlaceholderListDatabase.PlaceholderData(placeholder.Path, GVFSConstants.AllZeroSha)); + addPlaceholderToUpdatedPlaceholders(placeholder); break; case FSResult.FileOrPathNotFound: @@ -1288,15 +1462,21 @@ private void TryRemoveFolderPlaceholder( metadata.Add("result.Result", result.Result.ToString()); metadata.Add("result.RawResult", result.RawResult); metadata.Add("UpdateFailureCause", failureReason.ToString()); - this.context.Tracer.RelatedEvent(EventLevel.Informational, nameof(this.TryRemoveFolderPlaceholder) + "_DeleteFileFailure", metadata); + this.context.Tracer.RelatedEvent(EventLevel.Informational, nameof(this.RemoveFolderPlaceholderIfEmpty) + "_DeleteFileFailure", metadata); + + // TODO(Mac): Issue #245, handle failures DeleteFile on Mac. If we don't do anything we could leave an untracked folder + // placeholder on disk that will never be updated by Git or VFSForGit + break; } + + return true; } private void UpdateOrDeleteFilePlaceholder( BlobSizes.BlobSizesConnection blobSizesConnection, PlaceholderListDatabase.PlaceholderData placeholder, - ConcurrentBag updatedPlaceholderList, + Action addPlaceholderToUpdatedPlaceholders, ConcurrentHashSet folderPlaceholdersToKeep, Dictionary availableSizes) { @@ -1313,7 +1493,7 @@ private void UpdateOrDeleteFilePlaceholder( placeholder, string.Empty, result, - updatedPlaceholderList, + addPlaceholderToUpdatedPlaceholders, failureReason, parentKey, folderPlaceholdersToKeep, @@ -1356,7 +1536,7 @@ private void UpdateOrDeleteFilePlaceholder( placeholder, projectedSha, result, - updatedPlaceholderList, + addPlaceholderToUpdatedPlaceholders, failureReason, parentKey, folderPlaceholdersToKeep, @@ -1364,7 +1544,7 @@ private void UpdateOrDeleteFilePlaceholder( } else { - updatedPlaceholderList.Add(placeholder); + addPlaceholderToUpdatedPlaceholders(placeholder); this.AddParentFoldersToListToKeep(parentKey, folderPlaceholdersToKeep); } } @@ -1374,7 +1554,7 @@ private void ProcessGvUpdateDeletePlaceholderResult( PlaceholderListDatabase.PlaceholderData placeholder, string projectedSha, FileSystemResult result, - ConcurrentBag updatedPlaceholderList, + Action addPlaceholderToUpdatedPlaceholders, UpdateFailureReason failureReason, string parentKey, ConcurrentHashSet folderPlaceholdersToKeep, @@ -1386,7 +1566,7 @@ private void ProcessGvUpdateDeletePlaceholderResult( case FSResult.Ok: if (!deleteOperation) { - updatedPlaceholderList.Add(new PlaceholderListDatabase.PlaceholderData(placeholder.Path, projectedSha)); + addPlaceholderToUpdatedPlaceholders(new PlaceholderListDatabase.PlaceholderData(placeholder.Path, projectedSha)); this.AddParentFoldersToListToKeep(parentKey, folderPlaceholdersToKeep); } diff --git a/ProjFS.Mac/PrjFSKext/PrjFSKext/VirtualizationRoots.cpp b/ProjFS.Mac/PrjFSKext/PrjFSKext/VirtualizationRoots.cpp index 48c98edb8..d37281361 100644 --- a/ProjFS.Mac/PrjFSKext/PrjFSKext/VirtualizationRoots.cpp +++ b/ProjFS.Mac/PrjFSKext/PrjFSKext/VirtualizationRoots.cpp @@ -16,7 +16,7 @@ static RWLock s_rwLock = {}; // Arbitrary choice, but prevents user space attacker from causing // allocation of too much wired kernel memory. -static const size_t MaxVirtualizationRoots = 64; +static const size_t MaxVirtualizationRoots = 128; static VirtualizationRoot s_virtualizationRoots[MaxVirtualizationRoots] = {}; diff --git a/ProjFS.Mac/PrjFSLib/PrjFSLib.cpp b/ProjFS.Mac/PrjFSLib/PrjFSLib.cpp index 2fec82b75..a858a7fe3 100644 --- a/ProjFS.Mac/PrjFSLib/PrjFSLib.cpp +++ b/ProjFS.Mac/PrjFSLib/PrjFSLib.cpp @@ -411,7 +411,7 @@ PrjFS_Result PrjFS_DeleteFile( char fullPath[PrjFSMaxPath]; CombinePaths(s_virtualizationRootFullPath.c_str(), relativePath, fullPath); - if (0 != unlink(fullPath)) + if (0 != remove(fullPath)) { switch(errno) {