From 0d3f0901fb2d59df451157cd525f74f056405771 Mon Sep 17 00:00:00 2001 From: William Baker Date: Mon, 13 Aug 2018 09:57:23 -0700 Subject: [PATCH 001/272] Mac: File modified notifications --- GVFS/GVFS.Common/FileBasedCollection.cs | 1 + ...xtensions.cs => StreamWriterExtensions.cs} | 2 +- .../EnlistmentPerFixture/GitFilesTests.cs | 14 +- .../WorkingDirectoryTests.cs | 35 +- .../Tests/GitCommands/AddStageTests.cs | 2 + .../Tools/GVFSFunctionalTestEnlistment.cs | 26 +- GVFS/GVFS.Hooks/GVFS.Hooks.Mac.csproj | 4 +- GVFS/GVFS.Hooks/GVFS.Hooks.Windows.csproj | 4 +- .../MacFileSystemVirtualizer.cs | 40 +- .../FileSystemCallbacks.cs | 1 + .../MacFileSystemVirtualizer.cs | 12 +- .../WindowsFileSystemVirtualizer.cs | 18 +- .../PrjFSKext/PrjFSKext/KauthHandler.cpp | 444 ++++++++++++------ .../PrjFSKext/VirtualizationRoots.cpp | 1 + ProjFS.Mac/PrjFSKext/public/Message.h | 2 + .../PrjFSLib.Mac.Managed/CallbackDelegates.cs | 16 +- .../PrjFSLib.Mac.Managed/Interop/Callbacks.cs | 2 +- .../PrjFSLib.Mac.Managed/Interop/PrjFSLib.cs | 17 +- .../PrjFSLib.Mac.Managed.csproj | 5 + .../VirtualizationInstance.cs | 45 +- ProjFS.Mac/PrjFSLib/PrjFSLib.cpp | 63 ++- 21 files changed, 546 insertions(+), 208 deletions(-) rename GVFS/GVFS.Common/NamedPipes/{NamedPipeStreamWriterExtensions.cs => StreamWriterExtensions.cs} (84%) diff --git a/GVFS/GVFS.Common/FileBasedCollection.cs b/GVFS/GVFS.Common/FileBasedCollection.cs index 3600fbbfa..bb15a7571 100644 --- a/GVFS/GVFS.Common/FileBasedCollection.cs +++ b/GVFS/GVFS.Common/FileBasedCollection.cs @@ -390,6 +390,7 @@ private void WriteToDisk(string value) throw new InvalidOperationException(nameof(this.WriteToDisk) + " requires that collectionAppendsDirectlyToFile be true"); } + // TODO(Mac): Decide if we want to stick with \r\n on all platforms, move them all to \n, or use a mix byte[] bytes = Encoding.UTF8.GetBytes(value + "\r\n"); lock (this.fileLock) { diff --git a/GVFS/GVFS.Common/NamedPipes/NamedPipeStreamWriterExtensions.cs b/GVFS/GVFS.Common/NamedPipes/StreamWriterExtensions.cs similarity index 84% rename from GVFS/GVFS.Common/NamedPipes/NamedPipeStreamWriterExtensions.cs rename to GVFS/GVFS.Common/NamedPipes/StreamWriterExtensions.cs index 0de90561e..58db185d6 100644 --- a/GVFS/GVFS.Common/NamedPipes/NamedPipeStreamWriterExtensions.cs +++ b/GVFS/GVFS.Common/NamedPipes/StreamWriterExtensions.cs @@ -2,7 +2,7 @@ namespace GVFS.Common.NamedPipes { - public static class NamedPipeStreamWriterExtensions + public static class StreamWriterExtensions { public const int Foo = 0; diff --git a/GVFS/GVFS.FunctionalTests/Tests/EnlistmentPerFixture/GitFilesTests.cs b/GVFS/GVFS.FunctionalTests/Tests/EnlistmentPerFixture/GitFilesTests.cs index c9caaf65c..c85238ebb 100644 --- a/GVFS/GVFS.FunctionalTests/Tests/EnlistmentPerFixture/GitFilesTests.cs +++ b/GVFS/GVFS.FunctionalTests/Tests/EnlistmentPerFixture/GitFilesTests.cs @@ -129,10 +129,11 @@ public void CaseOnlyRenameOfNewFolderKeepsExcludeEntries() } [TestCase, Order(6)] + [Category(Categories.Mac.M2)] public void ReadingFileDoesNotUpdateIndexOrSparseCheckout() { string gitFileToCheck = "GVFS/GVFS.FunctionalTests/Category/CategoryConstants.cs"; - string virtualFile = Path.Combine(this.Enlistment.RepoRoot, gitFileToCheck.Replace('/', '\\')); + string virtualFile = this.Enlistment.GetVirtualPathTo(gitFileToCheck); ProcessResult initialResult = GitProcess.InvokeProcess(this.Enlistment.RepoRoot, "ls-files --debug -svmodc " + gitFileToCheck); initialResult.ShouldNotBeNull(); initialResult.Output.ShouldNotBeNull(); @@ -158,9 +159,8 @@ public void ReadingFileDoesNotUpdateIndexOrSparseCheckout() [TestCase, Order(7)] public void ModifiedFileWillGetSkipworktreeBitCleared() { - string fileToTest = "GVFS\\GVFS.Common\\RetryWrapper.cs"; - string fileToCreate = Path.Combine(this.Enlistment.RepoRoot, fileToTest); - string gitFileToTest = fileToTest.Replace('\\', '/'); + string gitFileToTest = "GVFS/GVFS.Common/RetryWrapper.cs"; + string fileToCreate = this.Enlistment.GetVirtualPathTo(gitFileToTest); this.VerifyWorktreeBit(gitFileToTest, LsFilesStatus.SkipWorktree); ManualResetEventSlim resetEvent = GitHelpers.AcquireGVFSLock(this.Enlistment, out _); @@ -286,10 +286,11 @@ public void FileRenamedOutOfRepoAddedToSparseCheckoutAndSkipWorktreeBitCleared() } [TestCase, Order(13)] + [Category(Categories.Mac.M2)] public void OverwrittenFileAddedToSparseCheckoutAndSkipWorktreeBitCleared() { string fileToOverwriteEntry = "Test_EPF_WorkingDirectoryTests/1/2/3/4/ReadDeepProjectedFile.cpp"; - string fileToOverwriteVirtualPath = this.Enlistment.GetVirtualPathTo("Test_EPF_WorkingDirectoryTests\\1\\2\\3\\4\\ReadDeepProjectedFile.cpp"); + string fileToOverwriteVirtualPath = this.Enlistment.GetVirtualPathTo(fileToOverwriteEntry); this.VerifyWorktreeBit(fileToOverwriteEntry, LsFilesStatus.SkipWorktree); string testContents = "Test contents for FileRenamedOutOfRepoWillBeAddedToSparseCheckoutAndHaveSkipWorktreeBitCleared"; @@ -299,7 +300,8 @@ public void OverwrittenFileAddedToSparseCheckoutAndSkipWorktreeBitCleared() fileToOverwriteVirtualPath.ShouldBeAFile(this.fileSystem).WithContents(testContents); - GVFSHelpers.ModifiedPathsShouldContain(this.fileSystem, this.Enlistment.DotGVFSRoot, fileToOverwriteEntry + Environment.NewLine); + // TODO(Mac): Switch from "\r\n" to Environment.NewLine or "\n" depending on how FileBasedCollection is updated + GVFSHelpers.ModifiedPathsShouldContain(this.fileSystem, this.Enlistment.DotGVFSRoot, fileToOverwriteEntry + "\r\n"); // Verify skip-worktree cleared this.VerifyWorktreeBit(fileToOverwriteEntry, LsFilesStatus.Cached); diff --git a/GVFS/GVFS.FunctionalTests/Tests/EnlistmentPerFixture/WorkingDirectoryTests.cs b/GVFS/GVFS.FunctionalTests/Tests/EnlistmentPerFixture/WorkingDirectoryTests.cs index 35470fd4f..82eba5814 100644 --- a/GVFS/GVFS.FunctionalTests/Tests/EnlistmentPerFixture/WorkingDirectoryTests.cs +++ b/GVFS/GVFS.FunctionalTests/Tests/EnlistmentPerFixture/WorkingDirectoryTests.cs @@ -60,11 +60,9 @@ public void ProjectedFileHasExpectedContents() } [TestCase, Order(2)] - [Category(Categories.Mac.M3)] public void StreamAccessReadWriteMemoryMappedProjectedFile() { - string filename = @"Test_EPF_WorkingDirectoryTests\StreamAccessReadWriteMemoryMappedProjectedFile.cs"; - string fileVirtualPath = this.Enlistment.GetVirtualPathTo(filename); + string fileVirtualPath = this.Enlistment.GetVirtualPathTo("Test_EPF_WorkingDirectoryTests", "StreamAccessReadWriteMemoryMappedProjectedFile.cs"); string contents = fileVirtualPath.ShouldBeAFile(this.fileSystem).WithContents(); StringBuilder contentsBuilder = new StringBuilder(contents); @@ -124,11 +122,9 @@ public void StreamAccessReadWriteMemoryMappedProjectedFile() } [TestCase, Order(3)] - [Category(Categories.Mac.M3)] public void RandomAccessReadWriteMemoryMappedProjectedFile() { - string filename = @"Test_EPF_WorkingDirectoryTests\RandomAccessReadWriteMemoryMappedProjectedFile.cs"; - string fileVirtualPath = this.Enlistment.GetVirtualPathTo(filename); + string fileVirtualPath = this.Enlistment.GetVirtualPathTo("Test_EPF_WorkingDirectoryTests", "RandomAccessReadWriteMemoryMappedProjectedFile.cs"); string contents = fileVirtualPath.ShouldBeAFile(this.fileSystem).WithContents(); StringBuilder contentsBuilder = new StringBuilder(contents); @@ -185,11 +181,9 @@ public void RandomAccessReadWriteMemoryMappedProjectedFile() } [TestCase, Order(4)] - [Category(Categories.Mac.M3)] public void StreamAndRandomAccessReadWriteMemoryMappedProjectedFile() { - string filename = @"Test_EPF_WorkingDirectoryTests\StreamAndRandomAccessReadWriteMemoryMappedProjectedFile.cs"; - string fileVirtualPath = this.Enlistment.GetVirtualPathTo(filename); + string fileVirtualPath = this.Enlistment.GetVirtualPathTo("Test_EPF_WorkingDirectoryTests", "StreamAndRandomAccessReadWriteMemoryMappedProjectedFile.cs"); StringBuilder contentsBuilder = new StringBuilder(); @@ -353,11 +347,9 @@ public void HydratingNestedFileUsesNameCaseFromRepo() } [TestCase, Order(9)] - [Category(Categories.Mac.M3)] public void WriteToHydratedFileAfterRemount() { - string fileName = "Test_EPF_WorkingDirectoryTests\\WriteToHydratedFileAfterRemount.cpp"; - string virtualFilePath = this.Enlistment.GetVirtualPathTo(fileName); + string virtualFilePath = this.Enlistment.GetVirtualPathTo("Test_EPF_WorkingDirectoryTests", "WriteToHydratedFileAfterRemount.cpp"); string fileContents = virtualFilePath.ShouldBeAFile(this.fileSystem).WithContents(); // Remount @@ -371,7 +363,7 @@ public void WriteToHydratedFileAfterRemount() [TestCase, Order(10)] public void ReadDeepProjectedFile() - { + { string testFilePath = Path.Combine("Test_EPF_WorkingDirectoryTests", "1", "2", "3", "4", "ReadDeepProjectedFile.cpp"); this.Enlistment.GetVirtualPathTo(testFilePath).ShouldBeAFile(this.fileSystem).WithContents(TestFileContents); } @@ -407,8 +399,8 @@ public void FolderContentsProjectedAfterFolderCreateAndCheckout() { string folderName = "GVFlt_MultiThreadTest"; - // 54ea499de78eafb4dfd30b90e0bd4bcec26c4349 did not have the folder GVFlt_MultiThreadTest - GitProcess.InvokeProcess(this.Enlistment.RepoRoot, "checkout 54ea499de78eafb4dfd30b90e0bd4bcec26c4349"); + // 575d597cf09b2cd1c0ddb4db21ce96979010bbcb did not have the folder GVFlt_MultiThreadTest + GitProcess.InvokeProcess(this.Enlistment.RepoRoot, "checkout 575d597cf09b2cd1c0ddb4db21ce96979010bbcb"); // Confirm that no other test has created GVFlt_MultiThreadTest or put it in the modified files GVFSHelpers.ModifiedPathsShouldNotContain(this.fileSystem, this.Enlistment.DotGVFSRoot, folderName); @@ -417,10 +409,10 @@ public void FolderContentsProjectedAfterFolderCreateAndCheckout() virtualFolderPath.ShouldNotExistOnDisk(this.fileSystem); this.fileSystem.CreateDirectory(virtualFolderPath); - // b3ddcf43b997cba3fbf9d2341b297e22bf48601a was the commit prior to deleting GVFLT_MultiThreadTest + // b5fd7d23706a18cff3e2b8225588d479f7e51138 was the commit prior to deleting GVFLT_MultiThreadTest // 692765: Note that test also validates case insensitivity as GVFlt_MultiThreadTest is named GVFLT_MultiThreadTest // in this commit - GitProcess.InvokeProcess(this.Enlistment.RepoRoot, "checkout b3ddcf43b997cba3fbf9d2341b297e22bf48601a"); + GitProcess.InvokeProcess(this.Enlistment.RepoRoot, "checkout b5fd7d23706a18cff3e2b8225588d479f7e51138"); this.Enlistment.GetVirtualPathTo(folderName + "\\OpenForReadsSameTime\\test").ShouldBeAFile(this.fileSystem).WithContents("123 \r\n"); this.Enlistment.GetVirtualPathTo(folderName + "\\OpenForWritesSameTime\\test").ShouldBeAFile(this.fileSystem).WithContents("123 \r\n"); @@ -431,8 +423,8 @@ public void FolderContentsProjectedAfterFolderCreateAndCheckout() [Category(Categories.Mac.M3)] public void FolderContentsCorrectAfterCreateNewFolderRenameAndCheckoutCommitWithSameFolder() { - // 3a55d3b760c87642424e834228a3408796501e7c is the commit prior to adding Test_EPF_MoveRenameFileTests - GitProcess.InvokeProcess(this.Enlistment.RepoRoot, "checkout 3a55d3b760c87642424e834228a3408796501e7c"); + // 1ca414ced40f64bf94fc6c7f885974708bc600be is the commit prior to adding Test_EPF_MoveRenameFileTests + GitProcess.InvokeProcess(this.Enlistment.RepoRoot, "checkout 1ca414ced40f64bf94fc6c7f885974708bc600be"); // Confirm that no other test has created this folder or put it in the modified files string folderName = "Test_EPF_MoveRenameFileTests"; @@ -457,9 +449,7 @@ public void FolderContentsCorrectAfterCreateNewFolderRenameAndCheckoutCommitWith (folder + @"\MoveUnhydratedFileToDotGitFolder\Program.cs").ShouldBeAFile(this.fileSystem).WithContents(MoveRenameFileTests.TestFileContents); } - // TODO(Mac) This test is technically part of M2, but we need further investigation of why this test fails on build agents, but not on dev machines. [TestCase, Order(15)] - [Category(Categories.Mac.M3)] public void FilterNonUTF8FileName() { string encodingFilename = "ريلٌأكتوبرûمارسأغسطسºٰٰۂْٗ۵ريلٌأك.txt"; @@ -594,8 +584,7 @@ private void FolderEnumerationShouldHaveSingleEntry(string folderVirtualPath, st } folderEntries.Count().ShouldEqual(1); - FileSystemInfo singleEntry = folderEntries.First(); - singleEntry.Name.ShouldEqual(expectedEntryName, $"Actual name: {singleEntry.Name} does not equal expected name {expectedEntryName}"); + folderEntries.ShouldContain(file => file.Name.Equals(expectedEntryName)); } private void EnumerateAndReadShouldNotChangeEnumerationOrder(string folderRelativePath) diff --git a/GVFS/GVFS.FunctionalTests/Tests/GitCommands/AddStageTests.cs b/GVFS/GVFS.FunctionalTests/Tests/GitCommands/AddStageTests.cs index 28d5b2faf..eb9f74ab2 100644 --- a/GVFS/GVFS.FunctionalTests/Tests/GitCommands/AddStageTests.cs +++ b/GVFS/GVFS.FunctionalTests/Tests/GitCommands/AddStageTests.cs @@ -13,6 +13,7 @@ public AddStageTests() : base(enlistmentPerTest: false) } [TestCase, Order(1)] + [Category(Categories.Mac.M2)] public void AddBasicTest() { this.EditFile("Readme.md", "Some new content."); @@ -21,6 +22,7 @@ public void AddBasicTest() } [TestCase, Order(2)] + [Category(Categories.Mac.M2)] public void StageBasicTest() { this.EditFile("AuthoringTests.md", "Some new content."); diff --git a/GVFS/GVFS.FunctionalTests/Tools/GVFSFunctionalTestEnlistment.cs b/GVFS/GVFS.FunctionalTests/Tools/GVFSFunctionalTestEnlistment.cs index 1326cfd3b..a6f4f0270 100644 --- a/GVFS/GVFS.FunctionalTests/Tools/GVFSFunctionalTestEnlistment.cs +++ b/GVFS/GVFS.FunctionalTests/Tools/GVFSFunctionalTestEnlistment.cs @@ -2,6 +2,7 @@ using GVFS.FunctionalTests.Should; using GVFS.FunctionalTests.Tests; using GVFS.Tests.Should; +using NUnit.Framework; using System; using System.Collections.Generic; using System.IO; @@ -233,9 +234,30 @@ public void UnmountAndDeleteAll() this.DeleteEnlistment(); } - public string GetVirtualPathTo(params string[] pathInRepo) + public string GetVirtualPathTo(string path) { - return Path.Combine(this.RepoRoot, Path.Combine(pathInRepo)); + string cleanedPath; + switch (Path.DirectorySeparatorChar) + { + case '/': + cleanedPath = path.Replace('\\', Path.DirectorySeparatorChar); + break; + + case '\\': + cleanedPath = path.Replace('/', Path.DirectorySeparatorChar); + break; + + default: + Assert.Fail($"Unrecognized Path.DirectorySeparatorChar: '{Path.DirectorySeparatorChar}'"); + return string.Empty; + } + + return Path.Combine(this.RepoRoot, cleanedPath); + } + + public string GetVirtualPathTo(params string[] pathParts) + { + return Path.Combine(this.RepoRoot, Path.Combine(pathParts)); } public string GetObjectPathTo(string objectHash) diff --git a/GVFS/GVFS.Hooks/GVFS.Hooks.Mac.csproj b/GVFS/GVFS.Hooks/GVFS.Hooks.Mac.csproj index 08531517e..ddde81618 100644 --- a/GVFS/GVFS.Hooks/GVFS.Hooks.Mac.csproj +++ b/GVFS/GVFS.Hooks/GVFS.Hooks.Mac.csproj @@ -59,8 +59,8 @@ Common\NamedPipes\NamedPipeClient.cs - - Common\NamedPipes\NamedPipeStreamWriterExtensions.cs + + Common\NamedPipes\StreamWriterExtensions.cs Common\NativeMethods.Shared.cs diff --git a/GVFS/GVFS.Hooks/GVFS.Hooks.Windows.csproj b/GVFS/GVFS.Hooks/GVFS.Hooks.Windows.csproj index 18a4c3c30..692c8832a 100644 --- a/GVFS/GVFS.Hooks/GVFS.Hooks.Windows.csproj +++ b/GVFS/GVFS.Hooks/GVFS.Hooks.Windows.csproj @@ -83,8 +83,8 @@ Common\NamedPipes\NamedPipeClient.cs - - Common\NamedPipes\NamedPipeStreamWriterExtensions.cs + + Common\NamedPipes\StreamWriterExtensions.cs Common\NativeMethods.Shared.cs diff --git a/GVFS/GVFS.Platform.Mac/MacFileSystemVirtualizer.cs b/GVFS/GVFS.Platform.Mac/MacFileSystemVirtualizer.cs index 54c11cd12..8d7da3aa0 100644 --- a/GVFS/GVFS.Platform.Mac/MacFileSystemVirtualizer.cs +++ b/GVFS/GVFS.Platform.Mac/MacFileSystemVirtualizer.cs @@ -99,6 +99,7 @@ protected override bool TryStart(out string error) // Callbacks this.virtualizationInstance.OnEnumerateDirectory = this.OnEnumerateDirectory; this.virtualizationInstance.OnGetFileStream = this.OnGetFileStream; + this.virtualizationInstance.OnFileModified = this.OnFileModified; uint threadCount = (uint)Environment.ProcessorCount * 2; @@ -115,8 +116,8 @@ protected override bool TryStart(out string error) this.Context.Tracer.RelatedEvent(EventLevel.Informational, $"{nameof(this.TryStart)}_StartedVirtualization", metadata: null); return true; - } - + } + private static byte[] ToVersionIdByteArray(byte[] version) { byte[] bytes = new byte[VirtualizationInstance.PlaceholderIdLength]; @@ -240,6 +241,41 @@ private Result OnGetFileStream( return Result.EIOError; } + private void OnFileModified(string relativePath) + { + try + { + if (!this.FileSystemCallbacks.IsMounted) + { + EventMetadata metadata = this.CreateEventMetadata(relativePath); + metadata.Add(TracingConstants.MessageKey.InfoMessage, nameof(this.OnFileModified) + ": Mount has not yet completed"); + this.Context.Tracer.RelatedEvent(EventLevel.Informational, $"{nameof(this.OnFileModified)}_MountNotComplete", metadata); + return; + } + + if (Virtualization.FileSystemCallbacks.IsPathInsideDotGit(relativePath)) + { + this.OnDotGitFileOrFolderChanged(relativePath); + } + else + { + // TODO(Mac): As a temporary work around (until we have a ConvertToFull type notification) treat every modification + // as the first write to the file + bool isFolder; + string fileName; + bool isPathProjected = this.FileSystemCallbacks.GitIndexProjection.IsPathProjected(relativePath, out fileName, out isFolder); + if (isPathProjected) + { + this.FileSystemCallbacks.OnFileConvertedToFull(relativePath); + } + } + } + catch (Exception e) + { + this.LogUnhandledExceptionAndExit(nameof(this.OnFileModified), this.CreateEventMetadata(relativePath, e)); + } + } + private Result OnEnumerateDirectory( ulong commandId, string relativePath, diff --git a/GVFS/GVFS.Virtualization/FileSystemCallbacks.cs b/GVFS/GVFS.Virtualization/FileSystemCallbacks.cs index 5ef406b92..69973817a 100644 --- a/GVFS/GVFS.Virtualization/FileSystemCallbacks.cs +++ b/GVFS/GVFS.Virtualization/FileSystemCallbacks.cs @@ -192,6 +192,7 @@ public void Stop() this.stopping = true; lock (this.postFetchJobLock) { + // TODO(Mac): System.PlatformNotSupportedException: Thread abort is not supported on this platform this.postFetchJobThread?.Abort(); } diff --git a/MirrorProvider/MirrorProvider.Mac/MacFileSystemVirtualizer.cs b/MirrorProvider/MirrorProvider.Mac/MacFileSystemVirtualizer.cs index 2321c0061..2e5b1a54a 100644 --- a/MirrorProvider/MirrorProvider.Mac/MacFileSystemVirtualizer.cs +++ b/MirrorProvider/MirrorProvider.Mac/MacFileSystemVirtualizer.cs @@ -19,7 +19,8 @@ public override bool TryConvertVirtualizationRoot(string directory, out string e public override bool TryStartVirtualizationInstance(Enlistment enlistment, out string error) { this.virtualizationInstance.OnEnumerateDirectory = this.OnEnumerateDirectory; - this.virtualizationInstance.OnGetFileStream = this.OnGetFileStream; + this.virtualizationInstance.OnGetFileStream = this.OnGetFileStream; + this.virtualizationInstance.OnFileModified = this.OnFileModified; Result result = this.virtualizationInstance.StartVirtualizationInstance( enlistment.SrcRoot, @@ -34,8 +35,8 @@ public override bool TryStartVirtualizationInstance(Enlistment enlistment, out s error = result.ToString(); return false; } - } - + } + private Result OnEnumerateDirectory( ulong commandId, string relativePath, @@ -144,6 +145,11 @@ private Result OnGetFileStream( } return Result.Success; + } + + private void OnFileModified(string relativePath) + { + Console.WriteLine("OnFileModified: " + relativePath); } private static byte[] ToVersionIdByteArray(byte version) diff --git a/MirrorProvider/MirrorProvider.Windows/WindowsFileSystemVirtualizer.cs b/MirrorProvider/MirrorProvider.Windows/WindowsFileSystemVirtualizer.cs index f89d1b0d8..d619e2ed9 100644 --- a/MirrorProvider/MirrorProvider.Windows/WindowsFileSystemVirtualizer.cs +++ b/MirrorProvider/MirrorProvider.Windows/WindowsFileSystemVirtualizer.cs @@ -34,14 +34,21 @@ public override bool TryStartVirtualizationInstance(Enlistment enlistment, out s this.virtualizationInstance.OnGetPlaceholderInformation = this.GetPlaceholderInformation; this.virtualizationInstance.OnGetFileStream = this.GetFileStream; + this.virtualizationInstance.OnNotifyFileHandleClosedFileModifiedOrDeleted = this.OnFileModifiedOrDeleted; uint threadCount = (uint)Environment.ProcessorCount * 2; + + NotificationMapping[] notificationMappings = new NotificationMapping[] + { + new NotificationMapping(NotificationType.FileHandleClosedFileModified, string.Empty), + }; + HResult result = this.virtualizationInstance.StartVirtualizationInstance( enlistment.SrcRoot, poolThreadCount: threadCount, concurrentThreadCount: threadCount, enableNegativePathCache: false, - notificationMappings: new NotificationMapping[] { }); + notificationMappings: notificationMappings); if (result == HResult.Ok) { @@ -276,6 +283,15 @@ private HResult GetFileStream( return HResult.Ok; } + private void OnFileModifiedOrDeleted(string relativePath, bool isDirectory, bool isFileModified, bool isFileDeleted) + { + // To keep WindowsFileSystemVirtualizer in sync with MacFileSystemVirtualizer we're only registering for + // NotificationType.FileHandleClosedFileModified and so this method will only be called for modifications. + // Once MacFileSystemVirtualizer supports delete notifications we'll register for + // NotificationType.FileHandleClosedFileDeleted and this method will be called for both modifications and deletions. + Console.WriteLine($"OnFileModifiedOrDeleted: `{relativePath}`, isDirectory: {isDirectory}, isModfied: {isFileDeleted}, isDeleted: {isFileDeleted}"); + } + // TODO: Add this to the ProjFS API private static HResult HResultFromWin32(int win32error) { diff --git a/ProjFS.Mac/PrjFSKext/PrjFSKext/KauthHandler.cpp b/ProjFS.Mac/PrjFSKext/PrjFSKext/KauthHandler.cpp index 1ac130b15..5f23cb9b1 100644 --- a/ProjFS.Mac/PrjFSKext/PrjFSKext/KauthHandler.cpp +++ b/ProjFS.Mac/PrjFSKext/PrjFSKext/KauthHandler.cpp @@ -23,6 +23,15 @@ static int HandleVnodeOperation( uintptr_t arg2, uintptr_t arg3); +static int HandleFileOpOperation( + kauth_cred_t credential, + void* idata, + kauth_action_t action, + uintptr_t arg0, + uintptr_t arg1, + uintptr_t arg2, + uintptr_t arg3); + static int GetPid(vfs_context_t context); static uint32_t ReadVNodeFileFlags(vnode_t vn, vfs_context_t context); @@ -46,6 +55,20 @@ static bool TrySendRequestAndWaitForResponse( static void AbortAllOutstandingEvents(); static bool ShouldIgnoreVnodeType(vtype vnodeType, vnode_t vnode); +static bool ShouldHandleEvent( + // In params: + vfs_context_t context, + const vnode_t vnode, + kauth_action_t action, + + // Out params: + VirtualizationRoot** root, + vtype* vnodeType, + uint32_t* vnodeFileFlags, + int* pid, + char procname[MAXCOMLEN + 1], + int* kauthResult); + // Structs typedef struct OutstandingMessage @@ -60,6 +83,7 @@ typedef struct OutstandingMessage // State static kauth_listener_t s_vnodeListener = nullptr; +static kauth_listener_t s_fileopListener = nullptr; static LIST_HEAD(OutstandingMessage_Head, OutstandingMessage) s_outstandingMessages = LIST_HEAD_INITIALIZER(OutstandingMessage_Head); static Mutex s_outstandingMessagesMutex = {}; @@ -98,6 +122,12 @@ kern_return_t KauthHandler_Init() goto CleanupAndFail; } + s_fileopListener = kauth_listen_scope(KAUTH_SCOPE_FILEOP, HandleFileOpOperation, nullptr); + if (nullptr == s_fileopListener) + { + goto CleanupAndFail; + } + return KERN_SUCCESS; CleanupAndFail: @@ -119,6 +149,16 @@ kern_return_t KauthHandler_Cleanup() { result = KERN_FAILURE; } + + if (nullptr != s_fileopListener) + { + kauth_unlisten_scope(s_fileopListener); + s_fileopListener = nullptr; + } + else + { + result = KERN_FAILURE; + } // Then, ensure there are no more callbacks in flight. AbortAllOutstandingEvents(); @@ -140,6 +180,37 @@ kern_return_t KauthHandler_Cleanup() return result; } +void KauthHandler_HandleKernelMessageResponse(uint64_t messageId, MessageType responseType) +{ + switch (responseType) + { + case MessageType_Response_Success: + case MessageType_Response_Fail: + { + Mutex_Acquire(s_outstandingMessagesMutex); + { + OutstandingMessage* outstandingMessage; + LIST_FOREACH(outstandingMessage, &s_outstandingMessages, _list_privates) + { + if (outstandingMessage->request.messageId == messageId) + { + // Save the response for the blocked thread. + outstandingMessage->response = responseType; + outstandingMessage->receivedResponse = true; + + wakeup(outstandingMessage); + + break; + } + } + } + Mutex_Release(s_outstandingMessagesMutex); + } + } + + return; +} + // Private functions static int HandleVnodeOperation( kauth_cred_t credential, @@ -151,131 +222,31 @@ static int HandleVnodeOperation( uintptr_t arg3) { atomic_fetch_add(&s_numActiveKauthEvents, 1); - - int kauthResult = KAUTH_RESULT_DEFER; - - int pid; - vtype vnodeType; - char procname[MAXCOMLEN + 1]; - VirtualizationRoot* root = nullptr; - int16_t rootIndex; - uint32_t currentVnodeFileFlags; vfs_context_t context = reinterpret_cast(arg0); vnode_t currentVnode = reinterpret_cast(arg1); // arg2 is the (vnode_t) parent vnode int* kauthError = reinterpret_cast(arg3); - - if (!VirtualizationRoot_VnodeIsOnAllowedFilesystem(currentVnode)) - { - kauthResult = KAUTH_RESULT_DEFER; - goto CleanupAndReturn; - } - - vnodeType = vnode_vtype(currentVnode); - if (ShouldIgnoreVnodeType(vnodeType, currentVnode)) - { - kauthResult = KAUTH_RESULT_DEFER; - goto CleanupAndReturn; - } - - pid = GetPid(context); - - currentVnodeFileFlags = ReadVNodeFileFlags(currentVnode, context); - if (!FileFlagsBitIsSet(currentVnodeFileFlags, FileFlags_IsInVirtualizationRoot)) - { - // This vnode is not part of ANY virtualization root, so exit now before doing any more work. - // This gives us a cheap way to avoid adding overhead to IO outside of a virtualization root. - - kauthResult = KAUTH_RESULT_DEFER; - goto CleanupAndReturn; - } - - proc_name(pid, procname, sizeof(procname)); - - if (FileFlagsBitIsSet(currentVnodeFileFlags, FileFlags_IsEmpty)) - { - // This vnode is not yet hydrated, so do not allow a file system crawler to force hydration. - // Once a vnode is hydrated, it's fine to allow crawlers to access those contents. - - if (IsFileSystemCrawler(procname)) - { - // We must DENY file system crawlers rather than DEFER. - // If we allow the crawler's access to succeed without hydrating, the kauth result will be cached and we won't - // get called again, so we lose the opportunity to hydrate the file/directory and it will appear as though - // it is missing its contents. - kauthResult = KAUTH_RESULT_DENY; - goto CleanupAndReturn; - } - } - - root = VirtualizationRoots_FindForVnode(currentVnode); - rootIndex = nullptr == root ? -1 : root->index; - - if (nullptr == root) - { - KextLog_FileNote(currentVnode, "No virtualization root found for file with set flag."); + VirtualizationRoot* root = nullptr; + vtype vnodeType; + uint32_t currentVnodeFileFlags; + int pid; + char procname[MAXCOMLEN + 1]; - kauthResult = KAUTH_RESULT_DEFER; - goto CleanupAndReturn; - } - else if (nullptr == root->providerUserClient) - { - if (rootIndex >= 0) - { - bool vnodeIsDir = (vnodeType == VDIR); - // File/directory is within an offline virtualization root (no provider) - // Allow read-only access to hydrated files, deny any writes except - // deletions, and prevent most read accesses to empty files. - // Empty directories need to be read/searched in order for them to - // be deleted by rm -r - if (ActionBitsNotSet(action, KAUTH_VNODE_ACCESS) - && ActionBitIsSet( - action, - KAUTH_VNODE_WRITE_ATTRIBUTES | - KAUTH_VNODE_WRITE_EXTATTRIBUTES | - KAUTH_VNODE_WRITE_DATA | - KAUTH_VNODE_APPEND_DATA | - KAUTH_VNODE_WRITE_SECURITY | - KAUTH_VNODE_LINKTARGET)) - { - KextLog_FileNote(currentVnode, "HandleVnodeOperation - write action 0x%x by process %u (%s) DENIED on %s with offline provider.", action, pid, procname, vnodeIsDir ? "directory" : "file"); - kauthResult = KAUTH_RESULT_DENY; - goto CleanupAndReturn; - } - - if (FileFlagsBitIsSet(currentVnodeFileFlags, FileFlags_IsEmpty)) - { - // Empty files/directories with offline provider may only be queried or deleted - if (ActionBitIsSet(action, (KAUTH_VNODE_ACCESS | KAUTH_VNODE_DELETE_CHILD | KAUTH_VNODE_DELETE | KAUTH_VNODE_READ_EXTATTRIBUTES))) - { - kauthResult = KAUTH_RESULT_DEFER; - goto CleanupAndReturn; - } - - // Empty directories may additionally have their attributes and security read, and contents listed/searched (otherwise rm -r doesn't work) - if (vnodeIsDir && ActionBitIsSet(action, (KAUTH_VNODE_READ_ATTRIBUTES | KAUTH_VNODE_READ_SECURITY | KAUTH_VNODE_LIST_DIRECTORY | KAUTH_VNODE_SEARCH))) - { - kauthResult = KAUTH_RESULT_DEFER; - goto CleanupAndReturn; - } - - // Disallow any other operations on empty placeholders - KextLog_FileNote(currentVnode, "HandleVnodeOperation - action 0x%x by process %u (%s) DENIED on empty %s with offline provider.", action, pid, procname, vnodeIsDir ? "directory" : "file"); - kauthResult = KAUTH_RESULT_DENY; - goto CleanupAndReturn; - } - } - - kauthResult = KAUTH_RESULT_DEFER; - goto CleanupAndReturn; - } - - // If the calling process is the provider, we must exit right away to avoid deadlocks - if (pid == root->providerPid) + int kauthResult = KAUTH_RESULT_DEFER; + + if (!ShouldHandleEvent( + context, + currentVnode, + action, + &root, + &vnodeType, + ¤tVnodeFileFlags, + &pid, + procname, + &kauthResult)) { - kauthResult = KAUTH_RESULT_DEFER; goto CleanupAndReturn; } @@ -339,36 +310,233 @@ static int HandleVnodeOperation( return kauthResult; } -void KauthHandler_HandleKernelMessageResponse(uint64_t messageId, MessageType responseType) +static int HandleFileOpOperation( + kauth_cred_t credential, + void* idata, + kauth_action_t action, + uintptr_t arg0, + uintptr_t arg1, + uintptr_t arg2, + uintptr_t arg3) { - switch (responseType) + atomic_fetch_add(&s_numActiveKauthEvents, 1); + + // Note: a fileop listener MUST NOT return an error, or it will result in a kernel panic. + // Fileop events are informational only. + int kauthError; + int kauthResult; + + vfs_context_t context = vfs_context_create(NULL); + + if (KAUTH_FILEOP_CLOSE == action) { - case MessageType_Response_Success: - case MessageType_Response_Fail: + vnode_t currentVnode = reinterpret_cast(arg0); + // arg1 is the (const char *) path + int closeFlags = static_cast(arg2); + + VirtualizationRoot* root = nullptr; + vtype vnodeType; + uint32_t currentVnodeFileFlags; + int pid; + char procname[MAXCOMLEN + 1]; + + if (!ShouldHandleEvent( + context, + currentVnode, + action, + &root, + &vnodeType, + ¤tVnodeFileFlags, + &pid, + procname, + &kauthResult)) { - Mutex_Acquire(s_outstandingMessagesMutex); + goto CleanupAndReturn; + } + + if (KAUTH_FILEOP_CLOSE_MODIFIED == closeFlags) + { + if (!TrySendRequestAndWaitForResponse( + root, + MessageType_KtoU_NotifyFileModified, + currentVnode, + pid, + procname, + &kauthResult, + &kauthError)) { - OutstandingMessage* outstandingMessage; - LIST_FOREACH(outstandingMessage, &s_outstandingMessages, _list_privates) + goto CleanupAndReturn; + } + } + } + +CleanupAndReturn: + vfs_context_rele(context); + atomic_fetch_sub(&s_numActiveKauthEvents, 1); + + // We must always return DEFER from a fileop listener. The kernel does not allow any other + // result and will panic if we return anything else. + return KAUTH_RESULT_DEFER; +} + +static bool ShouldHandleEvent( + // In params: + vfs_context_t context, + const vnode_t vnode, + kauth_action_t action, + + // Out params: + VirtualizationRoot** root, + vtype* vnodeType, + uint32_t* vnodeFileFlags, + int* pid, + char procname[MAXCOMLEN + 1], + int* kauthResult) +{ + *kauthResult = KAUTH_RESULT_DEFER; + + if (!VirtualizationRoot_VnodeIsOnAllowedFilesystem(vnode)) + { + *kauthResult = KAUTH_RESULT_DEFER; + return false; + } + + *vnodeType = vnode_vtype(vnode); + if (ShouldIgnoreVnodeType(*vnodeType, vnode)) + { + *kauthResult = KAUTH_RESULT_DEFER; + return false; + } + + *vnodeFileFlags = ReadVNodeFileFlags(vnode, context); + if (!FileFlagsBitIsSet(*vnodeFileFlags, FileFlags_IsInVirtualizationRoot)) + { + // This vnode is not part of ANY virtualization root, so exit now before doing any more work. + // This gives us a cheap way to avoid adding overhead to IO outside of a virtualization root. + + *kauthResult = KAUTH_RESULT_DEFER; + return false; + } + + *pid = GetPid(context); + proc_name(*pid, procname, MAXCOMLEN + 1); + + if (FileFlagsBitIsSet(*vnodeFileFlags, FileFlags_IsEmpty)) + { + // This vnode is not yet hydrated, so do not allow a file system crawler to force hydration. + // Once a vnode is hydrated, it's fine to allow crawlers to access those contents. + + if (IsFileSystemCrawler(procname)) + { + // We must DENY file system crawlers rather than DEFER. + // If we allow the crawler's access to succeed without hydrating, the kauth result will be cached and we won't + // get called again, so we lose the opportunity to hydrate the file/directory and it will appear as though + // it is missing its contents. + + *kauthResult = KAUTH_RESULT_DENY; + return false; + } + } + + *root = VirtualizationRoots_FindForVnode(vnode); + int16_t rootIndex = nullptr == *root ? -1 : (*root)->index; + + if (nullptr == *root) + { + KextLog_FileNote(vnode, "No virtualization root found for file with set flag."); + + *kauthResult = KAUTH_RESULT_DEFER; + return false; + } + else if (nullptr == (*root)->providerUserClient) + { + // There is no registered provider for this root + + // TODO(Mac): why do we need to check rootIndex > 0 here? + // If root was null, we would have already exited. And if not null, it can't be < 0. + if (rootIndex >= 0) + { + bool vnodeIsDir = (*vnodeType == VDIR); + + // File/directory is within an offline virtualization root (no provider) + // Allow read-only access to hydrated files, deny any writes except + // deletions, and prevent most read accesses to empty files. + // Empty directories need to be read/searched in order for them to + // be deleted by rm -r + if (ActionBitsNotSet(action, KAUTH_VNODE_ACCESS) && + ActionBitIsSet( + action, + KAUTH_VNODE_WRITE_ATTRIBUTES | + KAUTH_VNODE_WRITE_EXTATTRIBUTES | + KAUTH_VNODE_WRITE_DATA | + KAUTH_VNODE_APPEND_DATA | + KAUTH_VNODE_WRITE_SECURITY | + KAUTH_VNODE_LINKTARGET)) + { + KextLog_FileNote( + vnode, + "ShouldHandleEvent - write action 0x%x by process %u (%s) DENIED on %s with offline provider.", + action, + *pid, + procname, + vnodeIsDir ? "directory" : "file"); + + *kauthResult = KAUTH_RESULT_DENY; + return false; + } + + if (FileFlagsBitIsSet(*vnodeFileFlags, FileFlags_IsEmpty)) + { + // Empty files/directories with offline provider may only be queried or deleted + if (ActionBitIsSet( + action, + KAUTH_VNODE_ACCESS | + KAUTH_VNODE_DELETE_CHILD | + KAUTH_VNODE_DELETE | + KAUTH_VNODE_READ_EXTATTRIBUTES)) { - if (outstandingMessage->request.messageId == messageId) - { - // Save the response for the blocked thread. - outstandingMessage->response = responseType; - outstandingMessage->receivedResponse = true; - - wakeup(outstandingMessage); - - break; - } + *kauthResult = KAUTH_RESULT_DEFER; + return false; + } + + // Empty directories may additionally have their attributes and security read, and contents listed/searched (otherwise rm -r doesn't work) + if (vnodeIsDir && + ActionBitIsSet( + action, + KAUTH_VNODE_READ_ATTRIBUTES | + KAUTH_VNODE_READ_SECURITY | + KAUTH_VNODE_LIST_DIRECTORY | + KAUTH_VNODE_SEARCH)) + { + *kauthResult = KAUTH_RESULT_DEFER; + return false; } + + // Disallow any other operations on empty placeholders + KextLog_FileNote( + vnode, + "ShouldHandleEvent - action 0x%x by process %u (%s) DENIED on empty %s with offline provider.", + action, + *pid, + procname, + vnodeIsDir ? "directory" : "file"); + *kauthResult = KAUTH_RESULT_DENY; + return false; } - Mutex_Release(s_outstandingMessagesMutex); - } + + *kauthResult = KAUTH_RESULT_DEFER; + return false; } - return; + // If the calling process is the provider, we must exit right away to avoid deadlocks + if (*pid == (*root)->providerPid) + { + *kauthResult = KAUTH_RESULT_DEFER; + return false; + } + + return true; } static bool TrySendRequestAndWaitForResponse( @@ -413,6 +581,8 @@ static bool TrySendRequestAndWaitForResponse( } Mutex_Release(s_outstandingMessagesMutex); + // TODO(Mac): Should we pass in the root directly, rather than root->index? + // The index seems more like a private implementation detail. if (!isShuttingDown && 0 != ActiveProvider_SendMessage(root->index, messageSpec)) { // TODO: appropriately handle unresponsive providers diff --git a/ProjFS.Mac/PrjFSKext/PrjFSKext/VirtualizationRoots.cpp b/ProjFS.Mac/PrjFSKext/PrjFSKext/VirtualizationRoots.cpp index cf9d5e432..48c98edb8 100644 --- a/ProjFS.Mac/PrjFSKext/PrjFSKext/VirtualizationRoots.cpp +++ b/ProjFS.Mac/PrjFSKext/PrjFSKext/VirtualizationRoots.cpp @@ -80,6 +80,7 @@ VirtualizationRoot* VirtualizationRoots_FindForVnode(vnode_t vnode) { vnode_put(vnode); } + return root; } diff --git a/ProjFS.Mac/PrjFSKext/public/Message.h b/ProjFS.Mac/PrjFSKext/public/Message.h index e64b525a8..f337b8f6d 100644 --- a/ProjFS.Mac/PrjFSKext/public/Message.h +++ b/ProjFS.Mac/PrjFSKext/public/Message.h @@ -16,6 +16,8 @@ typedef enum MessageType_KtoU_EnumerateDirectory, MessageType_KtoU_HydrateFile, + MessageType_KtoU_NotifyFileModified, + // Responses MessageType_Response_Success, MessageType_Response_Fail, diff --git a/ProjFS.Mac/PrjFSLib.Mac.Managed/CallbackDelegates.cs b/ProjFS.Mac/PrjFSLib.Mac.Managed/CallbackDelegates.cs index a28d88a7f..bb70f658e 100644 --- a/ProjFS.Mac/PrjFSLib.Mac.Managed/CallbackDelegates.cs +++ b/ProjFS.Mac/PrjFSLib.Mac.Managed/CallbackDelegates.cs @@ -13,17 +13,27 @@ public delegate Result EnumerateDirectoryCallback( public delegate Result GetFileStreamCallback( ulong commandId, string relativePath, - [MarshalAs(UnmanagedType.LPArray, SizeConst = Interop.PrjFSLib.PlaceholderIdLength)] byte[] providerId, - [MarshalAs(UnmanagedType.LPArray, SizeConst = Interop.PrjFSLib.PlaceholderIdLength)] byte[] contentId, - int triggeringProcessId, string triggeringProcessName, IntPtr fileHandle); + public delegate Result NotifyOperationCallback( + ulong commandId, + string relativePath, + [MarshalAs(UnmanagedType.LPArray, SizeConst = Interop.PrjFSLib.PlaceholderIdLength)] + byte[] providerId, + [MarshalAs(UnmanagedType.LPArray, SizeConst = Interop.PrjFSLib.PlaceholderIdLength)] + byte[] contentId, + int triggeringProcessId, + string triggeringProcessName, + bool isDirectory, + NotificationType notificationType, + string destinationRelativePath); + // Pre-event notifications public delegate Result NotifyPreDeleteEvent( string relativePath, diff --git a/ProjFS.Mac/PrjFSLib.Mac.Managed/Interop/Callbacks.cs b/ProjFS.Mac/PrjFSLib.Mac.Managed/Interop/Callbacks.cs index d966c93b3..9911bcfbd 100644 --- a/ProjFS.Mac/PrjFSLib.Mac.Managed/Interop/Callbacks.cs +++ b/ProjFS.Mac/PrjFSLib.Mac.Managed/Interop/Callbacks.cs @@ -7,6 +7,6 @@ internal struct Callbacks { public EnumerateDirectoryCallback OnEnumerateDirectory; public GetFileStreamCallback OnGetFileStream; - public NotifyPreDeleteEvent OnNotifyPreDelete; + public NotifyOperationCallback OnNotifyOperation; } } diff --git a/ProjFS.Mac/PrjFSLib.Mac.Managed/Interop/PrjFSLib.cs b/ProjFS.Mac/PrjFSLib.Mac.Managed/Interop/PrjFSLib.cs index abd6458aa..ca6c98594 100644 --- a/ProjFS.Mac/PrjFSLib.Mac.Managed/Interop/PrjFSLib.cs +++ b/ProjFS.Mac/PrjFSLib.Mac.Managed/Interop/PrjFSLib.cs @@ -1,8 +1,8 @@ using System; -using System.Runtime.InteropServices; - -namespace PrjFSLib.Mac.Interop -{ +using System.Runtime.InteropServices; + +namespace PrjFSLib.Mac.Interop +{ internal static class PrjFSLib { public const int PlaceholderIdLength = 128; @@ -25,20 +25,17 @@ public static extern Result WritePlaceholderDirectory( [DllImport(PrjFSLibPath, EntryPoint = "PrjFS_WritePlaceholderFile")] public static extern Result WritePlaceholderFile( string relativePath, - [MarshalAs(UnmanagedType.LPArray, SizeConst = PlaceholderIdLength)] byte[] providerId, - [MarshalAs(UnmanagedType.LPArray, SizeConst = PlaceholderIdLength)] byte[] contentId, - ulong fileSize, - UInt16 fileMode); + ushort fileMode); [DllImport(PrjFSLibPath, EntryPoint = "PrjFS_WriteFileContents")] public static extern Result WriteFileContents( IntPtr fileHandle, IntPtr bytes, uint byteCount); - } -} + } +} diff --git a/ProjFS.Mac/PrjFSLib.Mac.Managed/PrjFSLib.Mac.Managed.csproj b/ProjFS.Mac/PrjFSLib.Mac.Managed/PrjFSLib.Mac.Managed.csproj index f4c04908e..f25921c0f 100644 --- a/ProjFS.Mac/PrjFSLib.Mac.Managed/PrjFSLib.Mac.Managed.csproj +++ b/ProjFS.Mac/PrjFSLib.Mac.Managed/PrjFSLib.Mac.Managed.csproj @@ -29,6 +29,11 @@ TRACE;RELEASE + + + + + diff --git a/ProjFS.Mac/PrjFSLib.Mac.Managed/VirtualizationInstance.cs b/ProjFS.Mac/PrjFSLib.Mac.Managed/VirtualizationInstance.cs index 34685b5ca..a99254c54 100644 --- a/ProjFS.Mac/PrjFSLib.Mac.Managed/VirtualizationInstance.cs +++ b/ProjFS.Mac/PrjFSLib.Mac.Managed/VirtualizationInstance.cs @@ -6,10 +6,25 @@ namespace PrjFSLib.Mac public class VirtualizationInstance { public const int PlaceholderIdLength = Interop.PrjFSLib.PlaceholderIdLength; - + + // PrjFSLib delegate - We must hold a reference to prevent garbage collection + private readonly NotifyOperationCallback notifyOperationCallback; + + public VirtualizationInstance() + { + this.notifyOperationCallback = this.OnNotifyOperation; + } + public virtual EnumerateDirectoryCallback OnEnumerateDirectory { get; set; } public virtual GetFileStreamCallback OnGetFileStream { get; set; } + public virtual NotifyFileModified OnFileModified { get; set; } + + public static Result ConvertDirectoryToVirtualizationRoot(string fullPath) + { + return Interop.PrjFSLib.ConvertDirectoryToVirtualizationRoot(fullPath); + } + public virtual Result StartVirtualizationInstance( string virtualizationRootFullPath, uint poolThreadCount) @@ -18,8 +33,12 @@ public virtual Result StartVirtualizationInstance( { OnEnumerateDirectory = this.OnEnumerateDirectory, OnGetFileStream = this.OnGetFileStream, + + // Note: We must use this.notifyOperationCallback and not this.OnNotifyOperation + // to ensure we don't create a temporary delegate that will be garbage collected + OnNotifyOperation = this.notifyOperationCallback, }; - + return Interop.PrjFSLib.StartVirtualizationInstance( virtualizationRootFullPath, callbacks, @@ -69,7 +88,7 @@ public virtual Result WritePlaceholderFile( byte[] providerId, byte[] contentId, ulong fileSize, - UInt16 fileMode) + ushort fileMode) { if (providerId.Length != Interop.PrjFSLib.PlaceholderIdLength || contentId.Length != Interop.PrjFSLib.PlaceholderIdLength) @@ -109,9 +128,25 @@ public virtual Result ConvertDirectoryToPlaceholder( throw new NotImplementedException(); } - public static Result ConvertDirectoryToVirtualizationRoot(string fullPath) + private Result OnNotifyOperation( + ulong commandId, + string relativePath, + byte[] providerId, + byte[] contentId, + int triggeringProcessId, + string triggeringProcessName, + bool isDirectory, + NotificationType notificationType, + string destinationRelativePath) { - return Interop.PrjFSLib.ConvertDirectoryToVirtualizationRoot(fullPath); + switch (notificationType) + { + case NotificationType.FileModified: + this.OnFileModified(relativePath); + return Result.Success; + } + + return Result.ENotYetImplemented; } } } diff --git a/ProjFS.Mac/PrjFSLib/PrjFSLib.cpp b/ProjFS.Mac/PrjFSLib/PrjFSLib.cpp index c2755b5d7..f126808eb 100644 --- a/ProjFS.Mac/PrjFSLib/PrjFSLib.cpp +++ b/ProjFS.Mac/PrjFSLib/PrjFSLib.cpp @@ -51,6 +51,7 @@ static errno_t RegisterVirtualizationRootPath(const char* path); static void HandleKernelRequest(Message requestSpec, void* messageMemory); static PrjFS_Result HandleEnumerateDirectoryRequest(const MessageHeader* request, const char* path); static PrjFS_Result HandleHydrateFileRequest(const MessageHeader* request, const char* path); +static PrjFS_Result HandleFileModifiedNotification(const MessageHeader* request, const char* path); static Message ParseMessageMemory(const void* messageMemory, uint32_t size); @@ -91,16 +92,12 @@ PrjFS_Result PrjFS_StartVirtualizationInstance( if (nullptr == virtualizationRootFullPath || nullptr == callbacks.EnumerateDirectory || - nullptr == callbacks.GetFileStream) + nullptr == callbacks.GetFileStream || + nullptr == callbacks.NotifyOperation) { return PrjFS_Result_EInvalidArgs; } - if (nullptr != callbacks.NotifyOperation) - { - return PrjFS_Result_ENotSupported; - } - if (!s_virtualizationRootFullPath.empty()) { return PrjFS_Result_EInvalidOperation; @@ -415,6 +412,12 @@ static void HandleKernelRequest(Message request, void* messageMemory) result = HandleHydrateFileRequest(requestHeader, request.path); break; } + + case MessageType_KtoU_NotifyFileModified: + { + result = HandleFileModifiedNotification(requestHeader, request.path); + break; + } } // async callbacks are not yet implemented @@ -449,10 +452,14 @@ static void HandleKernelRequest(Message request, void* messageMemory) static PrjFS_Result HandleEnumerateDirectoryRequest(const MessageHeader* request, const char* path) { #ifdef DEBUG - std::cout << "PrjFSLib.HandleKernelRequest: MessageType_KtoU_EnumerateDirectory" << std::endl; + std::cout << "PrjFSLib.HandleEnumerateDirectoryRequest: " << path << std::endl; #endif - PrjFS_Result callbackResult = s_callbacks.EnumerateDirectory(0, path, request->pid, request->procname); + PrjFS_Result callbackResult = s_callbacks.EnumerateDirectory( + 0 /* commandId */, + path, + request->pid, + request->procname); if (PrjFS_Result_Success == callbackResult) { @@ -473,7 +480,7 @@ static PrjFS_Result HandleEnumerateDirectoryRequest(const MessageHeader* request static PrjFS_Result HandleHydrateFileRequest(const MessageHeader* request, const char* path) { #ifdef DEBUG - std::cout << "PrjFSLib.HandleKernelRequest: MessageType_KtoU_HydrateFile" << std::endl; + std::cout << "PrjFSLib.HandleHydrateFileRequest: " << path << std::endl; #endif char fullPath[PrjFSMaxPath]; @@ -504,7 +511,14 @@ static PrjFS_Result HandleHydrateFileRequest(const MessageHeader* request, const return PrjFS_Result_EIOError; } - PrjFS_Result callbackResult = s_callbacks.GetFileStream(0, path, xattrData.providerId, xattrData.contentId, request->pid, request->procname, &fileHandle); + PrjFS_Result callbackResult = s_callbacks.GetFileStream( + 0 /* comandId */, + path, + xattrData.providerId, + xattrData.contentId, + request->pid, + request->procname, + &fileHandle); // TODO: once we support async callbacks, we'll need to save off the fileHandle if the result is Pending @@ -533,6 +547,35 @@ static PrjFS_Result HandleHydrateFileRequest(const MessageHeader* request, const return callbackResult; } +static PrjFS_Result HandleFileModifiedNotification(const MessageHeader* request, const char* path) +{ +#ifdef DEBUG + std::cout << "PrjFSLib.HandleFileModifiedNotification: " << path << std::endl; +#endif + + char fullPath[PrjFSMaxPath]; + CombinePaths(s_virtualizationRootFullPath.c_str(), path, fullPath); + + PrjFSFileXAttrData xattrData = {}; + if (!GetXAttr(fullPath, PrjFSFileXAttrName, sizeof(PrjFSFileXAttrData), &xattrData)) + { + return PrjFS_Result_EIOError; + } + + s_callbacks.NotifyOperation( + 0 /* commandId */, + path, + xattrData.providerId, + xattrData.contentId, + request->pid, + request->procname, + false /* isDirectory */, + PrjFS_NotificationType_FileModified, + nullptr /* destinationRelativePath */); + + return PrjFS_Result_Success; +} + static bool InitializeEmptyPlaceholder(const char* fullPath) { return From 239873075c2af500eebc208e0ec651847b7c826c Mon Sep 17 00:00:00 2001 From: Nick Graczyk Date: Mon, 13 Aug 2018 11:15:57 -0700 Subject: [PATCH 002/272] Properly work around Mac GCM issues when the system Java is 8 (or below) --- Scripts/Mac/PrepFunctionalTests.sh | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/Scripts/Mac/PrepFunctionalTests.sh b/Scripts/Mac/PrepFunctionalTests.sh index 97cc02169..7b4214710 100755 --- a/Scripts/Mac/PrepFunctionalTests.sh +++ b/Scripts/Mac/PrepFunctionalTests.sh @@ -33,15 +33,13 @@ fi git-credential-manager install -# Determine what Java version we have so we can construct a valid path to java +# If our Java version is 9+ (the formatting of 'java -version' changed in Java 9), work around +# https://github.com/Microsoft/Git-Credential-Manager-for-Mac-and-Linux/issues/71 JAVAVERSION="$(java -version 2>&1 | egrep -o '"[[:digit:]]+.[[:digit:]]+.[[:digit:]]+"' | xargs)" -if [[ -z $JAVAVERSION ]]; then - JAVAVERSION="10.0.2" +if [[ ! -z $JAVAVERSION ]]; then + git config --global credential.helper "!/usr/bin/java -Ddebug=false --add-modules java.xml.bind -Djava.net.useSystemProxies=true -jar /usr/local/Cellar/git-credential-manager/2.0.3/libexec/git-credential-manager-2.0.3.jar" || exit 1 fi -# Work around https://github.com/Microsoft/Git-Credential-Manager-for-Mac-and-Linux/issues/71 -git config --global credential.helper "!/Library/Java/JavaVirtualMachines/jdk-$JAVAVERSION.jdk/Contents/Home/bin/java -Ddebug=false --add-modules java.xml.bind -Djava.net.useSystemProxies=true -jar /usr/local/Cellar/git-credential-manager/2.0.3/libexec/git-credential-manager-2.0.3.jar" || exit 1 - # If we're running on an agent where the PAT environment variable is set and a URL is passed into the script, add it to the keychain. PAT=$SYSTEM_ACCESSTOKEN PATURL=$1 From 403a0eaedd95682f0b6fee3b938543277b1c857f Mon Sep 17 00:00:00 2001 From: William Baker Date: Mon, 13 Aug 2018 11:47:25 -0700 Subject: [PATCH 003/272] Remove TODOs --- GVFS/GVFS.Common/FileBasedCollection.cs | 1 - .../Tests/EnlistmentPerFixture/GitFilesTests.cs | 1 - 2 files changed, 2 deletions(-) diff --git a/GVFS/GVFS.Common/FileBasedCollection.cs b/GVFS/GVFS.Common/FileBasedCollection.cs index bb15a7571..3600fbbfa 100644 --- a/GVFS/GVFS.Common/FileBasedCollection.cs +++ b/GVFS/GVFS.Common/FileBasedCollection.cs @@ -390,7 +390,6 @@ private void WriteToDisk(string value) throw new InvalidOperationException(nameof(this.WriteToDisk) + " requires that collectionAppendsDirectlyToFile be true"); } - // TODO(Mac): Decide if we want to stick with \r\n on all platforms, move them all to \n, or use a mix byte[] bytes = Encoding.UTF8.GetBytes(value + "\r\n"); lock (this.fileLock) { diff --git a/GVFS/GVFS.FunctionalTests/Tests/EnlistmentPerFixture/GitFilesTests.cs b/GVFS/GVFS.FunctionalTests/Tests/EnlistmentPerFixture/GitFilesTests.cs index c85238ebb..7b24445b9 100644 --- a/GVFS/GVFS.FunctionalTests/Tests/EnlistmentPerFixture/GitFilesTests.cs +++ b/GVFS/GVFS.FunctionalTests/Tests/EnlistmentPerFixture/GitFilesTests.cs @@ -300,7 +300,6 @@ public void OverwrittenFileAddedToSparseCheckoutAndSkipWorktreeBitCleared() fileToOverwriteVirtualPath.ShouldBeAFile(this.fileSystem).WithContents(testContents); - // TODO(Mac): Switch from "\r\n" to Environment.NewLine or "\n" depending on how FileBasedCollection is updated GVFSHelpers.ModifiedPathsShouldContain(this.fileSystem, this.Enlistment.DotGVFSRoot, fileToOverwriteEntry + "\r\n"); // Verify skip-worktree cleared From b5a696f23cdbe83daa008b3a61994d8ed52d3a1c Mon Sep 17 00:00:00 2001 From: Nick Graczyk Date: Mon, 13 Aug 2018 12:04:35 -0700 Subject: [PATCH 004/272] Properly error out in PrepFunctionalTests.sh if GVFS-aware Git hasn't been downloaded yet --- Scripts/Mac/PrepFunctionalTests.sh | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/Scripts/Mac/PrepFunctionalTests.sh b/Scripts/Mac/PrepFunctionalTests.sh index 7b4214710..032d19f48 100755 --- a/Scripts/Mac/PrepFunctionalTests.sh +++ b/Scripts/Mac/PrepFunctionalTests.sh @@ -7,6 +7,10 @@ GVFSPROPS=$SCRIPTDIR/../../GVFS/GVFS.Build/GVFS.props GITVERSION="$(cat $GVFSPROPS | grep GitPackageVersion | grep -Eo '[0-9.]{1,}')" ROOTDIR=$SCRIPTDIR/../../.. GITDIR=$ROOTDIR/packages/gitformac.gvfs.installer/$GITVERSION/tools +if [[ ! -d $GITDIR ]]; then + echo "GVFS-aware Git package not found. Run BuildGVFSForMac.sh and try again" + exit 1 +fi hdiutil attach $GITDIR/*.dmg || exit 1 GITPKG="$(find /Volumes/Git* -type f -name *.pkg)" || exit 1 sudo installer -pkg "$GITPKG" -target / || exit 1 From a7268c23a9a78ba7f047e54526ff9fd4186e4678 Mon Sep 17 00:00:00 2001 From: William Baker Date: Mon, 13 Aug 2018 13:08:26 -0700 Subject: [PATCH 005/272] Additional cleanup for PR feedback --- .../Tools/GVFSFunctionalTestEnlistment.cs | 20 +++---------------- .../VirtualizationInstance.cs | 15 ++++---------- 2 files changed, 7 insertions(+), 28 deletions(-) diff --git a/GVFS/GVFS.FunctionalTests/Tools/GVFSFunctionalTestEnlistment.cs b/GVFS/GVFS.FunctionalTests/Tools/GVFSFunctionalTestEnlistment.cs index a6f4f0270..911259fb2 100644 --- a/GVFS/GVFS.FunctionalTests/Tools/GVFSFunctionalTestEnlistment.cs +++ b/GVFS/GVFS.FunctionalTests/Tools/GVFSFunctionalTestEnlistment.cs @@ -236,23 +236,9 @@ public void UnmountAndDeleteAll() public string GetVirtualPathTo(string path) { - string cleanedPath; - switch (Path.DirectorySeparatorChar) - { - case '/': - cleanedPath = path.Replace('\\', Path.DirectorySeparatorChar); - break; - - case '\\': - cleanedPath = path.Replace('/', Path.DirectorySeparatorChar); - break; - - default: - Assert.Fail($"Unrecognized Path.DirectorySeparatorChar: '{Path.DirectorySeparatorChar}'"); - return string.Empty; - } - - return Path.Combine(this.RepoRoot, cleanedPath); + // Replace '/' with Path.DirectorySeparatorChar to ensure that any + // Git paths are converted to system paths + return Path.Combine(this.RepoRoot, path.Replace('/', Path.DirectorySeparatorChar)); } public string GetVirtualPathTo(params string[] pathParts) diff --git a/ProjFS.Mac/PrjFSLib.Mac.Managed/VirtualizationInstance.cs b/ProjFS.Mac/PrjFSLib.Mac.Managed/VirtualizationInstance.cs index a99254c54..a9a3edbd1 100644 --- a/ProjFS.Mac/PrjFSLib.Mac.Managed/VirtualizationInstance.cs +++ b/ProjFS.Mac/PrjFSLib.Mac.Managed/VirtualizationInstance.cs @@ -7,14 +7,10 @@ public class VirtualizationInstance { public const int PlaceholderIdLength = Interop.PrjFSLib.PlaceholderIdLength; - // PrjFSLib delegate - We must hold a reference to prevent garbage collection - private readonly NotifyOperationCallback notifyOperationCallback; - - public VirtualizationInstance() - { - this.notifyOperationCallback = this.OnNotifyOperation; - } + // We must hold a reference to the delegate to prevent garbage collection + private NotifyOperationCallback preventGCOnNotifyOperationDelegate; + // References held to these delegates via class properties public virtual EnumerateDirectoryCallback OnEnumerateDirectory { get; set; } public virtual GetFileStreamCallback OnGetFileStream { get; set; } @@ -33,10 +29,7 @@ public virtual Result StartVirtualizationInstance( { OnEnumerateDirectory = this.OnEnumerateDirectory, OnGetFileStream = this.OnGetFileStream, - - // Note: We must use this.notifyOperationCallback and not this.OnNotifyOperation - // to ensure we don't create a temporary delegate that will be garbage collected - OnNotifyOperation = this.notifyOperationCallback, + OnNotifyOperation = (this.preventGCOnNotifyOperationDelegate = new NotifyOperationCallback(this.OnNotifyOperation)), }; return Interop.PrjFSLib.StartVirtualizationInstance( From 7e5e97a0bf3e0a2df447f19e478acd9735c0292f Mon Sep 17 00:00:00 2001 From: William Baker Date: Mon, 13 Aug 2018 13:14:45 -0700 Subject: [PATCH 006/272] Fix StyleCop issue --- ProjFS.Mac/PrjFSLib.Mac.Managed/VirtualizationInstance.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ProjFS.Mac/PrjFSLib.Mac.Managed/VirtualizationInstance.cs b/ProjFS.Mac/PrjFSLib.Mac.Managed/VirtualizationInstance.cs index a9a3edbd1..da5a079b0 100644 --- a/ProjFS.Mac/PrjFSLib.Mac.Managed/VirtualizationInstance.cs +++ b/ProjFS.Mac/PrjFSLib.Mac.Managed/VirtualizationInstance.cs @@ -29,7 +29,7 @@ public virtual Result StartVirtualizationInstance( { OnEnumerateDirectory = this.OnEnumerateDirectory, OnGetFileStream = this.OnGetFileStream, - OnNotifyOperation = (this.preventGCOnNotifyOperationDelegate = new NotifyOperationCallback(this.OnNotifyOperation)), + OnNotifyOperation = this.preventGCOnNotifyOperationDelegate = new NotifyOperationCallback(this.OnNotifyOperation), }; return Interop.PrjFSLib.StartVirtualizationInstance( From 8505a6a77885c56ffa076003905e4aad96a670f8 Mon Sep 17 00:00:00 2001 From: Kevin Willford Date: Mon, 13 Aug 2018 12:18:21 -0600 Subject: [PATCH 007/272] Use the test repo PAT set by the build --- Scripts/Mac/PrepFunctionalTests.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Scripts/Mac/PrepFunctionalTests.sh b/Scripts/Mac/PrepFunctionalTests.sh index 97cc02169..dd0b539e2 100755 --- a/Scripts/Mac/PrepFunctionalTests.sh +++ b/Scripts/Mac/PrepFunctionalTests.sh @@ -43,8 +43,8 @@ fi git config --global credential.helper "!/Library/Java/JavaVirtualMachines/jdk-$JAVAVERSION.jdk/Contents/Home/bin/java -Ddebug=false --add-modules java.xml.bind -Djava.net.useSystemProxies=true -jar /usr/local/Cellar/git-credential-manager/2.0.3/libexec/git-credential-manager-2.0.3.jar" || exit 1 # If we're running on an agent where the PAT environment variable is set and a URL is passed into the script, add it to the keychain. -PAT=$SYSTEM_ACCESSTOKEN PATURL=$1 +PAT=$2 if [[ ! -z $PAT && ! -z $PATURL ]] ; then security add-generic-password -a "Personal Access Token" -s "gcm4ml:git:$PATURL" -D Credential -w $PAT || exit 1 fi From 9807d51d0227ecba0414ed58c9042a18a1b515f7 Mon Sep 17 00:00:00 2001 From: Saeed Noursalehi Date: Mon, 13 Aug 2018 14:34:36 -0700 Subject: [PATCH 008/272] Log improvements: build everything under one target, added utility for expanding action bitfield, print line number and timestamp, avoid perf hit of kprintf --- .../xcshareddata/WorkspaceSettings.xcsettings | 8 ++++ .../xcshareddata/xcschemes/PrjFS.xcscheme | 14 +++++++ ProjFS.Mac/PrjFSKext/PrjFSKext/KextLog.cpp | 9 ++-- ProjFS.Mac/PrjFSKext/PrjFSKext/KextLog.hpp | 42 +++++++++++++++++++ ProjFS.Mac/PrjFSLib/prjfs-log/prjfs-log.cpp | 16 +++++-- 5 files changed, 82 insertions(+), 7 deletions(-) create mode 100644 ProjFS.Mac/PrjFS.xcworkspace/xcshareddata/WorkspaceSettings.xcsettings diff --git a/ProjFS.Mac/PrjFS.xcworkspace/xcshareddata/WorkspaceSettings.xcsettings b/ProjFS.Mac/PrjFS.xcworkspace/xcshareddata/WorkspaceSettings.xcsettings new file mode 100644 index 000000000..08de0be8d --- /dev/null +++ b/ProjFS.Mac/PrjFS.xcworkspace/xcshareddata/WorkspaceSettings.xcsettings @@ -0,0 +1,8 @@ + + + + + IDEWorkspaceSharedSettings_AutocreateContextsIfNeeded + + + diff --git a/ProjFS.Mac/PrjFS.xcworkspace/xcshareddata/xcschemes/PrjFS.xcscheme b/ProjFS.Mac/PrjFS.xcworkspace/xcshareddata/xcschemes/PrjFS.xcscheme index f2ea9f350..558130ad6 100644 --- a/ProjFS.Mac/PrjFS.xcworkspace/xcshareddata/xcschemes/PrjFS.xcscheme +++ b/ProjFS.Mac/PrjFS.xcworkspace/xcshareddata/xcschemes/PrjFS.xcscheme @@ -34,6 +34,20 @@ ReferencedContainer = "container:PrjFSLib/PrjFSLib.xcodeproj"> + + + + machAbsoluteTimestamp = time; s_currentUserClient->sendLogMessage(messagePtr, messageSize); } - else - { - kprintf("%s\n", messagePtr->logString); - } } RWLock_ReleaseShared(s_kextLogRWLock); diff --git a/ProjFS.Mac/PrjFSKext/PrjFSKext/KextLog.hpp b/ProjFS.Mac/PrjFSKext/PrjFSKext/KextLog.hpp index 82bead23e..cf40321b8 100644 --- a/ProjFS.Mac/PrjFSKext/PrjFSKext/KextLog.hpp +++ b/ProjFS.Mac/PrjFSKext/PrjFSKext/KextLog.hpp @@ -42,5 +42,47 @@ template #define KextLog_FileInfo(vnode, format, ...) ({ _os_log_verify_format_str(format, ##__VA_ARGS__); KextLogFile_Printf(KEXTLOG_INFO, vnode, format " (vnode path: '%s')", ##__VA_ARGS__); }) #define KextLog_FileNote(vnode, format, ...) ({ _os_log_verify_format_str(format, ##__VA_ARGS__); KextLogFile_Printf(KEXTLOG_NOTE, vnode, format " (vnode path: '%s')", ##__VA_ARGS__); }) +#define KextLog_VnodeOp(vnode, vnodeType, procname, action, message) \ + do { \ + if (VDIR == vnodeType) \ + { \ + KextLog_FileNote( \ + vnode, \ + message ". Proc name: %s. Directory vnode action: %s%s%s%s%s%s%s%s%s%s%s%s%s \n ", \ + procname, \ + (action & KAUTH_VNODE_LIST_DIRECTORY) ? " \n KAUTH_VNODE_LIST_DIRECTORY" : "", \ + (action & KAUTH_VNODE_ADD_FILE) ? " \n KAUTH_VNODE_ADD_FILE" : "", \ + (action & KAUTH_VNODE_SEARCH) ? " \n KAUTH_VNODE_SEARCH" : "", \ + (action & KAUTH_VNODE_DELETE) ? " \n KAUTH_VNODE_DELETE" : "", \ + (action & KAUTH_VNODE_ADD_SUBDIRECTORY) ? " \n KAUTH_VNODE_ADD_SUBDIRECTORY" : "", \ + (action & KAUTH_VNODE_DELETE_CHILD) ? " \n KAUTH_VNODE_DELETE_CHILD" : "", \ + (action & KAUTH_VNODE_READ_ATTRIBUTES) ? " \n KAUTH_VNODE_READ_ATTRIBUTES" : "", \ + (action & KAUTH_VNODE_WRITE_ATTRIBUTES) ? " \n KAUTH_VNODE_WRITE_ATTRIBUTES" : "", \ + (action & KAUTH_VNODE_READ_EXTATTRIBUTES) ? " \n KAUTH_VNODE_READ_EXTATTRIBUTES" : "", \ + (action & KAUTH_VNODE_WRITE_EXTATTRIBUTES) ? " \n KAUTH_VNODE_WRITE_EXTATTRIBUTES" : "", \ + (action & KAUTH_VNODE_READ_SECURITY) ? " \n KAUTH_VNODE_READ_SECURITY" : "", \ + (action & KAUTH_VNODE_WRITE_SECURITY) ? " \n KAUTH_VNODE_WRITE_SECURITY" : "", \ + (action & KAUTH_VNODE_TAKE_OWNERSHIP) ? " \n KAUTH_VNODE_TAKE_OWNERSHIP" : ""); \ + } \ + else \ + { \ + KextLog_FileNote( \ + vnode, \ + message ". Proc name: %s. File vnode action: %s%s%s%s%s%s%s%s%s%s%s%s \n ", \ + procname, \ + (action & KAUTH_VNODE_READ_DATA) ? " \n KAUTH_VNODE_READ_DATA" : "", \ + (action & KAUTH_VNODE_WRITE_DATA) ? " \n KAUTH_VNODE_WRITE_DATA" : "", \ + (action & KAUTH_VNODE_EXECUTE) ? " \n KAUTH_VNODE_EXECUTE" : "", \ + (action & KAUTH_VNODE_DELETE) ? " \n KAUTH_VNODE_DELETE" : "", \ + (action & KAUTH_VNODE_APPEND_DATA) ? " \n KAUTH_VNODE_APPEND_DATA" : "", \ + (action & KAUTH_VNODE_READ_ATTRIBUTES) ? " \n KAUTH_VNODE_READ_ATTRIBUTES" : "", \ + (action & KAUTH_VNODE_WRITE_ATTRIBUTES) ? " \n KAUTH_VNODE_WRITE_ATTRIBUTES" : "", \ + (action & KAUTH_VNODE_READ_EXTATTRIBUTES) ? " \n KAUTH_VNODE_READ_EXTATTRIBUTES" : "", \ + (action & KAUTH_VNODE_WRITE_EXTATTRIBUTES) ? " \n KAUTH_VNODE_WRITE_EXTATTRIBUTES" : "", \ + (action & KAUTH_VNODE_READ_SECURITY) ? " \n KAUTH_VNODE_READ_SECURITY" : "", \ + (action & KAUTH_VNODE_WRITE_SECURITY) ? " \n KAUTH_VNODE_WRITE_SECURITY" : "", \ + (action & KAUTH_VNODE_TAKE_OWNERSHIP) ? " \n KAUTH_VNODE_TAKE_OWNERSHIP" : ""); \ + } \ + } while (0) #endif /* KextLog_h */ diff --git a/ProjFS.Mac/PrjFSLib/prjfs-log/prjfs-log.cpp b/ProjFS.Mac/PrjFSLib/prjfs-log/prjfs-log.cpp index 8e0f138bc..10692d620 100644 --- a/ProjFS.Mac/PrjFSLib/prjfs-log/prjfs-log.cpp +++ b/ProjFS.Mac/PrjFSLib/prjfs-log/prjfs-log.cpp @@ -5,7 +5,7 @@ #include #include #include - +#include static const char* KextLogLevelAsString(KextLog_Level level); @@ -24,7 +24,9 @@ int main(int argc, const char * argv[]) std::cerr << "Failed to set up shared data queue.\n"; return 1; } - + + __block int lineCount = 0; + dispatch_source_set_event_handler(dataQueue.dispatchSource, ^{ struct { mach_msg_header_t msgHdr; @@ -39,6 +41,7 @@ int main(int argc, const char * argv[]) { break; } + int messageSize = entry->size; if (messageSize >= sizeof(KextLog_MessageHeader) + 2) { @@ -46,8 +49,15 @@ int main(int argc, const char * argv[]) memcpy(&message, entry->data, sizeof(KextLog_MessageHeader)); const char* messageType = KextLogLevelAsString(message.level); int logStringLength = messageSize - sizeof(KextLog_MessageHeader) - 1; - printf("%s: %.*s\n", messageType, logStringLength, entry->data + sizeof(KextLog_MessageHeader)); + + struct timeval tp; + gettimeofday(&tp, NULL); + long int timestamp = tp.tv_sec * 1000 + tp.tv_usec / 1000; + + printf("(%d: %ld) %s: %.*s\n", lineCount, timestamp, messageType, logStringLength, entry->data + sizeof(KextLog_MessageHeader)); + lineCount++; } + IODataQueueDequeue(dataQueue.queueMemory, nullptr, nullptr); } }); From 77afa565712dd7e7f0a6af116258a9844c2b8117 Mon Sep 17 00:00:00 2001 From: William Baker Date: Mon, 13 Aug 2018 14:44:38 -0700 Subject: [PATCH 009/272] Revert unnecessary changes --- .../EnlistmentPerFixture/WorkingDirectoryTests.cs | 15 ++++++++------- .../Tools/GVFSFunctionalTestEnlistment.cs | 1 - 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/GVFS/GVFS.FunctionalTests/Tests/EnlistmentPerFixture/WorkingDirectoryTests.cs b/GVFS/GVFS.FunctionalTests/Tests/EnlistmentPerFixture/WorkingDirectoryTests.cs index 82eba5814..1283675e0 100644 --- a/GVFS/GVFS.FunctionalTests/Tests/EnlistmentPerFixture/WorkingDirectoryTests.cs +++ b/GVFS/GVFS.FunctionalTests/Tests/EnlistmentPerFixture/WorkingDirectoryTests.cs @@ -399,8 +399,8 @@ public void FolderContentsProjectedAfterFolderCreateAndCheckout() { string folderName = "GVFlt_MultiThreadTest"; - // 575d597cf09b2cd1c0ddb4db21ce96979010bbcb did not have the folder GVFlt_MultiThreadTest - GitProcess.InvokeProcess(this.Enlistment.RepoRoot, "checkout 575d597cf09b2cd1c0ddb4db21ce96979010bbcb"); + // 54ea499de78eafb4dfd30b90e0bd4bcec26c4349 did not have the folder GVFlt_MultiThreadTest + GitProcess.InvokeProcess(this.Enlistment.RepoRoot, "checkout 54ea499de78eafb4dfd30b90e0bd4bcec26c4349"); // Confirm that no other test has created GVFlt_MultiThreadTest or put it in the modified files GVFSHelpers.ModifiedPathsShouldNotContain(this.fileSystem, this.Enlistment.DotGVFSRoot, folderName); @@ -409,10 +409,10 @@ public void FolderContentsProjectedAfterFolderCreateAndCheckout() virtualFolderPath.ShouldNotExistOnDisk(this.fileSystem); this.fileSystem.CreateDirectory(virtualFolderPath); - // b5fd7d23706a18cff3e2b8225588d479f7e51138 was the commit prior to deleting GVFLT_MultiThreadTest + // b3ddcf43b997cba3fbf9d2341b297e22bf48601a was the commit prior to deleting GVFLT_MultiThreadTest // 692765: Note that test also validates case insensitivity as GVFlt_MultiThreadTest is named GVFLT_MultiThreadTest // in this commit - GitProcess.InvokeProcess(this.Enlistment.RepoRoot, "checkout b5fd7d23706a18cff3e2b8225588d479f7e51138"); + GitProcess.InvokeProcess(this.Enlistment.RepoRoot, "checkout b3ddcf43b997cba3fbf9d2341b297e22bf48601a"); this.Enlistment.GetVirtualPathTo(folderName + "\\OpenForReadsSameTime\\test").ShouldBeAFile(this.fileSystem).WithContents("123 \r\n"); this.Enlistment.GetVirtualPathTo(folderName + "\\OpenForWritesSameTime\\test").ShouldBeAFile(this.fileSystem).WithContents("123 \r\n"); @@ -423,8 +423,8 @@ public void FolderContentsProjectedAfterFolderCreateAndCheckout() [Category(Categories.Mac.M3)] public void FolderContentsCorrectAfterCreateNewFolderRenameAndCheckoutCommitWithSameFolder() { - // 1ca414ced40f64bf94fc6c7f885974708bc600be is the commit prior to adding Test_EPF_MoveRenameFileTests - GitProcess.InvokeProcess(this.Enlistment.RepoRoot, "checkout 1ca414ced40f64bf94fc6c7f885974708bc600be"); + // 3a55d3b760c87642424e834228a3408796501e7c is the commit prior to adding Test_EPF_MoveRenameFileTests + GitProcess.InvokeProcess(this.Enlistment.RepoRoot, "checkout 3a55d3b760c87642424e834228a3408796501e7c"); // Confirm that no other test has created this folder or put it in the modified files string folderName = "Test_EPF_MoveRenameFileTests"; @@ -584,7 +584,8 @@ private void FolderEnumerationShouldHaveSingleEntry(string folderVirtualPath, st } folderEntries.Count().ShouldEqual(1); - folderEntries.ShouldContain(file => file.Name.Equals(expectedEntryName)); + FileSystemInfo singleEntry = folderEntries.First(); + singleEntry.Name.ShouldEqual(expectedEntryName, $"Actual name: {singleEntry.Name} does not equal expected name {expectedEntryName}"); } private void EnumerateAndReadShouldNotChangeEnumerationOrder(string folderRelativePath) diff --git a/GVFS/GVFS.FunctionalTests/Tools/GVFSFunctionalTestEnlistment.cs b/GVFS/GVFS.FunctionalTests/Tools/GVFSFunctionalTestEnlistment.cs index 911259fb2..6ba115c5f 100644 --- a/GVFS/GVFS.FunctionalTests/Tools/GVFSFunctionalTestEnlistment.cs +++ b/GVFS/GVFS.FunctionalTests/Tools/GVFSFunctionalTestEnlistment.cs @@ -2,7 +2,6 @@ using GVFS.FunctionalTests.Should; using GVFS.FunctionalTests.Tests; using GVFS.Tests.Should; -using NUnit.Framework; using System; using System.Collections.Generic; using System.IO; From e079410d3c4d00fafcf45b3104fa714df7da4fa3 Mon Sep 17 00:00:00 2001 From: William Baker Date: Mon, 13 Aug 2018 14:53:14 -0700 Subject: [PATCH 010/272] Revert unintended change --- .../Tests/EnlistmentPerFixture/WorkingDirectoryTests.cs | 2 ++ 1 file changed, 2 insertions(+) diff --git a/GVFS/GVFS.FunctionalTests/Tests/EnlistmentPerFixture/WorkingDirectoryTests.cs b/GVFS/GVFS.FunctionalTests/Tests/EnlistmentPerFixture/WorkingDirectoryTests.cs index 1283675e0..2f537d307 100644 --- a/GVFS/GVFS.FunctionalTests/Tests/EnlistmentPerFixture/WorkingDirectoryTests.cs +++ b/GVFS/GVFS.FunctionalTests/Tests/EnlistmentPerFixture/WorkingDirectoryTests.cs @@ -449,7 +449,9 @@ public void FolderContentsCorrectAfterCreateNewFolderRenameAndCheckoutCommitWith (folder + @"\MoveUnhydratedFileToDotGitFolder\Program.cs").ShouldBeAFile(this.fileSystem).WithContents(MoveRenameFileTests.TestFileContents); } + // TODO(Mac) This test is technically part of M2, but we need further investigation of why this test fails on build agents, but not on dev machines. [TestCase, Order(15)] + [Category(Categories.Mac.M3)] public void FilterNonUTF8FileName() { string encodingFilename = "ريلٌأكتوبرûمارسأغسطسºٰٰۂْٗ۵ريلٌأك.txt"; From 9f746c0d6781acfd450a131f94f37d8a7b8247d5 Mon Sep 17 00:00:00 2001 From: Saeed Noursalehi Date: Mon, 13 Aug 2018 15:24:43 -0700 Subject: [PATCH 011/272] Remove incorrect file header --- ProjFS.Mac/PrjFSKext/PrjFSKext/Info.plist | 2 +- ProjFS.Mac/PrjFSKext/PrjFSKext/VnodeUtilities.cpp | 8 -------- 2 files changed, 1 insertion(+), 9 deletions(-) diff --git a/ProjFS.Mac/PrjFSKext/PrjFSKext/Info.plist b/ProjFS.Mac/PrjFSKext/PrjFSKext/Info.plist index fa9be6ae0..893ea6850 100644 --- a/ProjFS.Mac/PrjFSKext/PrjFSKext/Info.plist +++ b/ProjFS.Mac/PrjFSKext/PrjFSKext/Info.plist @@ -19,7 +19,7 @@ CFBundleVersion 1 NSHumanReadableCopyright - Copyright © 2018 Microsoft. All rights reserved. + Copyright © Microsoft IOKitPersonalities PrjFS diff --git a/ProjFS.Mac/PrjFSKext/PrjFSKext/VnodeUtilities.cpp b/ProjFS.Mac/PrjFSKext/PrjFSKext/VnodeUtilities.cpp index 5ca1ff557..f12908751 100644 --- a/ProjFS.Mac/PrjFSKext/PrjFSKext/VnodeUtilities.cpp +++ b/ProjFS.Mac/PrjFSKext/PrjFSKext/VnodeUtilities.cpp @@ -1,11 +1,3 @@ -// -// VnodeUtilities.cpp -// PrjFSKext -// -// Created by Phil Dennis-Jordan on 15/05/18. -// Copyright © 2018 GVFS. All rights reserved. -// - #include "VnodeUtilities.hpp" #include "kernel-header-wrappers/vnode.h" #include "kernel-header-wrappers/mount.h" From 34fc54037c324879cbb84031111c99a4ff6a50a8 Mon Sep 17 00:00:00 2001 From: Saeed Noursalehi Date: Tue, 14 Aug 2018 10:29:12 -0700 Subject: [PATCH 012/272] Print the timestamp in the message rather than the current time --- ProjFS.Mac/PrjFSLib/prjfs-log/prjfs-log.cpp | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/ProjFS.Mac/PrjFSLib/prjfs-log/prjfs-log.cpp b/ProjFS.Mac/PrjFSLib/prjfs-log/prjfs-log.cpp index 10692d620..220c58858 100644 --- a/ProjFS.Mac/PrjFSLib/prjfs-log/prjfs-log.cpp +++ b/ProjFS.Mac/PrjFSLib/prjfs-log/prjfs-log.cpp @@ -5,7 +5,6 @@ #include #include #include -#include static const char* KextLogLevelAsString(KextLog_Level level); @@ -49,12 +48,8 @@ int main(int argc, const char * argv[]) memcpy(&message, entry->data, sizeof(KextLog_MessageHeader)); const char* messageType = KextLogLevelAsString(message.level); int logStringLength = messageSize - sizeof(KextLog_MessageHeader) - 1; - - struct timeval tp; - gettimeofday(&tp, NULL); - long int timestamp = tp.tv_sec * 1000 + tp.tv_usec / 1000; - printf("(%d: %ld) %s: %.*s\n", lineCount, timestamp, messageType, logStringLength, entry->data + sizeof(KextLog_MessageHeader)); + printf("(%d: %llu) %s: %.*s\n", lineCount, message.machAbsoluteTimestamp, messageType, logStringLength, entry->data + sizeof(KextLog_MessageHeader)); lineCount++; } From 0630599afd295b183f0a8323cfb9b1fe55c3b67e Mon Sep 17 00:00:00 2001 From: Derrick Stolee Date: Tue, 14 Aug 2018 14:38:55 -0400 Subject: [PATCH 013/272] Do not fail on small pools --- .../Virtualization/Projection/LazyUTF8StringTests.cs | 8 ++++++++ .../Virtualization/Projection/SortedFolderEntriesTests.cs | 8 ++++++++ .../Projection/GitIndexProjection.ObjectPool.cs | 5 +++-- 3 files changed, 19 insertions(+), 2 deletions(-) diff --git a/GVFS/GVFS.UnitTests/Virtualization/Projection/LazyUTF8StringTests.cs b/GVFS/GVFS.UnitTests/Virtualization/Projection/LazyUTF8StringTests.cs index 74d16e9af..42c2c9669 100644 --- a/GVFS/GVFS.UnitTests/Virtualization/Projection/LazyUTF8StringTests.cs +++ b/GVFS/GVFS.UnitTests/Virtualization/Projection/LazyUTF8StringTests.cs @@ -299,6 +299,14 @@ public unsafe void NonASCIICharacters_Compare() }); } + [TestCase] + public void MinimumPoolSize() + { + LazyUTF8String.ResetPool(new MockTracer(), 0); + LazyUTF8String.FreePool(); + LazyUTF8String.InitializePools(new MockTracer(), 0); + } + private static void CheckPoolSizes(int expectedBytePoolSize, int expectedStringPoolSize) { LazyUTF8String.BytePoolSize().ShouldEqual(expectedBytePoolSize, $"{nameof(LazyUTF8String.BytePoolSize)} should be {expectedBytePoolSize}"); diff --git a/GVFS/GVFS.UnitTests/Virtualization/Projection/SortedFolderEntriesTests.cs b/GVFS/GVFS.UnitTests/Virtualization/Projection/SortedFolderEntriesTests.cs index d62cf53aa..fc6befdde 100644 --- a/GVFS/GVFS.UnitTests/Virtualization/Projection/SortedFolderEntriesTests.cs +++ b/GVFS/GVFS.UnitTests/Virtualization/Projection/SortedFolderEntriesTests.cs @@ -124,6 +124,14 @@ public void Clear() sfe.Count.ShouldEqual(0); } + [TestCase] + public void SmallEntries() + { + SortedFolderEntries.FreePool(); + SortedFolderEntries.InitializePools(new MockTracer(), indexEntryCount: 0); + SortedFolderEntries.ResetPool(new MockTracer(), indexEntryCount: 0); + } + private static int CaseInsensitiveStringCompare(string x, string y) { return string.Compare(x, y, StringComparison.OrdinalIgnoreCase); diff --git a/GVFS/GVFS.Virtualization/Projection/GitIndexProjection.ObjectPool.cs b/GVFS/GVFS.Virtualization/Projection/GitIndexProjection.ObjectPool.cs index 8b506e3e2..5f079ec59 100644 --- a/GVFS/GVFS.Virtualization/Projection/GitIndexProjection.ObjectPool.cs +++ b/GVFS/GVFS.Virtualization/Projection/GitIndexProjection.ObjectPool.cs @@ -15,6 +15,7 @@ public partial class GitIndexProjection /// The type of object to be stored in the array pool internal class ObjectPool { + private const int MinPoolSize = 100; private int allocationSize; private T[] pool; private int freeIndex; @@ -23,9 +24,9 @@ internal class ObjectPool public ObjectPool(ITracer tracer, int allocationSize, Func objectCreator) { - if (allocationSize <= 0) + if (allocationSize < MinPoolSize) { - throw new ArgumentOutOfRangeException(nameof(allocationSize), "Must be greater than zero"); + allocationSize = MinPoolSize; } this.tracer = tracer; From 9dc5a1da658954820d9aa4e86b718a6e8a6cfbcc Mon Sep 17 00:00:00 2001 From: Derrick Stolee Date: Tue, 14 Aug 2018 15:04:29 -0400 Subject: [PATCH 014/272] Assert that a 0 input size creates a positive pool size --- .../Virtualization/Projection/LazyUTF8StringTests.cs | 4 ++++ .../Virtualization/Projection/SortedFolderEntriesTests.cs | 6 ++++++ 2 files changed, 10 insertions(+) diff --git a/GVFS/GVFS.UnitTests/Virtualization/Projection/LazyUTF8StringTests.cs b/GVFS/GVFS.UnitTests/Virtualization/Projection/LazyUTF8StringTests.cs index 42c2c9669..c9abaf052 100644 --- a/GVFS/GVFS.UnitTests/Virtualization/Projection/LazyUTF8StringTests.cs +++ b/GVFS/GVFS.UnitTests/Virtualization/Projection/LazyUTF8StringTests.cs @@ -303,8 +303,12 @@ public unsafe void NonASCIICharacters_Compare() public void MinimumPoolSize() { LazyUTF8String.ResetPool(new MockTracer(), 0); + LazyUTF8String.FreePool(); + LazyUTF8String.BytePoolSize().ShouldBeAtLeast(1); + LazyUTF8String.InitializePools(new MockTracer(), 0); + LazyUTF8String.BytePoolSize().ShouldBeAtLeast(1); } private static void CheckPoolSizes(int expectedBytePoolSize, int expectedStringPoolSize) diff --git a/GVFS/GVFS.UnitTests/Virtualization/Projection/SortedFolderEntriesTests.cs b/GVFS/GVFS.UnitTests/Virtualization/Projection/SortedFolderEntriesTests.cs index fc6befdde..a5b3459d8 100644 --- a/GVFS/GVFS.UnitTests/Virtualization/Projection/SortedFolderEntriesTests.cs +++ b/GVFS/GVFS.UnitTests/Virtualization/Projection/SortedFolderEntriesTests.cs @@ -128,8 +128,14 @@ public void Clear() public void SmallEntries() { SortedFolderEntries.FreePool(); + SortedFolderEntries.InitializePools(new MockTracer(), indexEntryCount: 0); + SortedFolderEntries.FilePoolSize().ShouldBeAtLeast(1); + SortedFolderEntries.FolderPoolSize().ShouldBeAtLeast(1); + SortedFolderEntries.ResetPool(new MockTracer(), indexEntryCount: 0); + SortedFolderEntries.FilePoolSize().ShouldBeAtLeast(1); + SortedFolderEntries.FolderPoolSize().ShouldBeAtLeast(1); } private static int CaseInsensitiveStringCompare(string x, string y) From 08022f75e2fe28ca9ee9a2453de7e5af34c9e05e Mon Sep 17 00:00:00 2001 From: Kevin Willford Date: Tue, 14 Aug 2018 13:40:53 -0600 Subject: [PATCH 015/272] Update git version to 2.18.0.gvfs.1.26.gf61ade4 This updates the git version to 2.18.0 which includes changes to the rename detection to use config setting which was previously turned off with a patch in git. --- GVFS/GVFS.Build/GVFS.props | 2 +- .../Tests/GitCommands/MergeConflictTests.cs | 27 +++++-------------- .../Tools/ControlGitRepo.cs | 1 + GVFS/GVFS.Hooks/Program.cs | 27 +++++++------------ GVFS/GVFS/CommandLine/GVFSVerb.cs | 1 + 5 files changed, 19 insertions(+), 39 deletions(-) diff --git a/GVFS/GVFS.Build/GVFS.props b/GVFS/GVFS.Build/GVFS.props index f38bc7af3..2c39f7500 100644 --- a/GVFS/GVFS.Build/GVFS.props +++ b/GVFS/GVFS.Build/GVFS.props @@ -3,7 +3,7 @@ 0.2.173.2 - 2.20180730.3 + 2.20180814.4 diff --git a/GVFS/GVFS.FunctionalTests/Tests/GitCommands/MergeConflictTests.cs b/GVFS/GVFS.FunctionalTests/Tests/GitCommands/MergeConflictTests.cs index acc6da6c1..594d8aeb4 100644 --- a/GVFS/GVFS.FunctionalTests/Tests/GitCommands/MergeConflictTests.cs +++ b/GVFS/GVFS.FunctionalTests/Tests/GitCommands/MergeConflictTests.cs @@ -84,7 +84,7 @@ public void MergeConflict_UsingStrategyOurs() public void MergeConflictEnsureStatusFailsDueToConfig() { // This is compared against the message emitted by GVFS.Hooks\Program.cs - string expectedErrorMessagePart = "--no-renames --no-breaks"; + string expectedErrorMessagePart = "--no-renames"; this.ValidateGitCommand("checkout " + GitRepoTests.ConflictTargetBranch); this.RunGitCommand("merge " + GitRepoTests.ConflictSourceBranch, checkStatus: false); @@ -95,25 +95,8 @@ public void MergeConflictEnsureStatusFailsDueToConfig() ProcessResult result2 = GitHelpers.InvokeGitAgainstGVFSRepo(this.Enlistment.RepoRoot, "status --no-renames"); result2.Errors.Contains(expectedErrorMessagePart); - ProcessResult result3 = GitHelpers.InvokeGitAgainstGVFSRepo(this.Enlistment.RepoRoot, "status --no-breaks"); - result3.Errors.Contains(expectedErrorMessagePart); - - // only renames in config - GitHelpers.InvokeGitAgainstGVFSRepo(this.Enlistment.RepoRoot, "config --local status.renames false"); - GitHelpers.InvokeGitAgainstGVFSRepo(this.Enlistment.RepoRoot, "status --no-breaks").Errors.ShouldBeEmpty(); - ProcessResult result4 = GitHelpers.InvokeGitAgainstGVFSRepo(this.Enlistment.RepoRoot, "status"); - result4.Errors.Contains(expectedErrorMessagePart); - - // only breaks in config - GitHelpers.InvokeGitAgainstGVFSRepo(this.Enlistment.RepoRoot, "config --local --unset status.renames"); - GitHelpers.InvokeGitAgainstGVFSRepo(this.Enlistment.RepoRoot, "config --local status.breaks false"); - GitHelpers.InvokeGitAgainstGVFSRepo(this.Enlistment.RepoRoot, "status --no-renames").Errors.ShouldBeEmpty(); - ProcessResult result5 = GitHelpers.InvokeGitAgainstGVFSRepo(this.Enlistment.RepoRoot, "status"); - result5.Errors.Contains(expectedErrorMessagePart); - // Complete setup to ensure teardown succeeds - GitHelpers.InvokeGitAgainstGVFSRepo(this.Enlistment.RepoRoot, "config --local status.renames false"); - GitHelpers.InvokeGitAgainstGVFSRepo(this.Enlistment.RepoRoot, "config --local status.breaks false"); + GitHelpers.InvokeGitAgainstGVFSRepo(this.Enlistment.RepoRoot, "config --local test.renames false"); } protected override void CreateEnlistment() @@ -127,8 +110,10 @@ protected override void CreateEnlistment() private void SetupRenameDetectionAvoidanceInConfig() { - this.ValidateGitCommand("config --local status.renames false"); - this.ValidateGitCommand("config --local status.breaks false"); + // Tell the pre-command hook that it shouldn't check for "--no-renames" when runing "git status" + // as the control repo won't do that. When the pre-command hook has been updated to properly + // check for "status.renames" we can set that value here instead. + this.ValidateGitCommand("config --local test.renames false"); } } } diff --git a/GVFS/GVFS.FunctionalTests/Tools/ControlGitRepo.cs b/GVFS/GVFS.FunctionalTests/Tools/ControlGitRepo.cs index 58381fdfd..fbfb81e36 100644 --- a/GVFS/GVFS.FunctionalTests/Tools/ControlGitRepo.cs +++ b/GVFS/GVFS.FunctionalTests/Tools/ControlGitRepo.cs @@ -50,6 +50,7 @@ public void Initialize() GitProcess.Invoke(this.RootPath, "init"); GitProcess.Invoke(this.RootPath, "config core.autocrlf false"); GitProcess.Invoke(this.RootPath, "config merge.stat false"); + GitProcess.Invoke(this.RootPath, "config merge.renames false"); GitProcess.Invoke(this.RootPath, "config advice.statusUoption false"); GitProcess.Invoke(this.RootPath, "config core.abbrev 40"); GitProcess.Invoke(this.RootPath, "config status.aheadbehind false"); diff --git a/GVFS/GVFS.Hooks/Program.cs b/GVFS/GVFS.Hooks/Program.cs index 4e172857a..5ffaea8c0 100644 --- a/GVFS/GVFS.Hooks/Program.cs +++ b/GVFS/GVFS.Hooks/Program.cs @@ -159,35 +159,28 @@ private static void VerifyRenameDetectionSettings(string[] args) if (File.Exists(Path.Combine(srcRoot, GVFSConstants.DotGit.MergeHead)) || File.Exists(Path.Combine(srcRoot, GVFSConstants.DotGit.RevertHead))) { - // If no-renames and no-breaks are specified, avoid reading config. - if (!args.Contains("--no-renames") || !args.Contains("--no-breaks")) + // If no-renames is specified, avoid reading config. + if (!args.Contains("--no-renames")) { + // To behave properly, this needs to check for the status.renames setting the same + // way that git does including global and local config files, setting inheritance from + // diff.renames, etc. This is probably best accomplished by calling "git config --get status.renames" + // to ensure we are getting the correct value and then checking for "true" (rather than + // just existance like below). Dictionary statusConfig = GitConfigHelper.GetSettings( File.ReadAllLines(Path.Combine(srcRoot, GVFSConstants.DotGit.Config)), - "status"); + "test"); - if (!IsRunningWithParamOrSetting(args, statusConfig, "--no-renames", "renames") || - !IsRunningWithParamOrSetting(args, statusConfig, "--no-breaks", "breaks")) + if (!statusConfig.ContainsKey("renames")) { ExitWithError( "git status requires rename detection to be disabled during a merge or revert conflict.", - "Run 'git status --no-renames --no-breaks'"); + "Run 'git status --no-renames'"); } } } } - private static bool IsRunningWithParamOrSetting( - string[] args, - Dictionary configSettings, - string expectedArg, - string expectedSetting) - { - return - args.Contains(expectedArg) || - configSettings.ContainsKey(expectedSetting); - } - private static void RunLockRequest(string[] args, bool unattended, LockRequestDelegate requestToRun) { try diff --git a/GVFS/GVFS/CommandLine/GVFSVerb.cs b/GVFS/GVFS/CommandLine/GVFSVerb.cs index 13464833c..945682c40 100644 --- a/GVFS/GVFS/CommandLine/GVFSVerb.cs +++ b/GVFS/GVFS/CommandLine/GVFSVerb.cs @@ -100,6 +100,7 @@ public static bool TrySetRequiredGitConfigSettings(Enlistment enlistment) { "gui.gcwarning", "false" }, { "index.version", "4" }, { "merge.stat", "false" }, + { "merge.renames", "false" }, { "receive.autogc", "false" }, { "status.deserializePath", gitStatusCachePath }, }; From 0833d78511288674d68e41244d136dd2fd63d9be Mon Sep 17 00:00:00 2001 From: Nick Graczyk Date: Tue, 14 Aug 2018 17:42:32 -0700 Subject: [PATCH 016/272] Correct regex used to match GVFS-aware Git package to actually match the NuGet spec for what defines a package --- Scripts/Mac/BuildGVFSForMac.sh | 2 +- Scripts/Mac/DownloadGVFSGit.sh | 2 +- Scripts/Mac/PrepFunctionalTests.sh | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/Scripts/Mac/BuildGVFSForMac.sh b/Scripts/Mac/BuildGVFSForMac.sh index 5078f9cb0..c62426043 100755 --- a/Scripts/Mac/BuildGVFSForMac.sh +++ b/Scripts/Mac/BuildGVFSForMac.sh @@ -34,7 +34,7 @@ fi $SCRIPTDIR/DownloadGVFSGit.sh || exit 1 GVFSPROPS=$SRCDIR/GVFS/GVFS.Build/GVFS.props -GITVERSION="$(cat $GVFSPROPS | grep GitPackageVersion | grep -Eo '[0-9.]{1,}')" +GITVERSION="$(cat $GVFSPROPS | grep GitPackageVersion | grep -Eo '[0-9.]+(-\w+)?')" GITPATH="$(find $PACKAGES/gitformac.gvfs.installer/$GITVERSION -type f -name *.dmg)" || exit 1 # Now that we have a path containing the version number, generate GVFSConstants.GitVersion.cs $SCRIPTDIR/GenerateGitVersionConstants.sh "$GITPATH" $BUILDDIR || exit 1 diff --git a/Scripts/Mac/DownloadGVFSGit.sh b/Scripts/Mac/DownloadGVFSGit.sh index 3d1ced9c9..4ca725b9e 100755 --- a/Scripts/Mac/DownloadGVFSGit.sh +++ b/Scripts/Mac/DownloadGVFSGit.sh @@ -3,7 +3,7 @@ SRCDIR=$SCRIPTDIR/../.. BUILDDIR=$SRCDIR/../BuildOutput/GVFS.Build PACKAGESDIR=$SRCDIR/../packages GVFSPROPS=$SRCDIR/GVFS/GVFS.Build/GVFS.props -GITVERSION="$(cat $GVFSPROPS | grep GitPackageVersion | grep -Eo '[0-9.]{1,}')" +GITVERSION="$(cat $GVFSPROPS | grep GitPackageVersion | grep -Eo '[0-9.]+(-\w+)?')" cp $SRCDIR/nuget.config $BUILDDIR dotnet new classlib -n GVFS.Restore -o $BUILDDIR --force dotnet add $BUILDDIR/GVFS.Restore.csproj package --package-directory $PACKAGESDIR GitForMac.GVFS.Installer --version $GITVERSION \ No newline at end of file diff --git a/Scripts/Mac/PrepFunctionalTests.sh b/Scripts/Mac/PrepFunctionalTests.sh index d7ef9b2b1..f1eca5248 100755 --- a/Scripts/Mac/PrepFunctionalTests.sh +++ b/Scripts/Mac/PrepFunctionalTests.sh @@ -4,7 +4,7 @@ SCRIPTDIR=$(dirname ${BASH_SOURCE[0]}) # Install GVFS-aware Git (that was downloaded by the build script) GVFSPROPS=$SCRIPTDIR/../../GVFS/GVFS.Build/GVFS.props -GITVERSION="$(cat $GVFSPROPS | grep GitPackageVersion | grep -Eo '[0-9.]{1,}')" +GITVERSION="$(cat $GVFSPROPS | grep GitPackageVersion | grep -Eo '[0-9.]+(-\w+)?')" ROOTDIR=$SCRIPTDIR/../../.. GITDIR=$ROOTDIR/packages/gitformac.gvfs.installer/$GITVERSION/tools if [[ ! -d $GITDIR ]]; then From 32cb916ada09e41133f98527535c05db8365617f Mon Sep 17 00:00:00 2001 From: Jameson Miller Date: Tue, 14 Aug 2018 15:05:24 -0400 Subject: [PATCH 017/272] VirtualFileSystemHook: ensure string is properly null terminated When the VirtualFileSystem hook receives an error reading data from GVFS over a named pipe, it will include the pipe message in the error message. The pipe message is not necassarily null terminated, so we need to append a null terminating character before passing it to string formatting functions that expect a null terminated string. This change makes sure the message is null terminated. --- GVFS/GVFS.VirtualFileSystemHook/main.cpp | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/GVFS/GVFS.VirtualFileSystemHook/main.cpp b/GVFS/GVFS.VirtualFileSystemHook/main.cpp index e37f31cdc..d0c1aea18 100644 --- a/GVFS/GVFS.VirtualFileSystemHook/main.cpp +++ b/GVFS/GVFS.VirtualFileSystemHook/main.cpp @@ -6,6 +6,8 @@ enum VirtualFileSystemErrorReturnCode ErrorVirtualFileSystemProtocol = ReturnCode::LastError + 1, }; +const int PIPE_BUFFER_SIZE = 1024; + int main(int argc, char *argv[]) { if (argc != 2) @@ -39,7 +41,10 @@ int main(int argc, char *argv[]) die(ReturnCode::PipeWriteFailed, "Failed to write to pipe (%d)\n", error); } - char message[1024]; + // Allow for 1 extra character in case we need to + // null terminate the message, and the message + // is PIPE_BUFFER_SIZE chars long. + char message[PIPE_BUFFER_SIZE + 1]; unsigned long bytesRead; int lastError; bool finishedReading = false; @@ -51,7 +56,7 @@ int main(int argc, char *argv[]) success = ReadFromPipe( pipeHandle, message, - sizeof(message), + PIPE_BUFFER_SIZE, &bytesRead, &lastError); @@ -59,7 +64,7 @@ int main(int argc, char *argv[]) { break; } - + messageLength = bytesRead; if (firstRead) @@ -67,6 +72,7 @@ int main(int argc, char *argv[]) firstRead = false; if (message[0] != 'S') { + message[bytesRead] = 0; die(ReturnCode::PipeReadFailed, "Read response from pipe failed (%s)\n", message); } From 0c09ed4dc00709870c5738cbe9a04dfc639413b6 Mon Sep 17 00:00:00 2001 From: Jameson Miller Date: Mon, 13 Aug 2018 14:30:38 -0400 Subject: [PATCH 018/272] Shutdown GitStatusCache before components it depends on Tweak the order that components are shutdown, to shutdown the GitStatusCache before other components that it depends on. This is to prevent problems that might arise from a component being disabled that GitStatusCache depends on. --- GVFS/GVFS.Virtualization/FileSystemCallbacks.cs | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/GVFS/GVFS.Virtualization/FileSystemCallbacks.cs b/GVFS/GVFS.Virtualization/FileSystemCallbacks.cs index 69973817a..906bad018 100644 --- a/GVFS/GVFS.Virtualization/FileSystemCallbacks.cs +++ b/GVFS/GVFS.Virtualization/FileSystemCallbacks.cs @@ -196,8 +196,11 @@ public void Stop() this.postFetchJobThread?.Abort(); } - this.fileSystemVirtualizer.PrepareToStop(); + // Shutdown the GitStatusCache before other + // components that it depends on. this.gitStatusCache.Shutdown(); + + this.fileSystemVirtualizer.PrepareToStop(); this.backgroundFileSystemTaskRunner.Shutdown(); this.GitIndexProjection.Shutdown(); this.BlobSizes.Shutdown(); From dfbe6579623acb650a633663c3212a19b3d15878 Mon Sep 17 00:00:00 2001 From: Jameson Miller Date: Tue, 14 Aug 2018 10:43:06 -0400 Subject: [PATCH 019/272] GVFSLock.Shared: correctly report whether lock was acquired This fixes a bug in TryAcquireGVFSLockForProcess where it does not correctly report whether it was able to acquire the GVFS lock. In cases where this method retries to acquire the lock, it would report success, even if it was not able to acquire the lock. This could lead to problems where commands that depended on the GVFS could run, even if it did not actually have the lock. --- GVFS/GVFS.Common/GVFSLock.Shared.cs | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/GVFS/GVFS.Common/GVFSLock.Shared.cs b/GVFS/GVFS.Common/GVFSLock.Shared.cs index 4f3fc5d7b..10785b445 100644 --- a/GVFS/GVFS.Common/GVFSLock.Shared.cs +++ b/GVFS/GVFS.Common/GVFSLock.Shared.cs @@ -81,13 +81,14 @@ public static bool TryAcquireGVFSLockForProcess( } }; + bool isSuccessfulLockResult; if (unattended) { - waitForLock(); + isSuccessfulLockResult = waitForLock(); } else { - ConsoleHelper.ShowStatusWhileRunning( + isSuccessfulLockResult = ConsoleHelper.ShowStatusWhileRunning( waitForLock, message, output: Console.Out, @@ -96,7 +97,7 @@ public static bool TryAcquireGVFSLockForProcess( } result = null; - return true; + return isSuccessfulLockResult; } public static void ReleaseGVFSLock( From eda24a6013a6d83ebad0d9310829838790affbc2 Mon Sep 17 00:00:00 2001 From: William Baker Date: Wed, 15 Aug 2018 14:38:26 -0700 Subject: [PATCH 020/272] Add FileOp specific ShouldHandleFileOpEvent helper to kext --- .../PrjFSKext/PrjFSKext/KauthHandler.cpp | 109 +++++++++++++----- 1 file changed, 81 insertions(+), 28 deletions(-) diff --git a/ProjFS.Mac/PrjFSKext/PrjFSKext/KauthHandler.cpp b/ProjFS.Mac/PrjFSKext/PrjFSKext/KauthHandler.cpp index 5f23cb9b1..ea3ed4d87 100644 --- a/ProjFS.Mac/PrjFSKext/PrjFSKext/KauthHandler.cpp +++ b/ProjFS.Mac/PrjFSKext/PrjFSKext/KauthHandler.cpp @@ -55,7 +55,7 @@ static bool TrySendRequestAndWaitForResponse( static void AbortAllOutstandingEvents(); static bool ShouldIgnoreVnodeType(vtype vnodeType, vnode_t vnode); -static bool ShouldHandleEvent( +static bool ShouldHandleVnodeOpEvent( // In params: vfs_context_t context, const vnode_t vnode, @@ -69,6 +69,15 @@ static bool ShouldHandleEvent( char procname[MAXCOMLEN + 1], int* kauthResult); +static bool ShouldHandleFileOpEvent( + // In params: + vfs_context_t context, + const vnode_t vnode, + kauth_action_t action, + + // Out params: + VirtualizationRoot** root, + int* pid); // Structs typedef struct OutstandingMessage @@ -236,7 +245,7 @@ static int HandleVnodeOperation( int kauthResult = KAUTH_RESULT_DEFER; - if (!ShouldHandleEvent( + if (!ShouldHandleVnodeOpEvent( context, currentVnode, action, @@ -310,6 +319,8 @@ static int HandleVnodeOperation( return kauthResult; } +// Note: a fileop listener MUST NOT return an error, or it will result in a kernel panic. +// Fileop events are informational only. static int HandleFileOpOperation( kauth_cred_t credential, void* idata, @@ -321,11 +332,6 @@ static int HandleFileOpOperation( { atomic_fetch_add(&s_numActiveKauthEvents, 1); - // Note: a fileop listener MUST NOT return an error, or it will result in a kernel panic. - // Fileop events are informational only. - int kauthError; - int kauthResult; - vfs_context_t context = vfs_context_create(NULL); if (KAUTH_FILEOP_CLOSE == action) @@ -334,28 +340,25 @@ static int HandleFileOpOperation( // arg1 is the (const char *) path int closeFlags = static_cast(arg2); - VirtualizationRoot* root = nullptr; - vtype vnodeType; - uint32_t currentVnodeFileFlags; - int pid; - char procname[MAXCOMLEN + 1]; - - if (!ShouldHandleEvent( - context, - currentVnode, - action, - &root, - &vnodeType, - ¤tVnodeFileFlags, - &pid, - procname, - &kauthResult)) - { - goto CleanupAndReturn; - } - if (KAUTH_FILEOP_CLOSE_MODIFIED == closeFlags) { + VirtualizationRoot* root = nullptr; + int pid; + if (!ShouldHandleFileOpEvent( + context, + currentVnode, + action, + &root, + &pid)) + { + goto CleanupAndReturn; + } + + char procname[MAXCOMLEN + 1]; + proc_name(pid, procname, MAXCOMLEN + 1); + + int kauthResult; + int kauthError; if (!TrySendRequestAndWaitForResponse( root, MessageType_KtoU_NotifyFileModified, @@ -379,7 +382,7 @@ static int HandleFileOpOperation( return KAUTH_RESULT_DEFER; } -static bool ShouldHandleEvent( +static bool ShouldHandleVnodeOpEvent( // In params: vfs_context_t context, const vnode_t vnode, @@ -539,6 +542,56 @@ static bool ShouldHandleEvent( return true; } +static bool ShouldHandleFileOpEvent( + // In params: + vfs_context_t context, + const vnode_t vnode, + kauth_action_t action, + + // Out params: + VirtualizationRoot** root, + int* pid) +{ + vtype vnodeType = vnode_vtype(vnode); + if (ShouldIgnoreVnodeType(vnodeType, vnode)) + { + return false; + } + + // TODO(Mac): We will still want to handle renames into a root, and those vnodes would + // not yet have the FileFlags_IsInVirtualizationRoot set + uint32_t vnodeFileFlags = ReadVNodeFileFlags(vnode, context); + if (!FileFlagsBitIsSet(vnodeFileFlags, FileFlags_IsInVirtualizationRoot)) + { + // This vnode is not part of ANY virtualization root, so exit now before doing any more work. + // This gives us a cheap way to avoid adding overhead to IO outside of a virtualization root. + return false; + } + + *root = VirtualizationRoots_FindForVnode(vnode); + int16_t rootIndex = nullptr == *root ? -1 : (*root)->index; + + if (nullptr == *root) + { + KextLog_FileNote(vnode, "ShouldHandleFileOpEvent(%d): No virtualization root found for file with set flag.", action); + return false; + } + else if (nullptr == (*root)->providerUserClient) + { + // There is no registered provider for this root + return false; + } + + // If the calling process is the provider, we must exit right away to avoid deadlocks + *pid = GetPid(context); + if (*pid == (*root)->providerPid) + { + return false; + } + + return true; +} + static bool TrySendRequestAndWaitForResponse( const VirtualizationRoot* root, MessageType messageType, From adc1c9daa9ac57d1a91d4c2453c0929eca45453c Mon Sep 17 00:00:00 2001 From: Kevin Willford Date: Wed, 15 Aug 2018 12:24:42 -0600 Subject: [PATCH 021/272] Add build badge for Windows and Mac CI builds --- Readme.md | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/Readme.md b/Readme.md index 6371eb155..ffbf01864 100644 --- a/Readme.md +++ b/Readme.md @@ -1,5 +1,17 @@ # VFS for Git +## Windows + +|Branch|Unit Tests|Functional Tests| +|:--:|:--:|:--:| +|**master**|[![Build status](https://gvfs.visualstudio.com/ci/_apis/build/status/CI%20-%20Windows%20-%20master?branchName=master)](https://gvfs.visualstudio.com/ci/_build/latest?definitionId=7)|[![Build status](https://gvfs.visualstudio.com/ci/_apis/build/status/CI%20-%20Windows%20-%20Full%20Functional%20Tests?branchName=master)](https://gvfs.visualstudio.com/ci/_build/latest?definitionId=6)| + + +## Mac +|Branch|Unit Tests|Functional Tests| +|:--:|:--:|:--:| +|**master**|[![Build status](https://gvfs.visualstudio.com/ci/_apis/build/status/CI%20-%20Mac%20-%20master?branchName=master)](https://gvfs.visualstudio.com/ci/_build/latest?definitionId=15)|| + ## What is VFS for Git? VFS stands for Virtual File System. VFS for Git virtualizes the file system beneath your git repo so that git and all tools From 5db30b34ce6c6adf4eb376869dab59eda7bb012d Mon Sep 17 00:00:00 2001 From: Nick Graczyk Date: Wed, 15 Aug 2018 17:00:04 -0700 Subject: [PATCH 022/272] Rework Git version parsing into a single script --- Scripts/Mac/BuildGVFSForMac.sh | 3 +-- Scripts/Mac/DownloadGVFSGit.sh | 3 +-- Scripts/Mac/GetGitVersionNumber.sh | 4 ++++ Scripts/Mac/PrepFunctionalTests.sh | 3 +-- 4 files changed, 7 insertions(+), 6 deletions(-) create mode 100755 Scripts/Mac/GetGitVersionNumber.sh diff --git a/Scripts/Mac/BuildGVFSForMac.sh b/Scripts/Mac/BuildGVFSForMac.sh index c62426043..2eb875c77 100755 --- a/Scripts/Mac/BuildGVFSForMac.sh +++ b/Scripts/Mac/BuildGVFSForMac.sh @@ -33,8 +33,7 @@ if [ ! -d $BUILDDIR ]; then fi $SCRIPTDIR/DownloadGVFSGit.sh || exit 1 -GVFSPROPS=$SRCDIR/GVFS/GVFS.Build/GVFS.props -GITVERSION="$(cat $GVFSPROPS | grep GitPackageVersion | grep -Eo '[0-9.]+(-\w+)?')" +GITVERSION="$($SCRIPTDIR/GetGitVersionNumber.sh)" GITPATH="$(find $PACKAGES/gitformac.gvfs.installer/$GITVERSION -type f -name *.dmg)" || exit 1 # Now that we have a path containing the version number, generate GVFSConstants.GitVersion.cs $SCRIPTDIR/GenerateGitVersionConstants.sh "$GITPATH" $BUILDDIR || exit 1 diff --git a/Scripts/Mac/DownloadGVFSGit.sh b/Scripts/Mac/DownloadGVFSGit.sh index 4ca725b9e..74f8bc528 100755 --- a/Scripts/Mac/DownloadGVFSGit.sh +++ b/Scripts/Mac/DownloadGVFSGit.sh @@ -2,8 +2,7 @@ SCRIPTDIR="$(dirname ${BASH_SOURCE[0]})" SRCDIR=$SCRIPTDIR/../.. BUILDDIR=$SRCDIR/../BuildOutput/GVFS.Build PACKAGESDIR=$SRCDIR/../packages -GVFSPROPS=$SRCDIR/GVFS/GVFS.Build/GVFS.props -GITVERSION="$(cat $GVFSPROPS | grep GitPackageVersion | grep -Eo '[0-9.]+(-\w+)?')" +GITVERSION="$($SCRIPTDIR/GetGitVersionNumber.sh)" cp $SRCDIR/nuget.config $BUILDDIR dotnet new classlib -n GVFS.Restore -o $BUILDDIR --force dotnet add $BUILDDIR/GVFS.Restore.csproj package --package-directory $PACKAGESDIR GitForMac.GVFS.Installer --version $GITVERSION \ No newline at end of file diff --git a/Scripts/Mac/GetGitVersionNumber.sh b/Scripts/Mac/GetGitVersionNumber.sh new file mode 100755 index 000000000..6942b3a42 --- /dev/null +++ b/Scripts/Mac/GetGitVersionNumber.sh @@ -0,0 +1,4 @@ +SCRIPTDIR="$(dirname ${BASH_SOURCE[0]})" +GVFSPROPS=$SCRIPTDIR/../../GVFS/GVFS.Build/GVFS.props +GITVERSION="$(cat $GVFSPROPS | grep GitPackageVersion | grep -Eo '[0-9.]+(-\w+)?')" +echo $GITVERSION \ No newline at end of file diff --git a/Scripts/Mac/PrepFunctionalTests.sh b/Scripts/Mac/PrepFunctionalTests.sh index f1eca5248..f1a0c4260 100755 --- a/Scripts/Mac/PrepFunctionalTests.sh +++ b/Scripts/Mac/PrepFunctionalTests.sh @@ -3,8 +3,7 @@ SCRIPTDIR=$(dirname ${BASH_SOURCE[0]}) # Install GVFS-aware Git (that was downloaded by the build script) -GVFSPROPS=$SCRIPTDIR/../../GVFS/GVFS.Build/GVFS.props -GITVERSION="$(cat $GVFSPROPS | grep GitPackageVersion | grep -Eo '[0-9.]+(-\w+)?')" +GITVERSION="$($SCRIPTDIR/GetGitVersionNumber.sh)" ROOTDIR=$SCRIPTDIR/../../.. GITDIR=$ROOTDIR/packages/gitformac.gvfs.installer/$GITVERSION/tools if [[ ! -d $GITDIR ]]; then From 4a4f5e7c90eeedff4a95716c99800dd009d71688 Mon Sep 17 00:00:00 2001 From: William Baker Date: Wed, 15 Aug 2018 14:49:27 -0700 Subject: [PATCH 023/272] Remove unused local variable --- ProjFS.Mac/PrjFSKext/PrjFSKext/KauthHandler.cpp | 2 -- 1 file changed, 2 deletions(-) diff --git a/ProjFS.Mac/PrjFSKext/PrjFSKext/KauthHandler.cpp b/ProjFS.Mac/PrjFSKext/PrjFSKext/KauthHandler.cpp index ea3ed4d87..7d5f1b3d6 100644 --- a/ProjFS.Mac/PrjFSKext/PrjFSKext/KauthHandler.cpp +++ b/ProjFS.Mac/PrjFSKext/PrjFSKext/KauthHandler.cpp @@ -569,8 +569,6 @@ static bool ShouldHandleFileOpEvent( } *root = VirtualizationRoots_FindForVnode(vnode); - int16_t rootIndex = nullptr == *root ? -1 : (*root)->index; - if (nullptr == *root) { KextLog_FileNote(vnode, "ShouldHandleFileOpEvent(%d): No virtualization root found for file with set flag.", action); From 52ec02dba3419ccbb6885cb996a6b561c23148e0 Mon Sep 17 00:00:00 2001 From: William Baker Date: Thu, 16 Aug 2018 09:11:59 -0700 Subject: [PATCH 024/272] Fix test reliability issues with checks in GetObjectRoot --- .../Tools/GVFSFunctionalTestEnlistment.cs | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/GVFS/GVFS.FunctionalTests/Tools/GVFSFunctionalTestEnlistment.cs b/GVFS/GVFS.FunctionalTests/Tools/GVFSFunctionalTestEnlistment.cs index 6ba115c5f..2a0612163 100644 --- a/GVFS/GVFS.FunctionalTests/Tools/GVFSFunctionalTestEnlistment.cs +++ b/GVFS/GVFS.FunctionalTests/Tools/GVFSFunctionalTestEnlistment.cs @@ -115,7 +115,18 @@ public static string GetUniqueEnlistmentRootWithSpaces() public string GetObjectRoot(FileSystemRunner fileSystem) { IEnumerable localCacheRootItems = this.LocalCacheRoot.ShouldBeADirectory(fileSystem).WithItems(); - localCacheRootItems.Count().ShouldEqual(2, "Expected local cache root to contain 2 items. Actual items: " + string.Join(",", localCacheRootItems)); + + FileInfo[] files = localCacheRootItems.Where(info => info is FileInfo).Cast().ToArray(); + files.Where(f => string.Equals(f.Name, "mapping.dat", StringComparison.OrdinalIgnoreCase)) + .Count() + .ShouldEqual(1, "Local cache root should contain a single 'mapping.dat' file, actual files: " + string.Join(",", files)); + + IEnumerable unexpectedFiles = files.Where( + f => !string.Equals(f.Name, "mapping.dat", StringComparison.OrdinalIgnoreCase) && + !string.Equals(f.Name, "mapping.dat.lock", StringComparison.OrdinalIgnoreCase)); + + unexpectedFiles.Any().ShouldBeFalse("Local cache root contains unexpected files: " + string.Join(",", unexpectedFiles)); + DirectoryInfo[] directories = localCacheRootItems.Where(info => info is DirectoryInfo).Cast().ToArray(); directories.Length.ShouldEqual(1, this.LocalCacheRoot + " is expected to have only one folder. Actual: " + directories.Count()); return Path.Combine(directories[0].FullName, "gitObjects"); From 04e032237382c8d1ed2a1bcf04f0ec630e8e27f4 Mon Sep 17 00:00:00 2001 From: William Baker Date: Thu, 16 Aug 2018 10:42:27 -0700 Subject: [PATCH 025/272] Simplify checks in GetObjectRoot --- .../Should/FileSystemShouldExtensions.cs | 16 +++++++++++ .../Tools/GVFSFunctionalTestEnlistment.cs | 27 ++++++++++--------- .../Should/EnumerableShouldExtensions.cs | 2 +- 3 files changed, 31 insertions(+), 14 deletions(-) diff --git a/GVFS/GVFS.FunctionalTests/Should/FileSystemShouldExtensions.cs b/GVFS/GVFS.FunctionalTests/Should/FileSystemShouldExtensions.cs index 543581e74..449c898a1 100644 --- a/GVFS/GVFS.FunctionalTests/Should/FileSystemShouldExtensions.cs +++ b/GVFS/GVFS.FunctionalTests/Should/FileSystemShouldExtensions.cs @@ -173,6 +173,22 @@ public IEnumerable WithItems() return this.WithItems("*"); } + public IEnumerable WithFiles() + { + IEnumerable items = this.WithItems(); + IEnumerable files = items.Where(info => info is FileInfo).Cast(); + files.Any().ShouldEqual(true, this.Path + " does not have any files. Contents: " + string.Join(",", items)); + return files; + } + + public IEnumerable WithDirectories() + { + IEnumerable items = this.WithItems(); + IEnumerable directories = items.Where(info => info is DirectoryInfo).Cast(); + directories.Any().ShouldEqual(true, this.Path + " does not have any directories. Contents: " + string.Join(",", items)); + return directories; + } + public IEnumerable WithItems(string searchPattern) { DirectoryInfo directory = new DirectoryInfo(this.Path); diff --git a/GVFS/GVFS.FunctionalTests/Tools/GVFSFunctionalTestEnlistment.cs b/GVFS/GVFS.FunctionalTests/Tools/GVFSFunctionalTestEnlistment.cs index 2a0612163..bda8a8050 100644 --- a/GVFS/GVFS.FunctionalTests/Tools/GVFSFunctionalTestEnlistment.cs +++ b/GVFS/GVFS.FunctionalTests/Tools/GVFSFunctionalTestEnlistment.cs @@ -114,21 +114,22 @@ public static string GetUniqueEnlistmentRootWithSpaces() public string GetObjectRoot(FileSystemRunner fileSystem) { - IEnumerable localCacheRootItems = this.LocalCacheRoot.ShouldBeADirectory(fileSystem).WithItems(); + Path.Combine(this.LocalCacheRoot, "mapping.dat").ShouldBeAFile(fileSystem); - FileInfo[] files = localCacheRootItems.Where(info => info is FileInfo).Cast().ToArray(); - files.Where(f => string.Equals(f.Name, "mapping.dat", StringComparison.OrdinalIgnoreCase)) - .Count() - .ShouldEqual(1, "Local cache root should contain a single 'mapping.dat' file, actual files: " + string.Join(",", files)); - - IEnumerable unexpectedFiles = files.Where( - f => !string.Equals(f.Name, "mapping.dat", StringComparison.OrdinalIgnoreCase) && - !string.Equals(f.Name, "mapping.dat.lock", StringComparison.OrdinalIgnoreCase)); - - unexpectedFiles.Any().ShouldBeFalse("Local cache root contains unexpected files: " + string.Join(",", unexpectedFiles)); + HashSet allowedFileNames = new HashSet(StringComparer.OrdinalIgnoreCase) + { + "mapping.dat", + "mapping.dat.lock" // mapping.dat.lock can be present, but doesn't have to be present + }; - DirectoryInfo[] directories = localCacheRootItems.Where(info => info is DirectoryInfo).Cast().ToArray(); - directories.Length.ShouldEqual(1, this.LocalCacheRoot + " is expected to have only one folder. Actual: " + directories.Count()); + this.LocalCacheRoot.ShouldBeADirectory(fileSystem).WithFiles().ShouldNotContain(f => !allowedFileNames.Contains(f.Name)); + + + DirectoryInfo[] directories = this.LocalCacheRoot.ShouldBeADirectory(fileSystem).WithDirectories().ToArray(); + directories.Length.ShouldEqual( + 1, + this.LocalCacheRoot + " is expected to have only one folder. Actual folders: " + string.Join(",", directories)); + return Path.Combine(directories[0].FullName, "gitObjects"); } diff --git a/GVFS/GVFS.Tests/Should/EnumerableShouldExtensions.cs b/GVFS/GVFS.Tests/Should/EnumerableShouldExtensions.cs index 4db59a36e..320bc2a81 100644 --- a/GVFS/GVFS.Tests/Should/EnumerableShouldExtensions.cs +++ b/GVFS/GVFS.Tests/Should/EnumerableShouldExtensions.cs @@ -48,7 +48,7 @@ public static T ShouldContainSingle(this IEnumerable group, Func public static void ShouldNotContain(this IEnumerable group, Func predicate) { T item = group.SingleOrDefault(predicate); - item.ShouldEqual(default(T)); + item.ShouldEqual(default(T), "Unexpected matching entry found in {" + string.Join(",", group) + "}"); } public static IEnumerable ShouldNotContain(this IEnumerable group, IEnumerable unexpectedValues, Func predicate) From af53bddb5760d90d751f3a170aeab58fa7f85c3d Mon Sep 17 00:00:00 2001 From: William Baker Date: Thu, 16 Aug 2018 10:55:35 -0700 Subject: [PATCH 026/272] Fix StyleCop error --- GVFS/GVFS.FunctionalTests/Tools/GVFSFunctionalTestEnlistment.cs | 1 - 1 file changed, 1 deletion(-) diff --git a/GVFS/GVFS.FunctionalTests/Tools/GVFSFunctionalTestEnlistment.cs b/GVFS/GVFS.FunctionalTests/Tools/GVFSFunctionalTestEnlistment.cs index bda8a8050..d1f1734b3 100644 --- a/GVFS/GVFS.FunctionalTests/Tools/GVFSFunctionalTestEnlistment.cs +++ b/GVFS/GVFS.FunctionalTests/Tools/GVFSFunctionalTestEnlistment.cs @@ -124,7 +124,6 @@ public string GetObjectRoot(FileSystemRunner fileSystem) this.LocalCacheRoot.ShouldBeADirectory(fileSystem).WithFiles().ShouldNotContain(f => !allowedFileNames.Contains(f.Name)); - DirectoryInfo[] directories = this.LocalCacheRoot.ShouldBeADirectory(fileSystem).WithDirectories().ToArray(); directories.Length.ShouldEqual( 1, From 81afe7f9bca68a03d644bfc1516df4c5bf382bfc Mon Sep 17 00:00:00 2001 From: William Baker Date: Wed, 15 Aug 2018 16:30:28 -0700 Subject: [PATCH 027/272] Mac: Add PreDelete Notification --- GVFS/GVFS.FunctionalTests/Categories.cs | 1 + .../FileSystemRunners/BashRunner.cs | 15 ++- GVFS/GVFS.FunctionalTests/Program.cs | 1 + .../BasicFileSystemTests.cs | 1 - .../EnlistmentPerFixture/GitFilesTests.cs | 51 ++++---- .../GitMoveRenameTests.cs | 23 +++- .../MoveRenameFileTests.cs | 1 + .../MoveRenameFileTests_2.cs | 1 + .../MoveRenameFolderTests.cs | 4 +- .../WorkingDirectoryTests.cs | 7 +- .../GVFS.FunctionalTests/Tools/GVFSHelpers.cs | 2 + .../MacFileSystemVirtualizer.cs | 53 +++++++- .../WindowsFileSystemVirtualizer.cs | 16 +-- .../FileSystem/FileSystemVirtualizer.cs | 19 +++ .../PrjFSKext.xcodeproj/project.pbxproj | 3 + .../PrjFSKext/PrjFSKext/KauthHandler.cpp | 115 ++++++++++++++++-- ProjFS.Mac/PrjFSKext/public/Message.h | 2 + .../VirtualizationInstance.cs | 6 +- ProjFS.Mac/PrjFSLib/PrjFSLib.cpp | 87 ++++++++++--- 19 files changed, 338 insertions(+), 70 deletions(-) diff --git a/GVFS/GVFS.FunctionalTests/Categories.cs b/GVFS/GVFS.FunctionalTests/Categories.cs index cfbf5cdf1..14fa221da 100644 --- a/GVFS/GVFS.FunctionalTests/Categories.cs +++ b/GVFS/GVFS.FunctionalTests/Categories.cs @@ -12,6 +12,7 @@ public static class Mac { public const string M1 = "M1_CloneAndMount"; public const string M2 = "M2_StaticViewGitCommands"; + public const string M2TODO = "M2_StaticViewGitCommandsStillTODO"; public const string M3 = "M3_AllGitCommands"; public const string M4 = "M4_All"; } diff --git a/GVFS/GVFS.FunctionalTests/FileSystemRunners/BashRunner.cs b/GVFS/GVFS.FunctionalTests/FileSystemRunners/BashRunner.cs index 6b2310a70..00544dac3 100644 --- a/GVFS/GVFS.FunctionalTests/FileSystemRunners/BashRunner.cs +++ b/GVFS/GVFS.FunctionalTests/FileSystemRunners/BashRunner.cs @@ -4,6 +4,7 @@ using System; using System.Diagnostics; using System.IO; +using System.Runtime.InteropServices; using System.Threading; namespace GVFS.FunctionalTests.FileSystemRunners @@ -33,6 +34,11 @@ public class BashRunner : ShellRunner "Permission denied" }; + private static string[] resourceUnavailableMessage = new string[] + { + "Resource temporarily unavailable", + }; + private readonly string pathToBash; public BashRunner() @@ -233,7 +239,14 @@ public override void DeleteFile_FileShouldNotBeFound(string path) public override void DeleteFile_AccessShouldBeDenied(string path) { - this.DeleteFile(path).ShouldContain(permissionDeniedMessage); + if (RuntimeInformation.IsOSPlatform(OSPlatform.Windows)) + { + this.DeleteFile(path).ShouldContain(permissionDeniedMessage); + } + else + { + this.DeleteFile(path).ShouldContain(resourceUnavailableMessage); + } } public override void ReadAllText_FileShouldNotBeFound(string path) diff --git a/GVFS/GVFS.FunctionalTests/Program.cs b/GVFS/GVFS.FunctionalTests/Program.cs index a99040e47..62db991d8 100644 --- a/GVFS/GVFS.FunctionalTests/Program.cs +++ b/GVFS/GVFS.FunctionalTests/Program.cs @@ -67,6 +67,7 @@ public static void Main(string[] args) { includeCategories.Add(Categories.Mac.M1); includeCategories.Add(Categories.Mac.M2); + excludeCategories.Add(Categories.Mac.M2TODO); excludeCategories.Add(Categories.Mac.M3); excludeCategories.Add(Categories.Mac.M4); excludeCategories.Add(Categories.Windows); diff --git a/GVFS/GVFS.FunctionalTests/Tests/EnlistmentPerFixture/BasicFileSystemTests.cs b/GVFS/GVFS.FunctionalTests/Tests/EnlistmentPerFixture/BasicFileSystemTests.cs index ac87acb32..24c32ef4f 100644 --- a/GVFS/GVFS.FunctionalTests/Tests/EnlistmentPerFixture/BasicFileSystemTests.cs +++ b/GVFS/GVFS.FunctionalTests/Tests/EnlistmentPerFixture/BasicFileSystemTests.cs @@ -784,7 +784,6 @@ public void MoveDotGitFullFolderTreeToDotGitFullFolder(FileSystemRunner fileSyst } [TestCaseSource(typeof(FileSystemRunner), FileSystemRunner.TestRunners)] - [Category(Categories.Mac.M4)] public void DeleteIndexFileFails(FileSystemRunner fileSystem) { string indexFilePath = this.Enlistment.GetVirtualPathTo(Path.Combine(".git", "index")); diff --git a/GVFS/GVFS.FunctionalTests/Tests/EnlistmentPerFixture/GitFilesTests.cs b/GVFS/GVFS.FunctionalTests/Tests/EnlistmentPerFixture/GitFilesTests.cs index 7b24445b9..1ec43b310 100644 --- a/GVFS/GVFS.FunctionalTests/Tests/EnlistmentPerFixture/GitFilesTests.cs +++ b/GVFS/GVFS.FunctionalTests/Tests/EnlistmentPerFixture/GitFilesTests.cs @@ -13,6 +13,7 @@ namespace GVFS.FunctionalTests.Tests.EnlistmentPerFixture { [TestFixtureSource(typeof(GitFilesTestsRunners), GitFilesTestsRunners.TestRunners)] + [Category(Categories.Mac.M2)] public class GitFilesTests : TestsWithEnlistmentPerFixture { private FileSystemRunner fileSystem; @@ -23,6 +24,7 @@ public GitFilesTests(FileSystemRunner fileSystem) } [TestCase, Order(1)] + [Category(Categories.Mac.M2TODO)] public void CreateFileTest() { string fileName = "file1.txt"; @@ -36,6 +38,7 @@ public void CreateFileTest() } [TestCase, Order(2)] + [Category(Categories.Mac.M2TODO)] public void CreateFileInFolderTest() { string folderName = "folder2"; @@ -51,11 +54,12 @@ public void CreateFileInFolderTest() this.Enlistment.WaitForBackgroundOperations().ShouldEqual(true, "Background operations failed to complete."); - GVFSHelpers.ModifiedPathsShouldContain(this.fileSystem, this.Enlistment.DotGVFSRoot, folderName + "/" + Environment.NewLine); - GVFSHelpers.ModifiedPathsShouldContain(this.fileSystem, this.Enlistment.DotGVFSRoot, folderName + "/" + fileName + Environment.NewLine); + GVFSHelpers.ModifiedPathsShouldContain(this.fileSystem, this.Enlistment.DotGVFSRoot, folderName + "/" + GVFSHelpers.ModifiedPathsNewLine); + GVFSHelpers.ModifiedPathsShouldContain(this.fileSystem, this.Enlistment.DotGVFSRoot, folderName + "/" + fileName + GVFSHelpers.ModifiedPathsNewLine); } [TestCase, Order(3)] + [Category(Categories.Mac.M2TODO)] public void RenameEmptyFolderTest() { string folderName = "folder3a"; @@ -76,6 +80,7 @@ public void RenameEmptyFolderTest() } [TestCase, Order(4)] + [Category(Categories.Mac.M2TODO)] public void RenameFolderTest() { string folderName = "folder4a"; @@ -108,6 +113,7 @@ public void RenameFolderTest() } [TestCase, Order(5)] + [Category(Categories.Mac.M2TODO)] public void CaseOnlyRenameOfNewFolderKeepsExcludeEntries() { string[] expectedModifiedPathsEntries = @@ -129,7 +135,6 @@ public void CaseOnlyRenameOfNewFolderKeepsExcludeEntries() } [TestCase, Order(6)] - [Category(Categories.Mac.M2)] public void ReadingFileDoesNotUpdateIndexOrSparseCheckout() { string gitFileToCheck = "GVFS/GVFS.FunctionalTests/Category/CategoryConstants.cs"; @@ -153,11 +158,13 @@ public void ReadingFileDoesNotUpdateIndexOrSparseCheckout() afterUpdateResult.Output.StartsWith("S ").ShouldEqual(true); afterUpdateResult.Output.ShouldContain("ctime: 0:0", "mtime: 0:0", "size: 0\t"); - GVFSHelpers.ModifiedPathsShouldNotContain(this.fileSystem, this.Enlistment.DotGVFSRoot, gitFileToCheck + Environment.NewLine); + GVFSHelpers.ModifiedPathsShouldNotContain(this.fileSystem, this.Enlistment.DotGVFSRoot, gitFileToCheck + GVFSHelpers.ModifiedPathsNewLine); } + // TODO(Mac): Enable this test once the LockHolder is converted to .NET Core [TestCase, Order(7)] - public void ModifiedFileWillGetSkipworktreeBitCleared() + [Category(Categories.Mac.M2TODO)] + public void ModifiedFileWillGetAddedToModifiedPathsFile() { string gitFileToTest = "GVFS/GVFS.Common/RetryWrapper.cs"; string fileToCreate = this.Enlistment.GetVirtualPathTo(gitFileToTest); @@ -171,11 +178,12 @@ public void ModifiedFileWillGetSkipworktreeBitCleared() this.Enlistment.WaitForBackgroundOperations().ShouldEqual(true, "Background operations did not complete."); - GVFSHelpers.ModifiedPathsShouldContain(this.fileSystem, this.Enlistment.DotGVFSRoot, gitFileToTest + Environment.NewLine); + GVFSHelpers.ModifiedPathsShouldContain(this.fileSystem, this.Enlistment.DotGVFSRoot, gitFileToTest + GVFSHelpers.ModifiedPathsNewLine); this.VerifyWorktreeBit(gitFileToTest, LsFilesStatus.Cached); } [TestCase, Order(8)] + [Category(Categories.Mac.M2TODO)] public void RenamedFileAddedToSparseCheckoutAndSkipWorktreeBitCleared() { string fileToRenameEntry = "Test_EPF_MoveRenameFileTests/ChangeUnhydratedFileName/Program.cs"; @@ -189,14 +197,15 @@ public void RenamedFileAddedToSparseCheckoutAndSkipWorktreeBitCleared() this.Enlistment.GetVirtualPathTo(fileToRenameTargetRelativePath)); this.Enlistment.WaitForBackgroundOperations().ShouldEqual(true, "Background operations failed to complete."); - GVFSHelpers.ModifiedPathsShouldContain(this.fileSystem, this.Enlistment.DotGVFSRoot, fileToRenameEntry + Environment.NewLine); - GVFSHelpers.ModifiedPathsShouldContain(this.fileSystem, this.Enlistment.DotGVFSRoot, fileToRenameTargetEntry + Environment.NewLine); + GVFSHelpers.ModifiedPathsShouldContain(this.fileSystem, this.Enlistment.DotGVFSRoot, fileToRenameEntry + GVFSHelpers.ModifiedPathsNewLine); + GVFSHelpers.ModifiedPathsShouldContain(this.fileSystem, this.Enlistment.DotGVFSRoot, fileToRenameTargetEntry + GVFSHelpers.ModifiedPathsNewLine); // Verify skip-worktree cleared this.VerifyWorktreeBit(fileToRenameEntry, LsFilesStatus.Cached); } [TestCase, Order(9)] + [Category(Categories.Mac.M2TODO)] public void RenamedFileAndOverwrittenTargetAddedToSparseCheckoutAndSkipWorktreeBitCleared() { string fileToRenameEntry = "Test_EPF_MoveRenameFileTests_2/MoveUnhydratedFileToOverwriteUnhydratedFileAndWrite/RunUnitTests.bat"; @@ -211,8 +220,8 @@ public void RenamedFileAndOverwrittenTargetAddedToSparseCheckoutAndSkipWorktreeB this.Enlistment.GetVirtualPathTo(fileToRenameTargetRelativePath)); this.Enlistment.WaitForBackgroundOperations().ShouldEqual(true, "Background operations failed to complete."); - GVFSHelpers.ModifiedPathsShouldContain(this.fileSystem, this.Enlistment.DotGVFSRoot, fileToRenameEntry + Environment.NewLine); - GVFSHelpers.ModifiedPathsShouldContain(this.fileSystem, this.Enlistment.DotGVFSRoot, fileToRenameTargetEntry + Environment.NewLine); + GVFSHelpers.ModifiedPathsShouldContain(this.fileSystem, this.Enlistment.DotGVFSRoot, fileToRenameEntry + GVFSHelpers.ModifiedPathsNewLine); + GVFSHelpers.ModifiedPathsShouldContain(this.fileSystem, this.Enlistment.DotGVFSRoot, fileToRenameTargetEntry + GVFSHelpers.ModifiedPathsNewLine); // Verify skip-worktree cleared this.VerifyWorktreeBit(fileToRenameEntry, LsFilesStatus.Cached); @@ -220,23 +229,22 @@ public void RenamedFileAndOverwrittenTargetAddedToSparseCheckoutAndSkipWorktreeB } [TestCase, Order(10)] - public void DeletedFileAddedToSparseCheckoutAndSkipWorktreeBitCleared() + public void DeletedFileAddedToModifiedPathsFile() { string fileToDeleteEntry = "GVFlt_DeleteFileTest/GVFlt_DeleteFullFileWithoutFileContext_DeleteOnClose/a.txt"; - string fileToDeleteRelativePath = "GVFlt_DeleteFileTest\\GVFlt_DeleteFullFileWithoutFileContext_DeleteOnClose\\a.txt"; this.VerifyWorktreeBit(fileToDeleteEntry, LsFilesStatus.SkipWorktree); - this.fileSystem.DeleteFile(this.Enlistment.GetVirtualPathTo(fileToDeleteRelativePath)); + this.fileSystem.DeleteFile(this.Enlistment.GetVirtualPathTo(fileToDeleteEntry)); this.Enlistment.WaitForBackgroundOperations().ShouldEqual(true, "Background operations failed to complete."); - GVFSHelpers.ModifiedPathsShouldContain(this.fileSystem, this.Enlistment.DotGVFSRoot, fileToDeleteEntry + Environment.NewLine); + GVFSHelpers.ModifiedPathsShouldContain(this.fileSystem, this.Enlistment.DotGVFSRoot, fileToDeleteEntry + GVFSHelpers.ModifiedPathsNewLine); // Verify skip-worktree cleared this.VerifyWorktreeBit(fileToDeleteEntry, LsFilesStatus.Cached); } [TestCase, Order(11)] - public void DeletedFolderAndChildrenAddedToSparseCheckoutAndSkipWorktreeBitCleared() + public void DeletedFolderAndChildrenAddedToToModifiedPathsFile() { string folderToDelete = "Scripts"; string[] filesToDelete = new string[] @@ -268,25 +276,25 @@ public void DeletedFolderAndChildrenAddedToSparseCheckoutAndSkipWorktreeBitClear } [TestCase, Order(12)] - public void FileRenamedOutOfRepoAddedToSparseCheckoutAndSkipWorktreeBitCleared() + public void FileRenamedOutOfRepoAddedToModifiedPathsFile() { string fileToRenameEntry = "GVFlt_MoveFileTest/PartialToOutside/from/lessInFrom.txt"; - string fileToRenameVirtualPath = this.Enlistment.GetVirtualPathTo("GVFlt_MoveFileTest\\PartialToOutside\\from\\lessInFrom.txt"); + string fileToRenameVirtualPath = this.Enlistment.GetVirtualPathTo(fileToRenameEntry); this.VerifyWorktreeBit(fileToRenameEntry, LsFilesStatus.SkipWorktree); string fileOutsideRepoPath = Path.Combine(this.Enlistment.EnlistmentRoot, "FileRenamedOutOfRepoAddedToSparseCheckoutAndSkipWorktreeBitCleared.txt"); this.fileSystem.MoveFile(fileToRenameVirtualPath, fileOutsideRepoPath); + fileOutsideRepoPath.ShouldBeAFile(this.fileSystem).WithContents("lessData"); this.Enlistment.WaitForBackgroundOperations().ShouldEqual(true, "Background operations failed to complete."); - GVFSHelpers.ModifiedPathsShouldContain(this.fileSystem, this.Enlistment.DotGVFSRoot, fileToRenameEntry + Environment.NewLine); + GVFSHelpers.ModifiedPathsShouldContain(this.fileSystem, this.Enlistment.DotGVFSRoot, fileToRenameEntry + GVFSHelpers.ModifiedPathsNewLine); // Verify skip-worktree cleared this.VerifyWorktreeBit(fileToRenameEntry, LsFilesStatus.Cached); } [TestCase, Order(13)] - [Category(Categories.Mac.M2)] public void OverwrittenFileAddedToSparseCheckoutAndSkipWorktreeBitCleared() { string fileToOverwriteEntry = "Test_EPF_WorkingDirectoryTests/1/2/3/4/ReadDeepProjectedFile.cpp"; @@ -300,13 +308,14 @@ public void OverwrittenFileAddedToSparseCheckoutAndSkipWorktreeBitCleared() fileToOverwriteVirtualPath.ShouldBeAFile(this.fileSystem).WithContents(testContents); - GVFSHelpers.ModifiedPathsShouldContain(this.fileSystem, this.Enlistment.DotGVFSRoot, fileToOverwriteEntry + "\r\n"); + GVFSHelpers.ModifiedPathsShouldContain(this.fileSystem, this.Enlistment.DotGVFSRoot, fileToOverwriteEntry + GVFSHelpers.ModifiedPathsNewLine); // Verify skip-worktree cleared this.VerifyWorktreeBit(fileToOverwriteEntry, LsFilesStatus.Cached); } [TestCase, Order(14)] + [Category(Categories.Mac.M2TODO)] public void SupersededFileAddedToSparseCheckoutAndSkipWorktreeBitCleared() { string fileToSupersedeEntry = "GVFlt_FileOperationTest/WriteAndVerify.txt"; @@ -318,7 +327,7 @@ public void SupersededFileAddedToSparseCheckoutAndSkipWorktreeBitCleared() SupersedeFile(fileToSupersedePath, newContent).ShouldEqual(true); this.Enlistment.WaitForBackgroundOperations().ShouldEqual(true, "Background operations failed to complete."); - GVFSHelpers.ModifiedPathsShouldContain(this.fileSystem, this.Enlistment.DotGVFSRoot, fileToSupersedeEntry + Environment.NewLine); + GVFSHelpers.ModifiedPathsShouldContain(this.fileSystem, this.Enlistment.DotGVFSRoot, fileToSupersedeEntry + GVFSHelpers.ModifiedPathsNewLine); // Verify skip-worktree cleared this.VerifyWorktreeBit(fileToSupersedeEntry, LsFilesStatus.Cached); diff --git a/GVFS/GVFS.FunctionalTests/Tests/EnlistmentPerFixture/GitMoveRenameTests.cs b/GVFS/GVFS.FunctionalTests/Tests/EnlistmentPerFixture/GitMoveRenameTests.cs index 2e73ea288..7ec766bb8 100644 --- a/GVFS/GVFS.FunctionalTests/Tests/EnlistmentPerFixture/GitMoveRenameTests.cs +++ b/GVFS/GVFS.FunctionalTests/Tests/EnlistmentPerFixture/GitMoveRenameTests.cs @@ -11,6 +11,7 @@ namespace GVFS.FunctionalTests.Tests.EnlistmentPerFixture { [TestFixtureSource(typeof(FileSystemRunner), FileSystemRunner.TestRunners)] [Category(Categories.GitCommands)] + [Category(Categories.Mac.M2)] public class GitMoveRenameTests : TestsWithEnlistmentPerFixture { private string testFileContents = "0123456789"; @@ -31,6 +32,7 @@ public void GitStatus() } [TestCase, Order(2)] + [Category(Categories.Mac.M2TODO)] public void GitStatusAfterNewFile() { string filename = "new.cs"; @@ -48,6 +50,7 @@ public void GitStatusAfterNewFile() } [TestCase, Order(3)] + [Category(Categories.Mac.M2TODO)] public void GitStatusAfterFileNameCaseChange() { string oldFilename = "new.cs"; @@ -65,6 +68,7 @@ public void GitStatusAfterFileNameCaseChange() } [TestCase, Order(4)] + [Category(Categories.Mac.M2TODO)] public void GitStatusAfterFileRename() { string oldFilename = "New.cs"; @@ -82,6 +86,7 @@ public void GitStatusAfterFileRename() } [TestCase, Order(5)] + [Category(Categories.Mac.M3)] public void GitStatusAndObjectAfterGitAdd() { string existingFilename = "test.cs"; @@ -117,6 +122,7 @@ public void GitStatusAndObjectAfterGitAdd() } [TestCase, Order(6)] + [Category(Categories.Mac.M3)] public void GitStatusAfterUnstage() { string existingFilename = "test.cs"; @@ -139,7 +145,7 @@ public void GitStatusAfterUnstage() public void GitStatusAfterFileDelete() { string existingFilename = "test.cs"; - this.Enlistment.GetVirtualPathTo(existingFilename).ShouldBeAFile(this.fileSystem); + this.EnsureTestFileExists(existingFilename); this.fileSystem.DeleteFile(this.Enlistment.GetVirtualPathTo(existingFilename)); this.Enlistment.GetVirtualPathTo(existingFilename).ShouldNotExistOnDisk(this.fileSystem); @@ -170,6 +176,7 @@ public void GitWithEnvironmentVariables() } [TestCase, Order(9)] + [Category(Categories.Mac.M2TODO)] public void GitStatusAfterRenameFileIntoRepo() { string filename = "GitStatusAfterRenameFileIntoRepo.cs"; @@ -196,7 +203,7 @@ public void GitStatusAfterRenameFileIntoRepo() [TestCase, Order(10)] public void GitStatusAfterRenameFileOutOfRepo() { - string existingFilename = "Test_EPF_MoveRenameFileTests\\ChangeUnhydratedFileName\\Program.cs"; + string existingFilename = Path.Combine("Test_EPF_MoveRenameFileTests", "ChangeUnhydratedFileName", "Program.cs"); // Move the test file to this.Enlistment.EnlistmentRoot as it's outside of src // and is cleaned up when the functional tests run @@ -210,6 +217,7 @@ public void GitStatusAfterRenameFileOutOfRepo() } [TestCase, Order(11)] + [Category(Categories.Mac.M2TODO)] public void GitStatusAfterRenameFolderIntoRepo() { string folderName = "GitStatusAfterRenameFolderIntoRepo"; @@ -235,5 +243,16 @@ public void GitStatusAfterRenameFolderIntoRepo() folderName + "/", folderName + "/" + fileName); } + + private void EnsureTestFileExists(string relativePath) + { + string filePath = this.Enlistment.GetVirtualPathTo(relativePath); + if (!this.fileSystem.FileExists(filePath)) + { + this.fileSystem.WriteAllText(filePath, this.testFileContents); + } + + this.Enlistment.GetVirtualPathTo(relativePath).ShouldBeAFile(this.fileSystem); + } } } diff --git a/GVFS/GVFS.FunctionalTests/Tests/EnlistmentPerFixture/MoveRenameFileTests.cs b/GVFS/GVFS.FunctionalTests/Tests/EnlistmentPerFixture/MoveRenameFileTests.cs index 40e082093..5ea5b7ab0 100644 --- a/GVFS/GVFS.FunctionalTests/Tests/EnlistmentPerFixture/MoveRenameFileTests.cs +++ b/GVFS/GVFS.FunctionalTests/Tests/EnlistmentPerFixture/MoveRenameFileTests.cs @@ -10,6 +10,7 @@ namespace GVFS.FunctionalTests.Tests.EnlistmentPerFixture // TODO 452590 - Combine all of the MoveRenameTests into a single fixture, and have each use different // well known files [TestFixtureSource(typeof(FileSystemRunner), FileSystemRunner.TestRunners)] + [Category(Categories.Mac.M2TODO)] public class MoveRenameFileTests : TestsWithEnlistmentPerFixture { public const string TestFileContents = diff --git a/GVFS/GVFS.FunctionalTests/Tests/EnlistmentPerFixture/MoveRenameFileTests_2.cs b/GVFS/GVFS.FunctionalTests/Tests/EnlistmentPerFixture/MoveRenameFileTests_2.cs index c080c8967..01c2fe908 100644 --- a/GVFS/GVFS.FunctionalTests/Tests/EnlistmentPerFixture/MoveRenameFileTests_2.cs +++ b/GVFS/GVFS.FunctionalTests/Tests/EnlistmentPerFixture/MoveRenameFileTests_2.cs @@ -8,6 +8,7 @@ namespace GVFS.FunctionalTests.Tests.EnlistmentPerFixture // TODO 452590 - Combine all of the MoveRenameTests into a single fixture, and have each use different // well known files [TestFixtureSource(typeof(FileSystemRunner), FileSystemRunner.TestRunners)] + [Category(Categories.Mac.M2TODO)] public class MoveRenameFileTests_2 : TestsWithEnlistmentPerFixture { private const string TestFileFolder = "Test_EPF_MoveRenameFileTests_2"; diff --git a/GVFS/GVFS.FunctionalTests/Tests/EnlistmentPerFixture/MoveRenameFolderTests.cs b/GVFS/GVFS.FunctionalTests/Tests/EnlistmentPerFixture/MoveRenameFolderTests.cs index 54d059dbb..abd24be71 100644 --- a/GVFS/GVFS.FunctionalTests/Tests/EnlistmentPerFixture/MoveRenameFolderTests.cs +++ b/GVFS/GVFS.FunctionalTests/Tests/EnlistmentPerFixture/MoveRenameFolderTests.cs @@ -6,6 +6,7 @@ namespace GVFS.FunctionalTests.Tests.EnlistmentPerFixture { [TestFixtureSource(typeof(FileSystemRunner), FileSystemRunner.TestRunners)] + [Category(Categories.Mac.M2TODO)] public class MoveRenameFolderTests : TestsWithEnlistmentPerFixture { private const string TestFileContents = @@ -118,6 +119,7 @@ public void MoveUnhydratedFolderToFullFolderInDotGitFolder() } [TestCase] + [Category(Categories.Mac.M2)] public void MoveFullFolderToFullFolderInDotGitFolder() { string fileContents = "Test contents for MoveFullFolderToFullFolderInDotGitFolder"; @@ -132,7 +134,7 @@ public void MoveFullFolderToFullFolderInDotGitFolder() oldFilePath.ShouldBeAFile(this.fileSystem).WithContents(fileContents); string newFolderName = "NewMoveFullFolderToFullFolderInDotGitFolder"; - string newFolderPath = this.Enlistment.GetVirtualPathTo(".git\\" + newFolderName); + string newFolderPath = this.Enlistment.GetVirtualPathTo(".git", newFolderName); newFolderPath.ShouldNotExistOnDisk(this.fileSystem); this.fileSystem.CreateDirectory(newFolderPath); newFolderPath.ShouldBeADirectory(this.fileSystem); diff --git a/GVFS/GVFS.FunctionalTests/Tests/EnlistmentPerFixture/WorkingDirectoryTests.cs b/GVFS/GVFS.FunctionalTests/Tests/EnlistmentPerFixture/WorkingDirectoryTests.cs index 2f537d307..1f0a1e5b0 100644 --- a/GVFS/GVFS.FunctionalTests/Tests/EnlistmentPerFixture/WorkingDirectoryTests.cs +++ b/GVFS/GVFS.FunctionalTests/Tests/EnlistmentPerFixture/WorkingDirectoryTests.cs @@ -274,7 +274,6 @@ public void StreamAndRandomAccessReadWriteMemoryMappedProjectedFile() } [TestCase, Order(5)] - [Category(Categories.Mac.M3)] public void MoveProjectedFileToInvalidFolder() { string targetFolderName = "test_folder"; @@ -449,9 +448,9 @@ public void FolderContentsCorrectAfterCreateNewFolderRenameAndCheckoutCommitWith (folder + @"\MoveUnhydratedFileToDotGitFolder\Program.cs").ShouldBeAFile(this.fileSystem).WithContents(MoveRenameFileTests.TestFileContents); } - // TODO(Mac) This test is technically part of M2, but we need further investigation of why this test fails on build agents, but not on dev machines. + // TODO(Mac) This *should* be working already, we need further investigation of why this test fails on build agents, but not on dev machines. [TestCase, Order(15)] - [Category(Categories.Mac.M3)] + [Category(Categories.Mac.M2TODO)] public void FilterNonUTF8FileName() { string encodingFilename = "ريلٌأكتوبرûمارسأغسطسºٰٰۂْٗ۵ريلٌأك.txt"; @@ -498,8 +497,8 @@ public void AllNullObjectRedownloaded() (badObject as FileInfo).ShouldNotBeNull().Length.ShouldEqual(objectFileInfo.Length); } + // TODO(Mac): Figure out why git for Mac is not requesting a redownload of the truncated object [TestCase, Order(17)] - //// TODO(Mac): Figure out why git for Mac is not requesting a redownload of the truncated object [Category(Categories.Mac.M3)] public void TruncatedObjectRedownloaded() { diff --git a/GVFS/GVFS.FunctionalTests/Tools/GVFSHelpers.cs b/GVFS/GVFS.FunctionalTests/Tools/GVFSHelpers.cs index 95ee419bc..0831ab218 100644 --- a/GVFS/GVFS.FunctionalTests/Tools/GVFSHelpers.cs +++ b/GVFS/GVFS.FunctionalTests/Tools/GVFSHelpers.cs @@ -12,6 +12,8 @@ namespace GVFS.FunctionalTests.Tools { public static class GVFSHelpers { + public const string ModifiedPathsNewLine = "\r\n"; + public static readonly string BackgroundOpsFile = Path.Combine("databases", "BackgroundGitOperations.dat"); public static readonly string PlaceholderListFile = Path.Combine("databases", "PlaceholderList.dat"); public static readonly string RepoMetadataName = Path.Combine("databases", "RepoMetadata.dat"); diff --git a/GVFS/GVFS.Platform.Mac/MacFileSystemVirtualizer.cs b/GVFS/GVFS.Platform.Mac/MacFileSystemVirtualizer.cs index 8d7da3aa0..567226070 100644 --- a/GVFS/GVFS.Platform.Mac/MacFileSystemVirtualizer.cs +++ b/GVFS/GVFS.Platform.Mac/MacFileSystemVirtualizer.cs @@ -100,6 +100,7 @@ protected override bool TryStart(out string error) this.virtualizationInstance.OnEnumerateDirectory = this.OnEnumerateDirectory; this.virtualizationInstance.OnGetFileStream = this.OnGetFileStream; this.virtualizationInstance.OnFileModified = this.OnFileModified; + this.virtualizationInstance.OnPreDelete = this.OnPreDelete; uint threadCount = (uint)Environment.ProcessorCount * 2; @@ -165,7 +166,7 @@ private Result OnGetFileStream( activity.RelatedEvent(EventLevel.Informational, $"{nameof(this.OnGetFileStream)}_MountNotComplete", metadata); activity.Dispose(); - // TODO: Is this the correct Result to return? + // TODO(Mac): Is this the correct Result to return? return Result.EIOError; } @@ -174,7 +175,7 @@ private Result OnGetFileStream( activity.RelatedError(metadata, nameof(this.OnGetFileStream) + ": Unexpected placeholder version"); activity.Dispose(); - // TODO: Is this the correct Result to return? + // TODO(Mac): Is this the correct Result to return? return Result.EIOError; } @@ -276,6 +277,54 @@ private void OnFileModified(string relativePath) } } + private Result OnPreDelete(string relativePath, bool isDirectory) + { + try + { + if (!this.FileSystemCallbacks.IsMounted) + { + EventMetadata metadata = this.CreateEventMetadata(relativePath); + metadata.Add(nameof(isDirectory), isDirectory); + metadata.Add(TracingConstants.MessageKey.InfoMessage, $"{nameof(this.OnPreDelete)} failed, mount has not yet completed"); + this.Context.Tracer.RelatedEvent(EventLevel.Informational, $"{nameof(this.OnPreDelete)}_MountNotComplete", metadata); + + // TODO(Mac): Is this the correct Result to return? + return Result.EIOError; + } + + if (relativePath.Equals(GVFSConstants.DotGit.Index, StringComparison.OrdinalIgnoreCase)) + { + string lockedGitCommand = this.Context.Repository.GVFSLock.GetLockedGitCommand(); + if (string.IsNullOrEmpty(lockedGitCommand)) + { + EventMetadata metadata = this.CreateEventMetadata(relativePath); + metadata.Add(TracingConstants.MessageKey.WarningMessage, "Blocked index delete outside the lock"); + this.Context.Tracer.RelatedEvent(EventLevel.Warning, $"{nameof(this.OnPreDelete)}_BlockedIndexDelete", metadata); + + return Result.EAccessDenied; + } + } + + bool pathInsideDotGit = Virtualization.FileSystemCallbacks.IsPathInsideDotGit(relativePath); + if (pathInsideDotGit) + { + this.OnDotGitFileOrFolderDeleted(relativePath); + } + else + { + this.OnWorkingDirectoryFileOrFolderDeleted(relativePath, isDirectory); + } + } + catch (Exception e) + { + EventMetadata metadata = this.CreateEventMetadata(relativePath, e); + metadata.Add("isDirectory", isDirectory); + this.LogUnhandledExceptionAndExit(nameof(this.OnPreDelete), metadata); + } + + return Result.Success; + } + private Result OnEnumerateDirectory( ulong commandId, string relativePath, diff --git a/GVFS/GVFS.Platform.Windows/WindowsFileSystemVirtualizer.cs b/GVFS/GVFS.Platform.Windows/WindowsFileSystemVirtualizer.cs index 4f3b19b4e..02167bd3e 100644 --- a/GVFS/GVFS.Platform.Windows/WindowsFileSystemVirtualizer.cs +++ b/GVFS/GVFS.Platform.Windows/WindowsFileSystemVirtualizer.cs @@ -1284,21 +1284,7 @@ private void NotifyFileHandleClosedFileModifiedOrDeletedHandler( } else { - if (isDirectory) - { - // Don't want to add folders to the modified list if git is the one deleting the directory - GitCommandLineParser gitCommand = new GitCommandLineParser(this.Context.Repository.GVFSLock.GetLockedGitCommand()); - if (!gitCommand.IsValidGitCommand) - { - this.FileSystemCallbacks.OnFolderDeleted(virtualPath); - } - } - else - { - this.FileSystemCallbacks.OnFileDeleted(virtualPath); - } - - this.FileSystemCallbacks.InvalidateGitStatusCache(); + this.OnWorkingDirectoryFileOrFolderDeleted(virtualPath, isDirectory); } } } diff --git a/GVFS/GVFS.Virtualization/FileSystem/FileSystemVirtualizer.cs b/GVFS/GVFS.Virtualization/FileSystem/FileSystemVirtualizer.cs index 521b1aebb..33c90c988 100644 --- a/GVFS/GVFS.Virtualization/FileSystem/FileSystemVirtualizer.cs +++ b/GVFS/GVFS.Virtualization/FileSystem/FileSystemVirtualizer.cs @@ -190,6 +190,25 @@ protected void OnDotGitFileOrFolderDeleted(string relativePath) } } + protected void OnWorkingDirectoryFileOrFolderDeleted(string relativePath, bool isDirectory) + { + if (isDirectory) + { + // Don't want to add folders to the modified list if git is the one deleting the directory + GitCommandLineParser gitCommand = new GitCommandLineParser(this.Context.Repository.GVFSLock.GetLockedGitCommand()); + if (!gitCommand.IsValidGitCommand) + { + this.FileSystemCallbacks.OnFolderDeleted(relativePath); + } + } + else + { + this.FileSystemCallbacks.OnFileDeleted(relativePath); + } + + this.FileSystemCallbacks.InvalidateGitStatusCache(); + } + protected EventMetadata CreateEventMetadata( Guid enumerationId, string relativePath = null, diff --git a/ProjFS.Mac/PrjFSKext/PrjFSKext.xcodeproj/project.pbxproj b/ProjFS.Mac/PrjFSKext/PrjFSKext.xcodeproj/project.pbxproj index ae816c919..f55c07403 100644 --- a/ProjFS.Mac/PrjFSKext/PrjFSKext.xcodeproj/project.pbxproj +++ b/ProjFS.Mac/PrjFSKext/PrjFSKext.xcodeproj/project.pbxproj @@ -196,6 +196,7 @@ TargetAttributes = { C6C780AF207FC67200E7E054 = { CreatedOnToolsVersion = 9.3; + ProvisioningStyle = Automatic; }; }; }; @@ -413,6 +414,7 @@ PRODUCT_BUNDLE_IDENTIFIER = io.gvfs.PrjFSKext; PRODUCT_NAME = "$(TARGET_NAME)"; WARNING_CFLAGS = ( + "-Werror", "-Werror=undefined-internal", "-Werror=missing-prototypes", "-Werror=format", @@ -435,6 +437,7 @@ PRODUCT_BUNDLE_IDENTIFIER = io.gvfs.PrjFSKext; PRODUCT_NAME = "$(TARGET_NAME)"; WARNING_CFLAGS = ( + "-Werror", "-Werror=undefined-internal", "-Werror=missing-prototypes", "-Werror=format", diff --git a/ProjFS.Mac/PrjFSKext/PrjFSKext/KauthHandler.cpp b/ProjFS.Mac/PrjFSKext/PrjFSKext/KauthHandler.cpp index 7d5f1b3d6..b277e6fbe 100644 --- a/ProjFS.Mac/PrjFSKext/PrjFSKext/KauthHandler.cpp +++ b/ProjFS.Mac/PrjFSKext/PrjFSKext/KauthHandler.cpp @@ -35,6 +35,7 @@ static int HandleFileOpOperation( static int GetPid(vfs_context_t context); static uint32_t ReadVNodeFileFlags(vnode_t vn, vfs_context_t context); +static bool HasParentInVirtualizationRoot(vnode_t vnode, vfs_context_t context); static bool FileFlagsBitIsSet(uint32_t fileFlags, uint32_t bit); static bool ActionBitIsSet(kauth_action_t action, kauth_action_t mask); static bool ActionBitsNotSet(kauth_action_t action, kauth_action_t mask); @@ -215,6 +216,17 @@ void KauthHandler_HandleKernelMessageResponse(uint64_t messageId, MessageType re } Mutex_Release(s_outstandingMessagesMutex); } + + // The follow are not valid responses to kernel messages + case MessageType_Invalid: + case MessageType_UtoK_StartVirtualizationInstance: + case MessageType_UtoK_StopVirtualizationInstance: + case MessageType_KtoU_EnumerateDirectory: + case MessageType_KtoU_HydrateFile: + case MessageType_KtoU_NotifyFileModified: + case MessageType_KtoU_NotifyFilePreDelete: + case MessageType_KtoU_NotifyDirectoryPreDelete: + break; } return; @@ -284,10 +296,56 @@ static int HandleVnodeOperation( } } } + else if (ActionBitIsSet(action, KAUTH_VNODE_DELETE)) + { + if (!TrySendRequestAndWaitForResponse( + root, + MessageType_KtoU_NotifyDirectoryPreDelete, + currentVnode, + pid, + procname, + &kauthResult, + kauthError)) + { + goto CleanupAndReturn; + } + } } else { - if (ActionBitIsSet( + if (ActionBitIsSet(action, KAUTH_VNODE_DELETE)) + { + if (!TrySendRequestAndWaitForResponse( + root, + MessageType_KtoU_NotifyFilePreDelete, + currentVnode, + pid, + procname, + &kauthResult, + kauthError)) + { + goto CleanupAndReturn; + } + + // This delete could be part of a rename, and so we must hydrate the file now to + // ensure it's hydrated post-rename + if (kauthResult == KAUTH_RESULT_DEFER && + FileFlagsBitIsSet(currentVnodeFileFlags, FileFlags_IsEmpty)) + { + if (!TrySendRequestAndWaitForResponse( + root, + MessageType_KtoU_HydrateFile, + currentVnode, + pid, + procname, + &kauthResult, + kauthError)) + { + goto CleanupAndReturn; + } + } + } + else if (ActionBitIsSet( action, KAUTH_VNODE_READ_ATTRIBUTES | KAUTH_VNODE_WRITE_ATTRIBUTES | @@ -412,13 +470,21 @@ static bool ShouldHandleVnodeOpEvent( } *vnodeFileFlags = ReadVNodeFileFlags(vnode, context); - if (!FileFlagsBitIsSet(*vnodeFileFlags, FileFlags_IsInVirtualizationRoot)) + bool filedFlaggedAsInVirtualizationRoot = FileFlagsBitIsSet(*vnodeFileFlags, FileFlags_IsInVirtualizationRoot); + if (!filedFlaggedAsInVirtualizationRoot) { - // This vnode is not part of ANY virtualization root, so exit now before doing any more work. - // This gives us a cheap way to avoid adding overhead to IO outside of a virtualization root. - - *kauthResult = KAUTH_RESULT_DEFER; - return false; + // TODO(Mac): Determine the overhead of making this check for all actions and + // vnodes. If it's too high, this check can be further reduced to the scenarios + // that VFSForGit care about. + if (!HasParentInVirtualizationRoot(vnode, context)) + { + // This vnode is not part of ANY virtualization root (and none of its ancestors + // are either), so exit now before doing any more work. This gives us a cheap + // way to avoid adding overhead to IO outside of a virtualization root. + + *kauthResult = KAUTH_RESULT_DEFER; + return false; + } } *pid = GetPid(context); @@ -457,7 +523,12 @@ static bool ShouldHandleVnodeOpEvent( // TODO(Mac): why do we need to check rootIndex > 0 here? // If root was null, we would have already exited. And if not null, it can't be < 0. - if (rootIndex >= 0) + // + // TODO(Mac): Double check that rules are properly enforced when filedFlaggedAsInVirtualizationRoot + // is false. These checks are currently skipped when filedFlaggedAsInVirtualizationRoot is false to + // allow providers to create new files inside of their root before starting the virtualization + // instance + if (filedFlaggedAsInVirtualizationRoot && rootIndex >= 0) { bool vnodeIsDir = (*vnodeType == VDIR); @@ -740,6 +811,34 @@ static uint32_t ReadVNodeFileFlags(vnode_t vn, vfs_context_t context) return attributes.va_flags; } +static bool HasParentInVirtualizationRoot(vnode_t vnode, vfs_context_t context) +{ + vnode_get(vnode); + + bool inRoot = false; + // Search up the tree until we hit a folder know to be inside a virtualization root + // or THE root of the file system + while (NULLVP != vnode && !vnode_isvroot(vnode)) + { + uint32_t vnodeFileFlags = ReadVNodeFileFlags(vnode, context); + if (FileFlagsBitIsSet(vnodeFileFlags, FileFlags_IsInVirtualizationRoot)) + { + inRoot = true; + break; + } + vnode_t parent = vnode_getparent(vnode); + vnode_put(vnode); + vnode = parent; + } + + if (NULLVP != vnode) + { + vnode_put(vnode); + } + + return inRoot; +} + static bool FileFlagsBitIsSet(uint32_t fileFlags, uint32_t bit) { // Note: if multiple bits are set in 'bit', this will return true if ANY are set in fileFlags diff --git a/ProjFS.Mac/PrjFSKext/public/Message.h b/ProjFS.Mac/PrjFSKext/public/Message.h index f337b8f6d..5817586f2 100644 --- a/ProjFS.Mac/PrjFSKext/public/Message.h +++ b/ProjFS.Mac/PrjFSKext/public/Message.h @@ -17,6 +17,8 @@ typedef enum MessageType_KtoU_HydrateFile, MessageType_KtoU_NotifyFileModified, + MessageType_KtoU_NotifyFilePreDelete, + MessageType_KtoU_NotifyDirectoryPreDelete, // Responses MessageType_Response_Success, diff --git a/ProjFS.Mac/PrjFSLib.Mac.Managed/VirtualizationInstance.cs b/ProjFS.Mac/PrjFSLib.Mac.Managed/VirtualizationInstance.cs index da5a079b0..0fd6b161a 100644 --- a/ProjFS.Mac/PrjFSLib.Mac.Managed/VirtualizationInstance.cs +++ b/ProjFS.Mac/PrjFSLib.Mac.Managed/VirtualizationInstance.cs @@ -14,7 +14,8 @@ public class VirtualizationInstance public virtual EnumerateDirectoryCallback OnEnumerateDirectory { get; set; } public virtual GetFileStreamCallback OnGetFileStream { get; set; } - public virtual NotifyFileModified OnFileModified { get; set; } + public virtual NotifyFileModified OnFileModified { get; set; } + public virtual NotifyPreDeleteEvent OnPreDelete { get; set; } public static Result ConvertDirectoryToVirtualizationRoot(string fullPath) { @@ -134,6 +135,9 @@ private Result OnNotifyOperation( { switch (notificationType) { + case NotificationType.PreDelete: + return this.OnPreDelete(relativePath, isDirectory); + case NotificationType.FileModified: this.OnFileModified(relativePath); return Result.Success; diff --git a/ProjFS.Mac/PrjFSLib/PrjFSLib.cpp b/ProjFS.Mac/PrjFSLib/PrjFSLib.cpp index f126808eb..022718204 100644 --- a/ProjFS.Mac/PrjFSLib/PrjFSLib.cpp +++ b/ProjFS.Mac/PrjFSLib/PrjFSLib.cpp @@ -22,6 +22,8 @@ #include "PrjFSKext/public/Message.h" #include "PrjFSUser.hpp" +#define STRINGIFY(s) #s + using std::endl; using std::cerr; using std::unordered_map; using std::set; using std::string; using std::mutex; @@ -51,12 +53,18 @@ static errno_t RegisterVirtualizationRootPath(const char* path); static void HandleKernelRequest(Message requestSpec, void* messageMemory); static PrjFS_Result HandleEnumerateDirectoryRequest(const MessageHeader* request, const char* path); static PrjFS_Result HandleHydrateFileRequest(const MessageHeader* request, const char* path); -static PrjFS_Result HandleFileModifiedNotification(const MessageHeader* request, const char* path); +static PrjFS_Result HandleFileNotification( + const MessageHeader* request, + const char* path, + bool isDirectory, + PrjFS_NotificationType notificationType); static Message ParseMessageMemory(const void* messageMemory, uint32_t size); static void ClearMachNotification(mach_port_t port); +static const char* NotificationTypeToString(PrjFS_NotificationType notificationType); + // State static io_connect_t s_kernelServiceConnection = IO_OBJECT_NULL; static std::string s_virtualizationRootFullPath; @@ -415,7 +423,31 @@ static void HandleKernelRequest(Message request, void* messageMemory) case MessageType_KtoU_NotifyFileModified: { - result = HandleFileModifiedNotification(requestHeader, request.path); + result = HandleFileNotification( + requestHeader, + request.path, + false, // isDirectory + PrjFS_NotificationType_FileModified); + break; + } + + case MessageType_KtoU_NotifyFilePreDelete: + { + result = HandleFileNotification( + requestHeader, + request.path, + false, // isDirectory + PrjFS_NotificationType_PreDelete); + break; + } + + case MessageType_KtoU_NotifyDirectoryPreDelete: + { + result = HandleFileNotification( + requestHeader, + request.path, + true, // isDirectory + PrjFS_NotificationType_PreDelete); break; } } @@ -547,33 +579,33 @@ static PrjFS_Result HandleHydrateFileRequest(const MessageHeader* request, const return callbackResult; } -static PrjFS_Result HandleFileModifiedNotification(const MessageHeader* request, const char* path) +static PrjFS_Result HandleFileNotification( + const MessageHeader* request, + const char* path, + bool isDirectory, + PrjFS_NotificationType notificationType) { #ifdef DEBUG - std::cout << "PrjFSLib.HandleFileModifiedNotification: " << path << std::endl; + std::cout << "PrjFSLib.HandleFileNotification: " << path + << " notificationType: " << NotificationTypeToString(notificationType) << std::endl; #endif char fullPath[PrjFSMaxPath]; CombinePaths(s_virtualizationRootFullPath.c_str(), path, fullPath); PrjFSFileXAttrData xattrData = {}; - if (!GetXAttr(fullPath, PrjFSFileXAttrName, sizeof(PrjFSFileXAttrData), &xattrData)) - { - return PrjFS_Result_EIOError; - } - - s_callbacks.NotifyOperation( + GetXAttr(fullPath, PrjFSFileXAttrName, sizeof(PrjFSFileXAttrData), &xattrData); + + return s_callbacks.NotifyOperation( 0 /* commandId */, path, xattrData.providerId, xattrData.contentId, request->pid, request->procname, - false /* isDirectory */, - PrjFS_NotificationType_FileModified, + isDirectory, + notificationType, nullptr /* destinationRelativePath */); - - return PrjFS_Result_Success; } static bool InitializeEmptyPlaceholder(const char* fullPath) @@ -712,3 +744,30 @@ static void ClearMachNotification(mach_port_t port) } msg; mach_msg(&msg.msgHdr, MACH_RCV_MSG | MACH_RCV_TIMEOUT, 0, sizeof(msg), port, 0, MACH_PORT_NULL); } + +static const char* NotificationTypeToString(PrjFS_NotificationType notificationType) +{ + switch(notificationType) + { + case PrjFS_NotificationType_Invalid: + return STRINGIFY(PrjFS_NotificationType_Invalid); + + case PrjFS_NotificationType_None: + return STRINGIFY(PrjFS_NotificationType_None); + case PrjFS_NotificationType_NewFileCreated: + return STRINGIFY(PrjFS_NotificationType_NewFileCreated); + case PrjFS_NotificationType_PreDelete: + return STRINGIFY(PrjFS_NotificationType_PreDelete); + case PrjFS_NotificationType_FileRenamed: + return STRINGIFY(PrjFS_NotificationType_FileRenamed); + case PrjFS_NotificationType_PreConvertToFull: + return STRINGIFY(PrjFS_NotificationType_PreConvertToFull); + + case PrjFS_NotificationType_PreModify: + return STRINGIFY(PrjFS_NotificationType_PreModify); + case PrjFS_NotificationType_FileModified: + return STRINGIFY(PrjFS_NotificationType_FileModified); + case PrjFS_NotificationType_FileDeleted: + return STRINGIFY(PrjFS_NotificationType_FileDeleted); + } +} From ccbd974e185a15d14853e8319a6590781dc47975 Mon Sep 17 00:00:00 2001 From: Kyle Rader Date: Mon, 13 Aug 2018 14:32:32 -0700 Subject: [PATCH 028/272] Add --force flag to fast fetch (apply patch from VSTS work) --- GVFS/FastFetch/CheckoutPrefetcher.cs | 4 +- GVFS/FastFetch/FastFetchVerb.cs | 9 ++- GVFS/GVFS.Common/Prefetch/BlobPrefetcher.cs | 15 +++-- GVFS/GVFS.Common/Prefetch/Jobs/CheckoutJob.cs | 16 +++++- .../Tests/FastFetchTests.cs | 56 +++++++++++++++++++ 5 files changed, 89 insertions(+), 11 deletions(-) diff --git a/GVFS/FastFetch/CheckoutPrefetcher.cs b/GVFS/FastFetch/CheckoutPrefetcher.cs index 16a8a09bd..b344a5847 100644 --- a/GVFS/FastFetch/CheckoutPrefetcher.cs +++ b/GVFS/FastFetch/CheckoutPrefetcher.cs @@ -33,7 +33,7 @@ public CheckoutPrefetcher( } /// A specific branch to filter for, or null for all branches returned from info/refs - public override void Prefetch(string branchOrCommit, bool isBranch) + public override void Prefetch(string branchOrCommit, bool isBranch, bool force = false) { if (string.IsNullOrWhiteSpace(branchOrCommit)) { @@ -66,7 +66,7 @@ public override void Prefetch(string branchOrCommit, bool isBranch) // Configure pipeline // Checkout uses DiffHelper when running checkout.Start(), which we use instead of LsTreeHelper // Checkout diff output => FindMissingBlobs => BatchDownload => IndexPack => Checkout available blobs - CheckoutJob checkout = new CheckoutJob(this.checkoutThreadCount, this.FolderList, commitToFetch, this.Tracer, this.Enlistment); + CheckoutJob checkout = new CheckoutJob(this.checkoutThreadCount, this.FolderList, commitToFetch, this.Tracer, this.Enlistment, force: force); FindMissingBlobsJob blobFinder = new FindMissingBlobsJob(this.SearchThreadCount, checkout.RequiredBlobs, checkout.AvailableBlobShas, this.Tracer, this.Enlistment); BatchObjectDownloadJob downloader = new BatchObjectDownloadJob(this.DownloadThreadCount, this.ChunkSize, blobFinder.MissingBlobs, checkout.AvailableBlobShas, this.Tracer, this.Enlistment, this.ObjectRequestor, this.GitObjects); IndexPackJob packIndexer = new IndexPackJob(this.IndexThreadCount, downloader.AvailablePacks, checkout.AvailableBlobShas, this.Tracer, this.GitObjects); diff --git a/GVFS/FastFetch/FastFetchVerb.cs b/GVFS/FastFetch/FastFetchVerb.cs index 1c62a09b0..16f45dcab 100644 --- a/GVFS/FastFetch/FastFetchVerb.cs +++ b/GVFS/FastFetch/FastFetchVerb.cs @@ -135,6 +135,13 @@ public class FastFetchVerb HelpText = "The GUID of the caller - used for telemetry purposes.")] public string ParentActivityId { get; set; } + [Option( + "force", + Required = false, + Default = false, + HelpText = "Force FastFetch to download content as if the current repo was at commit id 0 (Had just been initialized).")] + public bool Force { get; set; } + public void Execute() { Environment.ExitCode = this.ExecuteWithExitCode(); @@ -237,7 +244,7 @@ private int ExecuteWithExitCode() try { bool isBranch = this.Commit == null; - prefetcher.Prefetch(commitish, isBranch); + prefetcher.Prefetch(commitish, isBranch, force: this.Force); return !prefetcher.HasFailures; } catch (BlobPrefetcher.FetchException e) diff --git a/GVFS/GVFS.Common/Prefetch/BlobPrefetcher.cs b/GVFS/GVFS.Common/Prefetch/BlobPrefetcher.cs index a3972b412..9b5455b97 100644 --- a/GVFS/GVFS.Common/Prefetch/BlobPrefetcher.cs +++ b/GVFS/GVFS.Common/Prefetch/BlobPrefetcher.cs @@ -162,13 +162,13 @@ public static void AppendToNewlineSeparatedFile(PhysicalFileSystem fileSystem, s } /// A specific branch to filter for, or null for all branches returned from info/refs - public virtual void Prefetch(string branchOrCommit, bool isBranch) + public virtual void Prefetch(string branchOrCommit, bool isBranch, bool force = false) { int matchedBlobCount; int downloadedBlobCount; int readFileCount; - this.PrefetchWithStats(branchOrCommit, isBranch, false, out matchedBlobCount, out downloadedBlobCount, out readFileCount); + this.PrefetchWithStats(branchOrCommit, isBranch, false, out matchedBlobCount, out downloadedBlobCount, out readFileCount, force: force); } public void PrefetchWithStats( @@ -177,7 +177,8 @@ public void PrefetchWithStats( bool readFilesAfterDownload, out int matchedBlobCount, out int downloadedBlobCount, - out int readFileCount) + out int readFileCount, + bool force = false) { matchedBlobCount = 0; downloadedBlobCount = 0; @@ -217,9 +218,9 @@ public void PrefetchWithStats( string previousCommit = null; - // Use the shallow file to find a recent commit to diff against to try and reduce the number of SHAs to check - DiffHelper blobEnumerator = new DiffHelper(this.Tracer, this.Enlistment, this.FileList, this.FolderList); - if (File.Exists(shallowFile)) + // Use the shallow file to find a recent commit to diff against to try and reduce the number of SHAs to check. + // Unless force flag has been given, in which case treat as if it's a fresh repo. + if (!force && File.Exists(shallowFile)) { previousCommit = File.ReadAllLines(shallowFile).Where(line => !string.IsNullOrWhiteSpace(line)).LastOrDefault(); if (string.IsNullOrWhiteSpace(previousCommit)) @@ -230,6 +231,8 @@ public void PrefetchWithStats( } } + DiffHelper blobEnumerator = new DiffHelper(this.Tracer, this.Enlistment, this.FileList, this.FolderList); + ThreadStart performDiff = () => { blobEnumerator.PerformDiff(previousCommit, commitToFetch); diff --git a/GVFS/GVFS.Common/Prefetch/Jobs/CheckoutJob.cs b/GVFS/GVFS.Common/Prefetch/Jobs/CheckoutJob.cs index bce0921cb..0a464a09c 100644 --- a/GVFS/GVFS.Common/Prefetch/Jobs/CheckoutJob.cs +++ b/GVFS/GVFS.Common/Prefetch/Jobs/CheckoutJob.cs @@ -19,6 +19,7 @@ public class CheckoutJob : Job private ITracer tracer; private Enlistment enlistment; private string targetCommitSha; + private bool force; private DiffHelper diff; @@ -31,13 +32,14 @@ public class CheckoutJob : Job // Checkout requires synchronization between the delete/directory/add stages, so control the parallelization private int maxParallel; - public CheckoutJob(int maxParallel, IEnumerable folderList, string targetCommitSha, ITracer tracer, Enlistment enlistment) + public CheckoutJob(int maxParallel, IEnumerable folderList, string targetCommitSha, ITracer tracer, Enlistment enlistment, bool force = false) : base(1) { this.tracer = tracer.StartActivity(AreaPath, EventLevel.Informational, Keywords.Telemetry, metadata: null); this.enlistment = enlistment; this.diff = new DiffHelper(tracer, enlistment, new string[0], folderList); this.targetCommitSha = targetCommitSha; + this.force = force; this.AvailableBlobShas = new BlockingCollection(); // Keep track of how parallel we're expected to be later during DoWork @@ -62,7 +64,17 @@ public bool UpdatedWholeTree protected override void DoBeforeWork() { - this.diff.PerformDiff(this.targetCommitSha); + if (this.force) + { + // When using force set the sourceCommitSha to null to act like an uninitialized repo + this.diff.PerformDiff(null, this.targetCommitSha); + } + else + { + // This variant of diff uses the HEAD as the SourceCommitSha. + this.diff.PerformDiff(this.targetCommitSha); + } + this.HasFailures = this.diff.HasFailures; } diff --git a/GVFS/GVFS.FunctionalTests/Tests/FastFetchTests.cs b/GVFS/GVFS.FunctionalTests/Tests/FastFetchTests.cs index 979c72649..521193ee7 100644 --- a/GVFS/GVFS.FunctionalTests/Tests/FastFetchTests.cs +++ b/GVFS/GVFS.FunctionalTests/Tests/FastFetchTests.cs @@ -100,6 +100,62 @@ public void CanFetchAndCheckoutASingleFolderIntoEmptyGitRepo() this.AllFetchedFilePathsShouldPassCheck(path => path.StartsWith("GVFS", StringComparison.OrdinalIgnoreCase)); } + [TestCase] + public void CanFetchAndCheckoutMultipleTimesUsingForceFlag() + { + this.RunFastFetch($"--checkout --folders \"/GVFS\" -b {Settings.Default.Commitish}"); + + this.CurrentBranchShouldEqual(Settings.Default.Commitish); + + this.fastFetchRepoRoot.ShouldBeADirectory(FileSystemRunner.DefaultRunner); + List dirs = Directory.EnumerateFileSystemEntries(this.fastFetchRepoRoot).ToList(); + dirs.SequenceEqual(new[] + { + Path.Combine(this.fastFetchRepoRoot, ".git"), + Path.Combine(this.fastFetchRepoRoot, "GVFS"), + Path.Combine(this.fastFetchRepoRoot, "GVFS.sln") + }); + + Directory.EnumerateFileSystemEntries(Path.Combine(this.fastFetchRepoRoot, "GVFS"), "*", SearchOption.AllDirectories) + .Count() + .ShouldEqual(345); + this.AllFetchedFilePathsShouldPassCheck(path => path.StartsWith("GVFS", StringComparison.OrdinalIgnoreCase)); + + // Run a second time in the same repo on the same branch with more folders. + this.RunFastFetch($"--checkout --folders \"/GVFS;/Scripts\" -b {Settings.Default.Commitish} --force"); + dirs = Directory.EnumerateFileSystemEntries(this.fastFetchRepoRoot).ToList(); + dirs.SequenceEqual(new[] + { + Path.Combine(this.fastFetchRepoRoot, ".git"), + Path.Combine(this.fastFetchRepoRoot, "GVFS"), + Path.Combine(this.fastFetchRepoRoot, "Scripts"), + Path.Combine(this.fastFetchRepoRoot, "GVFS.sln") + }); + Directory.EnumerateFileSystemEntries(Path.Combine(this.fastFetchRepoRoot, "Scripts"), "*", SearchOption.AllDirectories) + .Count() + .ShouldEqual(5); + } + + [TestCase] + public void CanFetchWithoutCheckoutMultipleTimesUsingForceFlagAndThenCheckout() + { + this.RunFastFetch($"--folders \"/GVFS\" -b {Settings.Default.Commitish}"); + + this.GetRefTreeSha("remotes/origin/" + Settings.Default.Commitish).ShouldNotBeNull(); + + // Run a second time in the same repo on the same branch with more folders. + this.RunFastFetch($"--folders \"/GVFS;/Scripts\" -b {Settings.Default.Commitish} --force"); + + this.RunFastFetch($"--checkout -b {Settings.Default.Commitish}"); + Directory.EnumerateFileSystemEntries(Path.Combine(this.fastFetchRepoRoot, "GVFS"), "*", SearchOption.AllDirectories) + .Count() + .ShouldEqual(345); + + Directory.EnumerateFileSystemEntries(Path.Combine(this.fastFetchRepoRoot, "Scripts"), "*", SearchOption.AllDirectories) + .Count() + .ShouldEqual(5); + } + [TestCase] public void FastFetchFolderWithOnlyOneFile() { From 2a6b4dac88af4a1ca4def288c4f5bd013c82daf8 Mon Sep 17 00:00:00 2001 From: Kyle Rader Date: Mon, 13 Aug 2018 14:48:09 -0700 Subject: [PATCH 029/272] Simplify help text for force flag --- GVFS/FastFetch/FastFetchVerb.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/GVFS/FastFetch/FastFetchVerb.cs b/GVFS/FastFetch/FastFetchVerb.cs index 16f45dcab..ce3ce312d 100644 --- a/GVFS/FastFetch/FastFetchVerb.cs +++ b/GVFS/FastFetch/FastFetchVerb.cs @@ -139,7 +139,7 @@ public class FastFetchVerb "force", Required = false, Default = false, - HelpText = "Force FastFetch to download content as if the current repo was at commit id 0 (Had just been initialized).")] + HelpText = "Force FastFetch to download content as if the current repository was just initialized.")] public bool Force { get; set; } public void Execute() From 22e41b56931dc8d31cda633286ee41e186d7956b Mon Sep 17 00:00:00 2001 From: Kyle Rader Date: Tue, 14 Aug 2018 10:13:01 -0700 Subject: [PATCH 030/272] Rename force to force-checkout and pull into FastFetch CheckoutJob --- .../Jobs => FastFetch}/CheckoutJob.cs | 20 +++++++------- GVFS/FastFetch/CheckoutPrefetcher.cs | 11 +++++--- GVFS/FastFetch/FastFetch.csproj | 1 + GVFS/FastFetch/FastFetchVerb.cs | 27 ++++++++++++------- GVFS/GVFS.Common/Prefetch/BlobPrefetcher.cs | 9 +++---- .../Tests/FastFetchTests.cs | 22 +++++---------- 6 files changed, 48 insertions(+), 42 deletions(-) rename GVFS/{GVFS.Common/Prefetch/Jobs => FastFetch}/CheckoutJob.cs (93%) diff --git a/GVFS/GVFS.Common/Prefetch/Jobs/CheckoutJob.cs b/GVFS/FastFetch/CheckoutJob.cs similarity index 93% rename from GVFS/GVFS.Common/Prefetch/Jobs/CheckoutJob.cs rename to GVFS/FastFetch/CheckoutJob.cs index 0a464a09c..a86079f22 100644 --- a/GVFS/GVFS.Common/Prefetch/Jobs/CheckoutJob.cs +++ b/GVFS/FastFetch/CheckoutJob.cs @@ -1,6 +1,8 @@ -using GVFS.Common.FileSystem; +using GVFS.Common; +using GVFS.Common.FileSystem; using GVFS.Common.Git; using GVFS.Common.Prefetch.Git; +using GVFS.Common.Prefetch.Jobs; using GVFS.Common.Tracing; using System; using System.Collections.Concurrent; @@ -9,7 +11,7 @@ using System.Threading; using System.Threading.Tasks; -namespace GVFS.Common.Prefetch.Jobs +namespace FastFetch { public class CheckoutJob : Job { @@ -19,7 +21,7 @@ public class CheckoutJob : Job private ITracer tracer; private Enlistment enlistment; private string targetCommitSha; - private bool force; + private bool forceCheckout; private DiffHelper diff; @@ -32,14 +34,14 @@ public class CheckoutJob : Job // Checkout requires synchronization between the delete/directory/add stages, so control the parallelization private int maxParallel; - public CheckoutJob(int maxParallel, IEnumerable folderList, string targetCommitSha, ITracer tracer, Enlistment enlistment, bool force = false) + public CheckoutJob(int maxParallel, IEnumerable folderList, string targetCommitSha, ITracer tracer, Enlistment enlistment, bool forceCheckout) : base(1) { this.tracer = tracer.StartActivity(AreaPath, EventLevel.Informational, Keywords.Telemetry, metadata: null); this.enlistment = enlistment; this.diff = new DiffHelper(tracer, enlistment, new string[0], folderList); this.targetCommitSha = targetCommitSha; - this.force = force; + this.forceCheckout = forceCheckout; this.AvailableBlobShas = new BlockingCollection(); // Keep track of how parallel we're expected to be later during DoWork @@ -64,14 +66,14 @@ public bool UpdatedWholeTree protected override void DoBeforeWork() { - if (this.force) + if (this.forceCheckout) { - // When using force set the sourceCommitSha to null to act like an uninitialized repo - this.diff.PerformDiff(null, this.targetCommitSha); + // Force search the entire tree by treating the repo as if it were brand new. + this.diff.PerformDiff(sourceTreeSha: null, targetTreeSha: this.targetCommitSha); } else { - // This variant of diff uses the HEAD as the SourceCommitSha. + // Let the diff find the sourceTreeSha on its own. this.diff.PerformDiff(this.targetCommitSha); } diff --git a/GVFS/FastFetch/CheckoutPrefetcher.cs b/GVFS/FastFetch/CheckoutPrefetcher.cs index b344a5847..863acca15 100644 --- a/GVFS/FastFetch/CheckoutPrefetcher.cs +++ b/GVFS/FastFetch/CheckoutPrefetcher.cs @@ -14,8 +14,9 @@ namespace FastFetch { public class CheckoutPrefetcher : BlobPrefetcher { - private readonly bool allowIndexMetadataUpdateFromWorkingTree; private readonly int checkoutThreadCount; + private readonly bool allowIndexMetadataUpdateFromWorkingTree; + private readonly bool forceCheckout; public CheckoutPrefetcher( ITracer tracer, @@ -26,14 +27,16 @@ public CheckoutPrefetcher( int downloadThreadCount, int indexThreadCount, int checkoutThreadCount, - bool allowIndexMetadataUpdateFromWorkingTree) : base(tracer, enlistment, objectRequestor, chunkSize, searchThreadCount, downloadThreadCount, indexThreadCount) + bool allowIndexMetadataUpdateFromWorkingTree, + bool forceCheckout) : base(tracer, enlistment, objectRequestor, chunkSize, searchThreadCount, downloadThreadCount, indexThreadCount) { this.checkoutThreadCount = checkoutThreadCount; this.allowIndexMetadataUpdateFromWorkingTree = allowIndexMetadataUpdateFromWorkingTree; + this.forceCheckout = forceCheckout; } /// A specific branch to filter for, or null for all branches returned from info/refs - public override void Prefetch(string branchOrCommit, bool isBranch, bool force = false) + public override void Prefetch(string branchOrCommit, bool isBranch) { if (string.IsNullOrWhiteSpace(branchOrCommit)) { @@ -66,7 +69,7 @@ public override void Prefetch(string branchOrCommit, bool isBranch, bool force = // Configure pipeline // Checkout uses DiffHelper when running checkout.Start(), which we use instead of LsTreeHelper // Checkout diff output => FindMissingBlobs => BatchDownload => IndexPack => Checkout available blobs - CheckoutJob checkout = new CheckoutJob(this.checkoutThreadCount, this.FolderList, commitToFetch, this.Tracer, this.Enlistment, force: force); + CheckoutJob checkout = new CheckoutJob(this.checkoutThreadCount, this.FolderList, commitToFetch, this.Tracer, this.Enlistment, this.forceCheckout); FindMissingBlobsJob blobFinder = new FindMissingBlobsJob(this.SearchThreadCount, checkout.RequiredBlobs, checkout.AvailableBlobShas, this.Tracer, this.Enlistment); BatchObjectDownloadJob downloader = new BatchObjectDownloadJob(this.DownloadThreadCount, this.ChunkSize, blobFinder.MissingBlobs, checkout.AvailableBlobShas, this.Tracer, this.Enlistment, this.ObjectRequestor, this.GitObjects); IndexPackJob packIndexer = new IndexPackJob(this.IndexThreadCount, downloader.AvailablePacks, checkout.AvailableBlobShas, this.Tracer, this.GitObjects); diff --git a/GVFS/FastFetch/FastFetch.csproj b/GVFS/FastFetch/FastFetch.csproj index 5658cdade..f4544cd62 100644 --- a/GVFS/FastFetch/FastFetch.csproj +++ b/GVFS/FastFetch/FastFetch.csproj @@ -56,6 +56,7 @@ CommonAssemblyVersion.cs + diff --git a/GVFS/FastFetch/FastFetchVerb.cs b/GVFS/FastFetch/FastFetchVerb.cs index ce3ce312d..bf961df91 100644 --- a/GVFS/FastFetch/FastFetchVerb.cs +++ b/GVFS/FastFetch/FastFetchVerb.cs @@ -54,6 +54,15 @@ public class FastFetchVerb HelpText = "Checkout the target commit into the working directory after fetching")] public bool Checkout { get; set; } + [Option( + "force-checkout", + Required = false, + Default = false, + HelpText = "Force FastFetch to fetch and checkout content as if the current repo had just been initialized." + + "This allows you to include more folders from the repo that were not originally checked out." + + "Can only be used with --checkout")] + public bool ForceCheckout { get; set; } + [Option( "search-thread-count", Required = false, @@ -135,13 +144,6 @@ public class FastFetchVerb HelpText = "The GUID of the caller - used for telemetry purposes.")] public string ParentActivityId { get; set; } - [Option( - "force", - Required = false, - Default = false, - HelpText = "Force FastFetch to download content as if the current repository was just initialized.")] - public bool Force { get; set; } - public void Execute() { Environment.ExitCode = this.ExecuteWithExitCode(); @@ -164,6 +166,12 @@ private int ExecuteWithExitCode() Console.WriteLine("Cannot specify both a commit sha and a branch name."); return ExitFailure; } + + if (this.ForceCheckout && !this.Checkout) + { + Console.WriteLine("Cannot use --force-checkout option without --checkout option."); + return ExitFailure; + } this.SearchThreadCount = this.SearchThreadCount > 0 ? this.SearchThreadCount : Environment.ProcessorCount; this.DownloadThreadCount = this.DownloadThreadCount > 0 ? this.DownloadThreadCount : Math.Min(Environment.ProcessorCount, MaxDefaultDownloadThreads); @@ -244,7 +252,7 @@ private int ExecuteWithExitCode() try { bool isBranch = this.Commit == null; - prefetcher.Prefetch(commitish, isBranch, force: this.Force); + prefetcher.Prefetch(commitish, isBranch); return !prefetcher.HasFailures; } catch (BlobPrefetcher.FetchException e) @@ -325,7 +333,8 @@ private BlobPrefetcher GetFolderPrefetcher(ITracer tracer, Enlistment enlistment this.DownloadThreadCount, this.IndexThreadCount, this.CheckoutThreadCount, - this.AllowIndexMetadataUpdateFromWorkingTree); + this.AllowIndexMetadataUpdateFromWorkingTree, + this.ForceCheckout); } else { diff --git a/GVFS/GVFS.Common/Prefetch/BlobPrefetcher.cs b/GVFS/GVFS.Common/Prefetch/BlobPrefetcher.cs index 9b5455b97..7c633a1de 100644 --- a/GVFS/GVFS.Common/Prefetch/BlobPrefetcher.cs +++ b/GVFS/GVFS.Common/Prefetch/BlobPrefetcher.cs @@ -162,13 +162,13 @@ public static void AppendToNewlineSeparatedFile(PhysicalFileSystem fileSystem, s } /// A specific branch to filter for, or null for all branches returned from info/refs - public virtual void Prefetch(string branchOrCommit, bool isBranch, bool force = false) + public virtual void Prefetch(string branchOrCommit, bool isBranch) { int matchedBlobCount; int downloadedBlobCount; int readFileCount; - this.PrefetchWithStats(branchOrCommit, isBranch, false, out matchedBlobCount, out downloadedBlobCount, out readFileCount, force: force); + this.PrefetchWithStats(branchOrCommit, isBranch, false, out matchedBlobCount, out downloadedBlobCount, out readFileCount); } public void PrefetchWithStats( @@ -177,8 +177,7 @@ public void PrefetchWithStats( bool readFilesAfterDownload, out int matchedBlobCount, out int downloadedBlobCount, - out int readFileCount, - bool force = false) + out int readFileCount) { matchedBlobCount = 0; downloadedBlobCount = 0; @@ -220,7 +219,7 @@ public void PrefetchWithStats( // Use the shallow file to find a recent commit to diff against to try and reduce the number of SHAs to check. // Unless force flag has been given, in which case treat as if it's a fresh repo. - if (!force && File.Exists(shallowFile)) + if (File.Exists(shallowFile)) { previousCommit = File.ReadAllLines(shallowFile).Where(line => !string.IsNullOrWhiteSpace(line)).LastOrDefault(); if (string.IsNullOrWhiteSpace(previousCommit)) diff --git a/GVFS/GVFS.FunctionalTests/Tests/FastFetchTests.cs b/GVFS/GVFS.FunctionalTests/Tests/FastFetchTests.cs index 521193ee7..69e1bdc5c 100644 --- a/GVFS/GVFS.FunctionalTests/Tests/FastFetchTests.cs +++ b/GVFS/GVFS.FunctionalTests/Tests/FastFetchTests.cs @@ -101,7 +101,7 @@ public void CanFetchAndCheckoutASingleFolderIntoEmptyGitRepo() } [TestCase] - public void CanFetchAndCheckoutMultipleTimesUsingForceFlag() + public void CanFetchAndCheckoutMultipleTimesUsingForceCheckoutFlag() { this.RunFastFetch($"--checkout --folders \"/GVFS\" -b {Settings.Default.Commitish}"); @@ -122,7 +122,7 @@ public void CanFetchAndCheckoutMultipleTimesUsingForceFlag() this.AllFetchedFilePathsShouldPassCheck(path => path.StartsWith("GVFS", StringComparison.OrdinalIgnoreCase)); // Run a second time in the same repo on the same branch with more folders. - this.RunFastFetch($"--checkout --folders \"/GVFS;/Scripts\" -b {Settings.Default.Commitish} --force"); + this.RunFastFetch($"--checkout --folders \"/GVFS;/Scripts\" -b {Settings.Default.Commitish} --force-checkout"); dirs = Directory.EnumerateFileSystemEntries(this.fastFetchRepoRoot).ToList(); dirs.SequenceEqual(new[] { @@ -137,23 +137,15 @@ public void CanFetchAndCheckoutMultipleTimesUsingForceFlag() } [TestCase] - public void CanFetchWithoutCheckoutMultipleTimesUsingForceFlagAndThenCheckout() + public void ForceCheckoutRequiresCheckout() { - this.RunFastFetch($"--folders \"/GVFS\" -b {Settings.Default.Commitish}"); - - this.GetRefTreeSha("remotes/origin/" + Settings.Default.Commitish).ShouldNotBeNull(); + this.RunFastFetch($"--checkout --folders \"/Scripts\" -b {Settings.Default.Commitish}"); // Run a second time in the same repo on the same branch with more folders. - this.RunFastFetch($"--folders \"/GVFS;/Scripts\" -b {Settings.Default.Commitish} --force"); - - this.RunFastFetch($"--checkout -b {Settings.Default.Commitish}"); - Directory.EnumerateFileSystemEntries(Path.Combine(this.fastFetchRepoRoot, "GVFS"), "*", SearchOption.AllDirectories) - .Count() - .ShouldEqual(345); + string result = this.RunFastFetch($"--force-checkout --folders \"/GVFS;/Scripts\" -b {Settings.Default.Commitish}"); - Directory.EnumerateFileSystemEntries(Path.Combine(this.fastFetchRepoRoot, "Scripts"), "*", SearchOption.AllDirectories) - .Count() - .ShouldEqual(5); + string[] expectedResults = new string[] { "Cannot use --force-checkout option without --checkout option." }; + result.ShouldContain(expectedResults); } [TestCase] From f41452da7ef70ca30f0a58c4d7b22a1fec9c7d83 Mon Sep 17 00:00:00 2001 From: Kyle Rader Date: Tue, 14 Aug 2018 13:21:30 -0700 Subject: [PATCH 031/272] Update comments and help text --- GVFS/FastFetch/FastFetchVerb.cs | 4 ++-- GVFS/GVFS.Common/Prefetch/BlobPrefetcher.cs | 1 - 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/GVFS/FastFetch/FastFetchVerb.cs b/GVFS/FastFetch/FastFetchVerb.cs index bf961df91..82cf59fcc 100644 --- a/GVFS/FastFetch/FastFetchVerb.cs +++ b/GVFS/FastFetch/FastFetchVerb.cs @@ -58,9 +58,9 @@ public class FastFetchVerb "force-checkout", Required = false, Default = false, - HelpText = "Force FastFetch to fetch and checkout content as if the current repo had just been initialized." + + HelpText = "Force FastFetch to checkout content as if the current repo had just been initialized." + "This allows you to include more folders from the repo that were not originally checked out." + - "Can only be used with --checkout")] + "Can only be used with the --checkout option.")] public bool ForceCheckout { get; set; } [Option( diff --git a/GVFS/GVFS.Common/Prefetch/BlobPrefetcher.cs b/GVFS/GVFS.Common/Prefetch/BlobPrefetcher.cs index 7c633a1de..66f1ad0f4 100644 --- a/GVFS/GVFS.Common/Prefetch/BlobPrefetcher.cs +++ b/GVFS/GVFS.Common/Prefetch/BlobPrefetcher.cs @@ -218,7 +218,6 @@ public void PrefetchWithStats( string previousCommit = null; // Use the shallow file to find a recent commit to diff against to try and reduce the number of SHAs to check. - // Unless force flag has been given, in which case treat as if it's a fresh repo. if (File.Exists(shallowFile)) { previousCommit = File.ReadAllLines(shallowFile).Where(line => !string.IsNullOrWhiteSpace(line)).LastOrDefault(); From 97975e4396893a9b44544bfe8a098196008e73b8 Mon Sep 17 00:00:00 2001 From: Jameson Miller Date: Wed, 15 Aug 2018 12:42:40 -0400 Subject: [PATCH 032/272] GitStatusCache: fix race condition in test setup This is to fix an issue that happened (intermittently) in the functional tests. There is a race condition where the test is attempting to delete the status cache file, but the status cache might not have been generated for the test repository. To fix this, the test setup will wait for the initial status cache to be generated, so it can proceed from a known state. --- .../Tests/GitCommands/StatusTests.cs | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/GVFS/GVFS.FunctionalTests/Tests/GitCommands/StatusTests.cs b/GVFS/GVFS.FunctionalTests/Tests/GitCommands/StatusTests.cs index a66d691c9..9865431c3 100644 --- a/GVFS/GVFS.FunctionalTests/Tests/GitCommands/StatusTests.cs +++ b/GVFS/GVFS.FunctionalTests/Tests/GitCommands/StatusTests.cs @@ -98,6 +98,8 @@ public void ModifyingHeadRefInvalidatesCache() private void RepositoryIgnoreTestSetup() { + this.WaitForUpToDateStatusCache(); + string statusCachePath = Path.Combine(this.Enlistment.DotGVFSRoot, "GitStatusCache", "GitStatusCache.dat"); File.Delete(statusCachePath); @@ -110,6 +112,18 @@ private void RepositoryIgnoreTestSetup() this.ValidateGitCommand("status"); } + /// + /// Wait for an up-to-date status cache file to exist on disk. + /// + private void WaitForUpToDateStatusCache() + { + // Run "git status" for the side effect that it will delete any stale status cache file. + this.ValidateGitCommand("status"); + + // Wait for a new status cache to be generated. + this.WaitForStatusCacheToBeGenerated(waitForNewFile: false); + } + private void WaitForStatusCacheToBeGenerated(bool waitForNewFile = true) { string statusCachePath = Path.Combine(this.Enlistment.DotGVFSRoot, "GitStatusCache", "GitStatusCache.dat"); From 25fc24a39eba3fff027141c30725b74964b2645f Mon Sep 17 00:00:00 2001 From: William Baker Date: Fri, 17 Aug 2018 08:54:34 -0700 Subject: [PATCH 033/272] Reduce scope of changes to be specific to PreDelete, update MirrorProvider --- .../FileSystemRunners/BashRunner.cs | 15 +--- .../BasicFileSystemTests.cs | 1 + .../MacFileSystemVirtualizer.cs | 13 ---- .../MacFileSystemVirtualizer.cs | 21 ++++-- .../WindowsFileSystemVirtualizer.cs | 8 ++ .../PrjFSKext/PrjFSKext/KauthHandler.cpp | 74 ++----------------- .../PrjFSKext/PrjFSKext/Message_Kernel.cpp | 10 +++ ProjFS.Mac/PrjFSLib/PrjFSLib.cpp | 5 +- 8 files changed, 44 insertions(+), 103 deletions(-) diff --git a/GVFS/GVFS.FunctionalTests/FileSystemRunners/BashRunner.cs b/GVFS/GVFS.FunctionalTests/FileSystemRunners/BashRunner.cs index 00544dac3..6b2310a70 100644 --- a/GVFS/GVFS.FunctionalTests/FileSystemRunners/BashRunner.cs +++ b/GVFS/GVFS.FunctionalTests/FileSystemRunners/BashRunner.cs @@ -4,7 +4,6 @@ using System; using System.Diagnostics; using System.IO; -using System.Runtime.InteropServices; using System.Threading; namespace GVFS.FunctionalTests.FileSystemRunners @@ -34,11 +33,6 @@ public class BashRunner : ShellRunner "Permission denied" }; - private static string[] resourceUnavailableMessage = new string[] - { - "Resource temporarily unavailable", - }; - private readonly string pathToBash; public BashRunner() @@ -239,14 +233,7 @@ public override void DeleteFile_FileShouldNotBeFound(string path) public override void DeleteFile_AccessShouldBeDenied(string path) { - if (RuntimeInformation.IsOSPlatform(OSPlatform.Windows)) - { - this.DeleteFile(path).ShouldContain(permissionDeniedMessage); - } - else - { - this.DeleteFile(path).ShouldContain(resourceUnavailableMessage); - } + this.DeleteFile(path).ShouldContain(permissionDeniedMessage); } public override void ReadAllText_FileShouldNotBeFound(string path) diff --git a/GVFS/GVFS.FunctionalTests/Tests/EnlistmentPerFixture/BasicFileSystemTests.cs b/GVFS/GVFS.FunctionalTests/Tests/EnlistmentPerFixture/BasicFileSystemTests.cs index 24c32ef4f..ac87acb32 100644 --- a/GVFS/GVFS.FunctionalTests/Tests/EnlistmentPerFixture/BasicFileSystemTests.cs +++ b/GVFS/GVFS.FunctionalTests/Tests/EnlistmentPerFixture/BasicFileSystemTests.cs @@ -784,6 +784,7 @@ public void MoveDotGitFullFolderTreeToDotGitFullFolder(FileSystemRunner fileSyst } [TestCaseSource(typeof(FileSystemRunner), FileSystemRunner.TestRunners)] + [Category(Categories.Mac.M4)] public void DeleteIndexFileFails(FileSystemRunner fileSystem) { string indexFilePath = this.Enlistment.GetVirtualPathTo(Path.Combine(".git", "index")); diff --git a/GVFS/GVFS.Platform.Mac/MacFileSystemVirtualizer.cs b/GVFS/GVFS.Platform.Mac/MacFileSystemVirtualizer.cs index 567226070..0ffc5f495 100644 --- a/GVFS/GVFS.Platform.Mac/MacFileSystemVirtualizer.cs +++ b/GVFS/GVFS.Platform.Mac/MacFileSystemVirtualizer.cs @@ -292,19 +292,6 @@ private Result OnPreDelete(string relativePath, bool isDirectory) return Result.EIOError; } - if (relativePath.Equals(GVFSConstants.DotGit.Index, StringComparison.OrdinalIgnoreCase)) - { - string lockedGitCommand = this.Context.Repository.GVFSLock.GetLockedGitCommand(); - if (string.IsNullOrEmpty(lockedGitCommand)) - { - EventMetadata metadata = this.CreateEventMetadata(relativePath); - metadata.Add(TracingConstants.MessageKey.WarningMessage, "Blocked index delete outside the lock"); - this.Context.Tracer.RelatedEvent(EventLevel.Warning, $"{nameof(this.OnPreDelete)}_BlockedIndexDelete", metadata); - - return Result.EAccessDenied; - } - } - bool pathInsideDotGit = Virtualization.FileSystemCallbacks.IsPathInsideDotGit(relativePath); if (pathInsideDotGit) { diff --git a/MirrorProvider/MirrorProvider.Mac/MacFileSystemVirtualizer.cs b/MirrorProvider/MirrorProvider.Mac/MacFileSystemVirtualizer.cs index 2e5b1a54a..3e2567548 100644 --- a/MirrorProvider/MirrorProvider.Mac/MacFileSystemVirtualizer.cs +++ b/MirrorProvider/MirrorProvider.Mac/MacFileSystemVirtualizer.cs @@ -20,7 +20,8 @@ public override bool TryStartVirtualizationInstance(Enlistment enlistment, out s { this.virtualizationInstance.OnEnumerateDirectory = this.OnEnumerateDirectory; this.virtualizationInstance.OnGetFileStream = this.OnGetFileStream; - this.virtualizationInstance.OnFileModified = this.OnFileModified; + this.virtualizationInstance.OnFileModified = this.OnFileModified; + this.virtualizationInstance.OnPreDelete = this.OnPreDelete; Result result = this.virtualizationInstance.StartVirtualizationInstance( enlistment.SrcRoot, @@ -61,7 +62,7 @@ private Result OnEnumerateDirectory( if (result != Result.Success) { - Console.WriteLine("WritePlaceholderDirectory failed: " + result); + Console.WriteLine($"WritePlaceholderDirectory failed: {result}"); return result; } } @@ -81,7 +82,7 @@ private Result OnEnumerateDirectory( fileMode: fileMode); if (result != Result.Success) { - Console.WriteLine("WritePlaceholderFile failed: " + result); + Console.WriteLine($"WritePlaceholderFile failed: {result}"); return result; } } @@ -89,7 +90,7 @@ private Result OnEnumerateDirectory( } catch (IOException e) { - Console.WriteLine("IOException in OnEnumerateDirectory: " + e.Message); + Console.WriteLine($"IOException in OnEnumerateDirectory: {e.Message}"); return Result.EIOError; } @@ -126,7 +127,7 @@ private Result OnGetFileStream( (uint)bytesToCopy); if (result != Result.Success) { - Console.WriteLine("WriteFileContents failed: " + result); + Console.WriteLine($"WriteFileContents failed: {result}"); return false; } @@ -140,7 +141,7 @@ private Result OnGetFileStream( } catch (IOException e) { - Console.WriteLine("IOException in OnGetFileStream: " + e.Message); + Console.WriteLine($"IOException in OnGetFileStream: {e.Message}"); return Result.EIOError; } @@ -149,7 +150,13 @@ private Result OnGetFileStream( private void OnFileModified(string relativePath) { - Console.WriteLine("OnFileModified: " + relativePath); + Console.WriteLine($"OnFileModified: {relativePath}"); + } + + private Result OnPreDelete(string relativePath, bool isDirectory) + { + Console.WriteLine($"OnPreDelete (isDirectory: {isDirectory}): {relativePath}"); + return Result.Success; } private static byte[] ToVersionIdByteArray(byte version) diff --git a/MirrorProvider/MirrorProvider.Windows/WindowsFileSystemVirtualizer.cs b/MirrorProvider/MirrorProvider.Windows/WindowsFileSystemVirtualizer.cs index d619e2ed9..dd49ca5e5 100644 --- a/MirrorProvider/MirrorProvider.Windows/WindowsFileSystemVirtualizer.cs +++ b/MirrorProvider/MirrorProvider.Windows/WindowsFileSystemVirtualizer.cs @@ -34,12 +34,14 @@ public override bool TryStartVirtualizationInstance(Enlistment enlistment, out s this.virtualizationInstance.OnGetPlaceholderInformation = this.GetPlaceholderInformation; this.virtualizationInstance.OnGetFileStream = this.GetFileStream; + this.virtualizationInstance.OnNotifyPreDelete = this.OnPreDelete; this.virtualizationInstance.OnNotifyFileHandleClosedFileModifiedOrDeleted = this.OnFileModifiedOrDeleted; uint threadCount = (uint)Environment.ProcessorCount * 2; NotificationMapping[] notificationMappings = new NotificationMapping[] { + new NotificationMapping(NotificationType.PreDelete, string.Empty), new NotificationMapping(NotificationType.FileHandleClosedFileModified, string.Empty), }; @@ -283,6 +285,12 @@ private HResult GetFileStream( return HResult.Ok; } + private HResult OnPreDelete(string relativePath, bool isDirectory) + { + Console.WriteLine($"OnPreDelete (isDirectory: {isDirectory}): {relativePath}"); + return HResult.Ok; + } + private void OnFileModifiedOrDeleted(string relativePath, bool isDirectory, bool isFileModified, bool isFileDeleted) { // To keep WindowsFileSystemVirtualizer in sync with MacFileSystemVirtualizer we're only registering for diff --git a/ProjFS.Mac/PrjFSKext/PrjFSKext/KauthHandler.cpp b/ProjFS.Mac/PrjFSKext/PrjFSKext/KauthHandler.cpp index b277e6fbe..5226cb162 100644 --- a/ProjFS.Mac/PrjFSKext/PrjFSKext/KauthHandler.cpp +++ b/ProjFS.Mac/PrjFSKext/PrjFSKext/KauthHandler.cpp @@ -35,7 +35,6 @@ static int HandleFileOpOperation( static int GetPid(vfs_context_t context); static uint32_t ReadVNodeFileFlags(vnode_t vn, vfs_context_t context); -static bool HasParentInVirtualizationRoot(vnode_t vnode, vfs_context_t context); static bool FileFlagsBitIsSet(uint32_t fileFlags, uint32_t bit); static bool ActionBitIsSet(kauth_action_t action, kauth_action_t mask); static bool ActionBitsNotSet(kauth_action_t action, kauth_action_t mask); @@ -326,24 +325,6 @@ static int HandleVnodeOperation( { goto CleanupAndReturn; } - - // This delete could be part of a rename, and so we must hydrate the file now to - // ensure it's hydrated post-rename - if (kauthResult == KAUTH_RESULT_DEFER && - FileFlagsBitIsSet(currentVnodeFileFlags, FileFlags_IsEmpty)) - { - if (!TrySendRequestAndWaitForResponse( - root, - MessageType_KtoU_HydrateFile, - currentVnode, - pid, - procname, - &kauthResult, - kauthError)) - { - goto CleanupAndReturn; - } - } } else if (ActionBitIsSet( action, @@ -470,21 +451,13 @@ static bool ShouldHandleVnodeOpEvent( } *vnodeFileFlags = ReadVNodeFileFlags(vnode, context); - bool filedFlaggedAsInVirtualizationRoot = FileFlagsBitIsSet(*vnodeFileFlags, FileFlags_IsInVirtualizationRoot); - if (!filedFlaggedAsInVirtualizationRoot) + if (!FileFlagsBitIsSet(*vnodeFileFlags, FileFlags_IsInVirtualizationRoot)) { - // TODO(Mac): Determine the overhead of making this check for all actions and - // vnodes. If it's too high, this check can be further reduced to the scenarios - // that VFSForGit care about. - if (!HasParentInVirtualizationRoot(vnode, context)) - { - // This vnode is not part of ANY virtualization root (and none of its ancestors - // are either), so exit now before doing any more work. This gives us a cheap - // way to avoid adding overhead to IO outside of a virtualization root. - - *kauthResult = KAUTH_RESULT_DEFER; - return false; - } + // This vnode is not part of ANY virtualization root, so exit now before doing any more work. + // This gives us a cheap way to avoid adding overhead to IO outside of a virtualization root. + + *kauthResult = KAUTH_RESULT_DEFER; + return false; } *pid = GetPid(context); @@ -523,12 +496,7 @@ static bool ShouldHandleVnodeOpEvent( // TODO(Mac): why do we need to check rootIndex > 0 here? // If root was null, we would have already exited. And if not null, it can't be < 0. - // - // TODO(Mac): Double check that rules are properly enforced when filedFlaggedAsInVirtualizationRoot - // is false. These checks are currently skipped when filedFlaggedAsInVirtualizationRoot is false to - // allow providers to create new files inside of their root before starting the virtualization - // instance - if (filedFlaggedAsInVirtualizationRoot && rootIndex >= 0) + if (rootIndex >= 0) { bool vnodeIsDir = (*vnodeType == VDIR); @@ -811,34 +779,6 @@ static uint32_t ReadVNodeFileFlags(vnode_t vn, vfs_context_t context) return attributes.va_flags; } -static bool HasParentInVirtualizationRoot(vnode_t vnode, vfs_context_t context) -{ - vnode_get(vnode); - - bool inRoot = false; - // Search up the tree until we hit a folder know to be inside a virtualization root - // or THE root of the file system - while (NULLVP != vnode && !vnode_isvroot(vnode)) - { - uint32_t vnodeFileFlags = ReadVNodeFileFlags(vnode, context); - if (FileFlagsBitIsSet(vnodeFileFlags, FileFlags_IsInVirtualizationRoot)) - { - inRoot = true; - break; - } - vnode_t parent = vnode_getparent(vnode); - vnode_put(vnode); - vnode = parent; - } - - if (NULLVP != vnode) - { - vnode_put(vnode); - } - - return inRoot; -} - static bool FileFlagsBitIsSet(uint32_t fileFlags, uint32_t bit) { // Note: if multiple bits are set in 'bit', this will return true if ANY are set in fileFlags diff --git a/ProjFS.Mac/PrjFSKext/PrjFSKext/Message_Kernel.cpp b/ProjFS.Mac/PrjFSKext/PrjFSKext/Message_Kernel.cpp index e683faa7c..1c0d80ced 100644 --- a/ProjFS.Mac/PrjFSKext/PrjFSKext/Message_Kernel.cpp +++ b/ProjFS.Mac/PrjFSKext/PrjFSKext/Message_Kernel.cpp @@ -12,6 +12,16 @@ void Message_Init( { header->messageId = messageId; header->messageType = messageType; + header->pid = pid; + + if (nullptr != procname) + { + strlcpy(header->procname, procname, sizeof(header->procname)); + } + else + { + header->procname[0] = '\0'; + } if (nullptr != path) { diff --git a/ProjFS.Mac/PrjFSLib/PrjFSLib.cpp b/ProjFS.Mac/PrjFSLib/PrjFSLib.cpp index 022718204..21b13c5d2 100644 --- a/ProjFS.Mac/PrjFSLib/PrjFSLib.cpp +++ b/ProjFS.Mac/PrjFSLib/PrjFSLib.cpp @@ -391,7 +391,7 @@ static Message ParseMessageMemory(const void* messageMemory, uint32_t size) abort(); } - const char* path = nullptr; + const char* path = ""; if (header->pathSizeBytes > 0) { path = static_cast(messageMemory) + sizeof(*header); @@ -587,7 +587,8 @@ static PrjFS_Result HandleFileNotification( { #ifdef DEBUG std::cout << "PrjFSLib.HandleFileNotification: " << path - << " notificationType: " << NotificationTypeToString(notificationType) << std::endl; + << " notificationType: " << NotificationTypeToString(notificationType) + << " isDirectory: " << isDirectory << std::endl; #endif char fullPath[PrjFSMaxPath]; From 553615dbf180390f0eb2f614f92cf31d044c028f Mon Sep 17 00:00:00 2001 From: John Briggs Date: Fri, 17 Aug 2018 09:02:17 -0400 Subject: [PATCH 034/272] Centralize business logic for starting background prefetch. Move the check for unattended enlistment from InProcessMount to BackgroundPrefetcher. --- GVFS/GVFS.Common/Prefetch/BackgroundPrefetcher.cs | 4 ++-- GVFS/GVFS.Mount/InProcessMount.cs | 6 +----- 2 files changed, 3 insertions(+), 7 deletions(-) diff --git a/GVFS/GVFS.Common/Prefetch/BackgroundPrefetcher.cs b/GVFS/GVFS.Common/Prefetch/BackgroundPrefetcher.cs index fb0a14efb..e9927393a 100644 --- a/GVFS/GVFS.Common/Prefetch/BackgroundPrefetcher.cs +++ b/GVFS/GVFS.Common/Prefetch/BackgroundPrefetcher.cs @@ -29,14 +29,14 @@ public BackgroundPrefetcher(ITracer tracer, GVFSEnlistment enlistment, PhysicalF this.prefetchJobThread = null; - if (gitObjects.IsUsingCacheServer()) + if (gitObjects.IsUsingCacheServer() && !GVFSEnlistment.IsUnattended(tracer)) { this.prefetchJobTimer = new Timer((state) => this.LaunchPrefetchJobIfIdle(), null, this.timerPeriod, this.timerPeriod); this.tracer.RelatedInfo(nameof(BackgroundPrefetcher) + ": starting background prefetch timer"); } else { - this.tracer.RelatedInfo(nameof(BackgroundPrefetcher) + ": no configured cache server, not starting background prefetch timer"); + this.tracer.RelatedInfo(nameof(BackgroundPrefetcher) + ": no configured cache server or enlistment is unattended, not starting background prefetch timer"); } } diff --git a/GVFS/GVFS.Mount/InProcessMount.cs b/GVFS/GVFS.Mount/InProcessMount.cs index 6fe957eab..038d31c79 100644 --- a/GVFS/GVFS.Mount/InProcessMount.cs +++ b/GVFS/GVFS.Mount/InProcessMount.cs @@ -515,11 +515,7 @@ private void MountAndStartWorkingDirectoryCallbacks(CacheServerInfo cache) } this.fileSystemCallbacks = this.CreateOrReportAndExit(() => new FileSystemCallbacks(this.context, this.gitObjects, RepoMetadata.Instance, virtualizer, gitStatusCache), "Failed to create src folder callback listener"); - - if (!this.context.Unattended) - { - this.prefetcher = this.CreateOrReportAndExit(() => new BackgroundPrefetcher(this.tracer, this.enlistment, this.context.FileSystem, this.gitObjects), "Failed to start background prefetcher"); - } + this.prefetcher = this.CreateOrReportAndExit(() => new BackgroundPrefetcher(this.tracer, this.enlistment, this.context.FileSystem, this.gitObjects), "Failed to start background prefetcher"); int majorVersion; int minorVersion; From e9a616c133bd324bde5f6e88dfeb5d62c90f3010 Mon Sep 17 00:00:00 2001 From: William Baker Date: Fri, 17 Aug 2018 09:58:32 -0700 Subject: [PATCH 035/272] Fix notification registration in MirrorProvider --- .../MirrorProvider.Windows/WindowsFileSystemVirtualizer.cs | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/MirrorProvider/MirrorProvider.Windows/WindowsFileSystemVirtualizer.cs b/MirrorProvider/MirrorProvider.Windows/WindowsFileSystemVirtualizer.cs index dd49ca5e5..46956a836 100644 --- a/MirrorProvider/MirrorProvider.Windows/WindowsFileSystemVirtualizer.cs +++ b/MirrorProvider/MirrorProvider.Windows/WindowsFileSystemVirtualizer.cs @@ -41,8 +41,7 @@ public override bool TryStartVirtualizationInstance(Enlistment enlistment, out s NotificationMapping[] notificationMappings = new NotificationMapping[] { - new NotificationMapping(NotificationType.PreDelete, string.Empty), - new NotificationMapping(NotificationType.FileHandleClosedFileModified, string.Empty), + new NotificationMapping(NotificationType.PreDelete | NotificationType.FileHandleClosedFileModified, string.Empty), }; HResult result = this.virtualizationInstance.StartVirtualizationInstance( From ac386e51586e6fef963841d24ed7164a138b0e49 Mon Sep 17 00:00:00 2001 From: William Baker Date: Fri, 17 Aug 2018 10:19:31 -0700 Subject: [PATCH 036/272] Move newline into ModifiedPathsShouldNotContain --- .../EnlistmentPerFixture/GitFilesTests.cs | 38 +++++++++---------- .../GVFS.FunctionalTests/Tools/GVFSHelpers.cs | 8 ++-- 2 files changed, 24 insertions(+), 22 deletions(-) diff --git a/GVFS/GVFS.FunctionalTests/Tests/EnlistmentPerFixture/GitFilesTests.cs b/GVFS/GVFS.FunctionalTests/Tests/EnlistmentPerFixture/GitFilesTests.cs index 1ec43b310..a0b7b55c1 100644 --- a/GVFS/GVFS.FunctionalTests/Tests/EnlistmentPerFixture/GitFilesTests.cs +++ b/GVFS/GVFS.FunctionalTests/Tests/EnlistmentPerFixture/GitFilesTests.cs @@ -34,7 +34,7 @@ public void CreateFileTest() this.Enlistment.WaitForBackgroundOperations().ShouldEqual(true, "Background operations failed to complete."); this.Enlistment.GetVirtualPathTo(fileName).ShouldBeAFile(this.fileSystem).WithContents("Some content here"); - GVFSHelpers.ModifiedPathsShouldContain(this.fileSystem, this.Enlistment.DotGVFSRoot, fileName + Environment.NewLine); + GVFSHelpers.ModifiedPathsShouldContain(this.fileSystem, this.Enlistment.DotGVFSRoot, fileName); } [TestCase, Order(2)] @@ -54,8 +54,8 @@ public void CreateFileInFolderTest() this.Enlistment.WaitForBackgroundOperations().ShouldEqual(true, "Background operations failed to complete."); - GVFSHelpers.ModifiedPathsShouldContain(this.fileSystem, this.Enlistment.DotGVFSRoot, folderName + "/" + GVFSHelpers.ModifiedPathsNewLine); - GVFSHelpers.ModifiedPathsShouldContain(this.fileSystem, this.Enlistment.DotGVFSRoot, folderName + "/" + fileName + GVFSHelpers.ModifiedPathsNewLine); + GVFSHelpers.ModifiedPathsShouldContain(this.fileSystem, this.Enlistment.DotGVFSRoot, folderName + "/"); + GVFSHelpers.ModifiedPathsShouldContain(this.fileSystem, this.Enlistment.DotGVFSRoot, folderName + "/" + fileName); } [TestCase, Order(3)] @@ -88,12 +88,12 @@ public void RenameFolderTest() string[] fileNames = { "a", "b", "c" }; string[] expectedModifiedEntries = { - renamedFolderName + "/" + fileNames[0] + Environment.NewLine, - renamedFolderName + "/" + fileNames[1] + Environment.NewLine, - renamedFolderName + "/" + fileNames[2] + Environment.NewLine, - folderName + "/" + fileNames[0] + Environment.NewLine, - folderName + "/" + fileNames[1] + Environment.NewLine, - folderName + "/" + fileNames[2] + Environment.NewLine, + renamedFolderName + "/" + fileNames[0], + renamedFolderName + "/" + fileNames[1], + renamedFolderName + "/" + fileNames[2], + folderName + "/" + fileNames[0], + folderName + "/" + fileNames[1], + folderName + "/" + fileNames[2], }; this.Enlistment.GetVirtualPathTo(folderName).ShouldNotExistOnDisk(this.fileSystem); @@ -158,7 +158,7 @@ public void ReadingFileDoesNotUpdateIndexOrSparseCheckout() afterUpdateResult.Output.StartsWith("S ").ShouldEqual(true); afterUpdateResult.Output.ShouldContain("ctime: 0:0", "mtime: 0:0", "size: 0\t"); - GVFSHelpers.ModifiedPathsShouldNotContain(this.fileSystem, this.Enlistment.DotGVFSRoot, gitFileToCheck + GVFSHelpers.ModifiedPathsNewLine); + GVFSHelpers.ModifiedPathsShouldNotContain(this.fileSystem, this.Enlistment.DotGVFSRoot, gitFileToCheck); } // TODO(Mac): Enable this test once the LockHolder is converted to .NET Core @@ -178,7 +178,7 @@ public void ModifiedFileWillGetAddedToModifiedPathsFile() this.Enlistment.WaitForBackgroundOperations().ShouldEqual(true, "Background operations did not complete."); - GVFSHelpers.ModifiedPathsShouldContain(this.fileSystem, this.Enlistment.DotGVFSRoot, gitFileToTest + GVFSHelpers.ModifiedPathsNewLine); + GVFSHelpers.ModifiedPathsShouldContain(this.fileSystem, this.Enlistment.DotGVFSRoot, gitFileToTest); this.VerifyWorktreeBit(gitFileToTest, LsFilesStatus.Cached); } @@ -197,8 +197,8 @@ public void RenamedFileAddedToSparseCheckoutAndSkipWorktreeBitCleared() this.Enlistment.GetVirtualPathTo(fileToRenameTargetRelativePath)); this.Enlistment.WaitForBackgroundOperations().ShouldEqual(true, "Background operations failed to complete."); - GVFSHelpers.ModifiedPathsShouldContain(this.fileSystem, this.Enlistment.DotGVFSRoot, fileToRenameEntry + GVFSHelpers.ModifiedPathsNewLine); - GVFSHelpers.ModifiedPathsShouldContain(this.fileSystem, this.Enlistment.DotGVFSRoot, fileToRenameTargetEntry + GVFSHelpers.ModifiedPathsNewLine); + GVFSHelpers.ModifiedPathsShouldContain(this.fileSystem, this.Enlistment.DotGVFSRoot, fileToRenameEntry); + GVFSHelpers.ModifiedPathsShouldContain(this.fileSystem, this.Enlistment.DotGVFSRoot, fileToRenameTargetEntry); // Verify skip-worktree cleared this.VerifyWorktreeBit(fileToRenameEntry, LsFilesStatus.Cached); @@ -220,8 +220,8 @@ public void RenamedFileAndOverwrittenTargetAddedToSparseCheckoutAndSkipWorktreeB this.Enlistment.GetVirtualPathTo(fileToRenameTargetRelativePath)); this.Enlistment.WaitForBackgroundOperations().ShouldEqual(true, "Background operations failed to complete."); - GVFSHelpers.ModifiedPathsShouldContain(this.fileSystem, this.Enlistment.DotGVFSRoot, fileToRenameEntry + GVFSHelpers.ModifiedPathsNewLine); - GVFSHelpers.ModifiedPathsShouldContain(this.fileSystem, this.Enlistment.DotGVFSRoot, fileToRenameTargetEntry + GVFSHelpers.ModifiedPathsNewLine); + GVFSHelpers.ModifiedPathsShouldContain(this.fileSystem, this.Enlistment.DotGVFSRoot, fileToRenameEntry); + GVFSHelpers.ModifiedPathsShouldContain(this.fileSystem, this.Enlistment.DotGVFSRoot, fileToRenameTargetEntry); // Verify skip-worktree cleared this.VerifyWorktreeBit(fileToRenameEntry, LsFilesStatus.Cached); @@ -237,7 +237,7 @@ public void DeletedFileAddedToModifiedPathsFile() this.fileSystem.DeleteFile(this.Enlistment.GetVirtualPathTo(fileToDeleteEntry)); this.Enlistment.WaitForBackgroundOperations().ShouldEqual(true, "Background operations failed to complete."); - GVFSHelpers.ModifiedPathsShouldContain(this.fileSystem, this.Enlistment.DotGVFSRoot, fileToDeleteEntry + GVFSHelpers.ModifiedPathsNewLine); + GVFSHelpers.ModifiedPathsShouldContain(this.fileSystem, this.Enlistment.DotGVFSRoot, fileToDeleteEntry); // Verify skip-worktree cleared this.VerifyWorktreeBit(fileToDeleteEntry, LsFilesStatus.Cached); @@ -288,7 +288,7 @@ public void FileRenamedOutOfRepoAddedToModifiedPathsFile() this.Enlistment.WaitForBackgroundOperations().ShouldEqual(true, "Background operations failed to complete."); - GVFSHelpers.ModifiedPathsShouldContain(this.fileSystem, this.Enlistment.DotGVFSRoot, fileToRenameEntry + GVFSHelpers.ModifiedPathsNewLine); + GVFSHelpers.ModifiedPathsShouldContain(this.fileSystem, this.Enlistment.DotGVFSRoot, fileToRenameEntry); // Verify skip-worktree cleared this.VerifyWorktreeBit(fileToRenameEntry, LsFilesStatus.Cached); @@ -308,7 +308,7 @@ public void OverwrittenFileAddedToSparseCheckoutAndSkipWorktreeBitCleared() fileToOverwriteVirtualPath.ShouldBeAFile(this.fileSystem).WithContents(testContents); - GVFSHelpers.ModifiedPathsShouldContain(this.fileSystem, this.Enlistment.DotGVFSRoot, fileToOverwriteEntry + GVFSHelpers.ModifiedPathsNewLine); + GVFSHelpers.ModifiedPathsShouldContain(this.fileSystem, this.Enlistment.DotGVFSRoot, fileToOverwriteEntry); // Verify skip-worktree cleared this.VerifyWorktreeBit(fileToOverwriteEntry, LsFilesStatus.Cached); @@ -327,7 +327,7 @@ public void SupersededFileAddedToSparseCheckoutAndSkipWorktreeBitCleared() SupersedeFile(fileToSupersedePath, newContent).ShouldEqual(true); this.Enlistment.WaitForBackgroundOperations().ShouldEqual(true, "Background operations failed to complete."); - GVFSHelpers.ModifiedPathsShouldContain(this.fileSystem, this.Enlistment.DotGVFSRoot, fileToSupersedeEntry + GVFSHelpers.ModifiedPathsNewLine); + GVFSHelpers.ModifiedPathsShouldContain(this.fileSystem, this.Enlistment.DotGVFSRoot, fileToSupersedeEntry); // Verify skip-worktree cleared this.VerifyWorktreeBit(fileToSupersedeEntry, LsFilesStatus.Cached); diff --git a/GVFS/GVFS.FunctionalTests/Tools/GVFSHelpers.cs b/GVFS/GVFS.FunctionalTests/Tools/GVFSHelpers.cs index 0831ab218..17fd64369 100644 --- a/GVFS/GVFS.FunctionalTests/Tools/GVFSHelpers.cs +++ b/GVFS/GVFS.FunctionalTests/Tools/GVFSHelpers.cs @@ -6,14 +6,13 @@ using NUnit.Framework; using System.Collections.Generic; using System.IO; +using System.Linq; using System.Reflection; namespace GVFS.FunctionalTests.Tools { public static class GVFSHelpers { - public const string ModifiedPathsNewLine = "\r\n"; - public static readonly string BackgroundOpsFile = Path.Combine("databases", "BackgroundGitOperations.dat"); public static readonly string PlaceholderListFile = Path.Combine("databases", "PlaceholderList.dat"); public static readonly string RepoMetadataName = Path.Combine("databases", "RepoMetadata.dat"); @@ -24,6 +23,8 @@ public static class GVFSHelpers private const string GitObjectsRootKey = "GitObjectsRoot"; private const string BlobSizesRootKey = "BlobSizesRoot"; + private const string ModifiedPathsNewLine = "\r\n"; + public static void SaveDiskLayoutVersion(string dotGVFSRoot, string majorVersion, string minorVersion) { SavePersistedValue(dotGVFSRoot, DiskLayoutMajorVersionKey, majorVersion); @@ -105,7 +106,8 @@ public static void ModifiedPathsShouldContain(FileSystemRunner fileSystem, strin { string modifiedPathsDatabase = Path.Combine(dotGVFSRoot, TestConstants.Databases.ModifiedPaths); modifiedPathsDatabase.ShouldBeAFile(fileSystem); - GVFSHelpers.ReadAllTextFromWriteLockedFile(modifiedPathsDatabase).ShouldContain(gitPaths); + GVFSHelpers.ReadAllTextFromWriteLockedFile(modifiedPathsDatabase).ShouldContain( + gitPaths.Select(path => path + ModifiedPathsNewLine).ToArray()); } public static void ModifiedPathsShouldNotContain(FileSystemRunner fileSystem, string dotGVFSRoot, params string[] gitPaths) From fa420073943435403126bd5ce1cfe61311cb5434 Mon Sep 17 00:00:00 2001 From: William Baker Date: Fri, 17 Aug 2018 10:31:03 -0700 Subject: [PATCH 037/272] Add logging to KauthHandler_HandleKernelMessageResponse --- ProjFS.Mac/PrjFSKext/PrjFSKext/KauthHandler.cpp | 1 + 1 file changed, 1 insertion(+) diff --git a/ProjFS.Mac/PrjFSKext/PrjFSKext/KauthHandler.cpp b/ProjFS.Mac/PrjFSKext/PrjFSKext/KauthHandler.cpp index 5226cb162..f6cff7aed 100644 --- a/ProjFS.Mac/PrjFSKext/PrjFSKext/KauthHandler.cpp +++ b/ProjFS.Mac/PrjFSKext/PrjFSKext/KauthHandler.cpp @@ -225,6 +225,7 @@ void KauthHandler_HandleKernelMessageResponse(uint64_t messageId, MessageType re case MessageType_KtoU_NotifyFileModified: case MessageType_KtoU_NotifyFilePreDelete: case MessageType_KtoU_NotifyDirectoryPreDelete: + KextLog_Error("KauthHandler_HandleKernelMessageResponse: Unexpected responseType: %d", responseType); break; } From 3091ebf1bf847e735905f48af62b6080c0b3c63c Mon Sep 17 00:00:00 2001 From: John Briggs Date: Fri, 17 Aug 2018 14:24:41 -0400 Subject: [PATCH 038/272] Extract the logic to schedule post-prefetch job to its own function. --- GVFS/GVFS.Common/Prefetch/CommitPrefetcher.cs | 89 ++++++++++--------- 1 file changed, 47 insertions(+), 42 deletions(-) diff --git a/GVFS/GVFS.Common/Prefetch/CommitPrefetcher.cs b/GVFS/GVFS.Common/Prefetch/CommitPrefetcher.cs index bdafe1dd2..f2ee21742 100644 --- a/GVFS/GVFS.Common/Prefetch/CommitPrefetcher.cs +++ b/GVFS/GVFS.Common/Prefetch/CommitPrefetcher.cs @@ -48,51 +48,12 @@ public static bool TryPrefetchCommitsAndTrees( } } - if (packIndexes == null || packIndexes.Count == 0) + if (packIndexes?.Count > 0) { - return true; + TrySchedulePostFetchJob(tracer, enlistment.NamedPipeName, packIndexes); } - // We make a best-effort request to run MIDX and commit-graph writes - using (NamedPipeClient pipeClient = new NamedPipeClient(enlistment.NamedPipeName)) - { - if (!pipeClient.Connect()) - { - tracer.RelatedWarning( - metadata: null, - message: "Failed to connect to GVFS. Skipping post-fetch job request.", - keywords: Keywords.Telemetry); - return true; - } - - NamedPipeMessages.RunPostFetchJob.Request request = new NamedPipeMessages.RunPostFetchJob.Request(packIndexes); - if (pipeClient.TrySendRequest(request.CreateMessage())) - { - NamedPipeMessages.Message response; - - if (pipeClient.TryReadResponse(out response)) - { - tracer.RelatedInfo("Requested post-fetch job with resonse '{0}'", response.Header); - return true; - } - else - { - tracer.RelatedWarning( - metadata: null, - message: "Requested post-fetch job failed to respond", - keywords: Keywords.Telemetry); - } - } - else - { - tracer.RelatedWarning( - metadata: null, - message: "Message to named pipe failed to send, skipping post-fetch job request.", - keywords: Keywords.Telemetry); - } - } - - return false; + return true; } public static bool TryGetMaxGoodPrefetchTimestamp( @@ -187,6 +148,50 @@ public static bool TryGetMaxGoodPrefetchTimestamp( return true; } + private static bool TrySchedulePostFetchJob(ITracer tracer, string namedPipeName, List packIndexes) + { + // We make a best-effort request to run MIDX and commit-graph writes + using (NamedPipeClient pipeClient = new NamedPipeClient(namedPipeName)) + { + if (!pipeClient.Connect()) + { + tracer.RelatedWarning( + metadata: null, + message: "Failed to connect to GVFS. Skipping post-fetch job request.", + keywords: Keywords.Telemetry); + return false; + } + + NamedPipeMessages.RunPostFetchJob.Request request = new NamedPipeMessages.RunPostFetchJob.Request(packIndexes); + if (pipeClient.TrySendRequest(request.CreateMessage())) + { + NamedPipeMessages.Message response; + + if (pipeClient.TryReadResponse(out response)) + { + tracer.RelatedInfo("Requested post-fetch job with resonse '{0}'", response.Header); + return true; + } + else + { + tracer.RelatedWarning( + metadata: null, + message: "Requested post-fetch job failed to respond", + keywords: Keywords.Telemetry); + } + } + else + { + tracer.RelatedWarning( + metadata: null, + message: "Message to named pipe failed to send, skipping post-fetch job request.", + keywords: Keywords.Telemetry); + } + } + + return false; + } + private static long? GetTimestamp(string packName) { string filename = Path.GetFileName(packName); From 52f6522788b9e6f9dcbc9b92aba4fc6a32a2c196 Mon Sep 17 00:00:00 2001 From: William Baker Date: Fri, 17 Aug 2018 11:27:13 -0700 Subject: [PATCH 039/272] Fix failing functional tests --- .../Tests/EnlistmentPerFixture/GitFilesTests.cs | 4 ++-- .../Tests/EnlistmentPerFixture/WorkingDirectoryTests.cs | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/GVFS/GVFS.FunctionalTests/Tests/EnlistmentPerFixture/GitFilesTests.cs b/GVFS/GVFS.FunctionalTests/Tests/EnlistmentPerFixture/GitFilesTests.cs index a0b7b55c1..a7d31e54f 100644 --- a/GVFS/GVFS.FunctionalTests/Tests/EnlistmentPerFixture/GitFilesTests.cs +++ b/GVFS/GVFS.FunctionalTests/Tests/EnlistmentPerFixture/GitFilesTests.cs @@ -66,8 +66,8 @@ public void RenameEmptyFolderTest() string renamedFolderName = "folder3b"; string[] expectedModifiedEntries = { - folderName + "/" + Environment.NewLine, - renamedFolderName + "/" + Environment.NewLine, + folderName + "/", + renamedFolderName + "/", }; this.Enlistment.GetVirtualPathTo(folderName).ShouldNotExistOnDisk(this.fileSystem); diff --git a/GVFS/GVFS.FunctionalTests/Tests/EnlistmentPerFixture/WorkingDirectoryTests.cs b/GVFS/GVFS.FunctionalTests/Tests/EnlistmentPerFixture/WorkingDirectoryTests.cs index 1f0a1e5b0..01ac6c8dc 100644 --- a/GVFS/GVFS.FunctionalTests/Tests/EnlistmentPerFixture/WorkingDirectoryTests.cs +++ b/GVFS/GVFS.FunctionalTests/Tests/EnlistmentPerFixture/WorkingDirectoryTests.cs @@ -437,7 +437,7 @@ public void FolderContentsCorrectAfterCreateNewFolderRenameAndCheckoutCommitWith this.fileSystem.MoveDirectory(newFolder, folder); this.Enlistment.WaitForBackgroundOperations().ShouldEqual(true, "Background operations failed to complete."); - GVFSHelpers.ModifiedPathsShouldContain(this.fileSystem, this.Enlistment.DotGVFSRoot, folderName); + GVFSHelpers.ModifiedPathsShouldContain(this.fileSystem, this.Enlistment.DotGVFSRoot, folderName + "/"); // Switch back to this.ControlGitRepo.Commitish and confirm that folder contents are correct GitProcess.InvokeProcess(this.Enlistment.RepoRoot, "checkout " + Properties.Settings.Default.Commitish); From 0d29e4637630d20ecbaa7f52e3d6f0adbc948a47 Mon Sep 17 00:00:00 2001 From: William Baker Date: Fri, 17 Aug 2018 12:25:12 -0700 Subject: [PATCH 040/272] Remove elses when checking for delete in HandleVnodeOperation --- ProjFS.Mac/PrjFSKext/PrjFSKext/KauthHandler.cpp | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/ProjFS.Mac/PrjFSKext/PrjFSKext/KauthHandler.cpp b/ProjFS.Mac/PrjFSKext/PrjFSKext/KauthHandler.cpp index f6cff7aed..3288854ad 100644 --- a/ProjFS.Mac/PrjFSKext/PrjFSKext/KauthHandler.cpp +++ b/ProjFS.Mac/PrjFSKext/PrjFSKext/KauthHandler.cpp @@ -296,7 +296,8 @@ static int HandleVnodeOperation( } } } - else if (ActionBitIsSet(action, KAUTH_VNODE_DELETE)) + + if (ActionBitIsSet(action, KAUTH_VNODE_DELETE)) { if (!TrySendRequestAndWaitForResponse( root, @@ -327,7 +328,8 @@ static int HandleVnodeOperation( goto CleanupAndReturn; } } - else if (ActionBitIsSet( + + if (ActionBitIsSet( action, KAUTH_VNODE_READ_ATTRIBUTES | KAUTH_VNODE_WRITE_ATTRIBUTES | From a4be78401774fec55a95ed940864ca4c7c95ff7f Mon Sep 17 00:00:00 2001 From: Phil Dennis-Jordan Date: Thu, 16 Aug 2018 15:15:13 +0200 Subject: [PATCH 041/272] Mac kext logging: Make time stamps more human readable This changes the prjfs-log tool to print log time stamps in seconds (with millisecond precision) since the start of the log rather than outputting the raw Mach absolute time stamp, which has fairly arbitrary and non-human-readable units. --- ProjFS.Mac/PrjFSLib/prjfs-log/prjfs-log.cpp | 16 +++++++++++++++- 1 file changed, 15 insertions(+), 1 deletion(-) diff --git a/ProjFS.Mac/PrjFSLib/prjfs-log/prjfs-log.cpp b/ProjFS.Mac/PrjFSLib/prjfs-log/prjfs-log.cpp index 220c58858..281fc2bd3 100644 --- a/ProjFS.Mac/PrjFSLib/prjfs-log/prjfs-log.cpp +++ b/ProjFS.Mac/PrjFSLib/prjfs-log/prjfs-log.cpp @@ -7,9 +7,15 @@ #include static const char* KextLogLevelAsString(KextLog_Level level); +static uint64_t nanosecondsFromAbsoluteTime(uint64_t machAbsoluteTime); + +static mach_timebase_info_data_t s_machTimebase; int main(int argc, const char * argv[]) { + mach_timebase_info(&s_machTimebase); + const uint64_t machStartTime = mach_absolute_time(); + io_connect_t connection = PrjFSService_ConnectToDriver(UserClientType_Log); if (connection == IO_OBJECT_NULL) { @@ -49,7 +55,10 @@ int main(int argc, const char * argv[]) const char* messageType = KextLogLevelAsString(message.level); int logStringLength = messageSize - sizeof(KextLog_MessageHeader) - 1; - printf("(%d: %llu) %s: %.*s\n", lineCount, message.machAbsoluteTimestamp, messageType, logStringLength, entry->data + sizeof(KextLog_MessageHeader)); + uint64_t timeOffsetNS = nanosecondsFromAbsoluteTime(message.machAbsoluteTimestamp - machStartTime); + uint64_t timeOffsetMS = timeOffsetNS / NSEC_PER_MSEC; + + printf("(%d: %5llu.%03llu) %s: %.*s\n", lineCount, timeOffsetMS / 1000u, timeOffsetMS % 1000u, messageType, logStringLength, entry->data + sizeof(KextLog_MessageHeader)); lineCount++; } @@ -76,3 +85,8 @@ static const char* KextLogLevelAsString(KextLog_Level level) return "Unknown"; } } + +static uint64_t nanosecondsFromAbsoluteTime(uint64_t machAbsoluteTime) +{ + return static_cast<__uint128_t>(machAbsoluteTime) * s_machTimebase.numer / s_machTimebase.denom; +} From ee207754ce7ca1e35593b7f03a1710e4674440a3 Mon Sep 17 00:00:00 2001 From: William Baker Date: Fri, 17 Aug 2018 12:58:12 -0700 Subject: [PATCH 042/272] Mac: New file notifications --- .../EnlistmentPerFixture/GitFilesTests.cs | 12 +- .../GitMoveRenameTests.cs | 10 +- .../PersistedModifiedPathsTests.cs | 3 +- .../Tests/GitCommands/AddStageTests.cs | 1 + .../Tests/GitCommands/GitCommandsTests.cs | 52 +++-- .../Tests/GitCommands/HashObjectTests.cs | 1 + .../MacFileSystemVirtualizer.cs | 36 ++++ .../PrjFSKext/PrjFSKext/KauthHandler.cpp | 204 ++++++++---------- ProjFS.Mac/PrjFSKext/public/Message.h | 1 + .../VirtualizationInstance.cs | 7 +- ProjFS.Mac/PrjFSLib/PrjFSLib.cpp | 14 ++ 11 files changed, 200 insertions(+), 141 deletions(-) diff --git a/GVFS/GVFS.FunctionalTests/Tests/EnlistmentPerFixture/GitFilesTests.cs b/GVFS/GVFS.FunctionalTests/Tests/EnlistmentPerFixture/GitFilesTests.cs index a7d31e54f..c2fac6926 100644 --- a/GVFS/GVFS.FunctionalTests/Tests/EnlistmentPerFixture/GitFilesTests.cs +++ b/GVFS/GVFS.FunctionalTests/Tests/EnlistmentPerFixture/GitFilesTests.cs @@ -24,17 +24,21 @@ public GitFilesTests(FileSystemRunner fileSystem) } [TestCase, Order(1)] - [Category(Categories.Mac.M2TODO)] public void CreateFileTest() { string fileName = "file1.txt"; GVFSHelpers.ModifiedPathsShouldNotContain(this.fileSystem, this.Enlistment.DotGVFSRoot, fileName); this.fileSystem.WriteAllText(this.Enlistment.GetVirtualPathTo(fileName), "Some content here"); - this.Enlistment.WaitForBackgroundOperations().ShouldEqual(true, "Background operations failed to complete."); - - this.Enlistment.GetVirtualPathTo(fileName).ShouldBeAFile(this.fileSystem).WithContents("Some content here"); GVFSHelpers.ModifiedPathsShouldContain(this.fileSystem, this.Enlistment.DotGVFSRoot, fileName); + this.Enlistment.GetVirtualPathTo(fileName).ShouldBeAFile(this.fileSystem).WithContents("Some content here"); + + string emptyFileName = "file1empty.txt"; + GVFSHelpers.ModifiedPathsShouldNotContain(this.fileSystem, this.Enlistment.DotGVFSRoot, emptyFileName); + this.fileSystem.CreateEmptyFile(this.Enlistment.GetVirtualPathTo(emptyFileName)); + this.Enlistment.WaitForBackgroundOperations().ShouldEqual(true, "Background operations failed to complete."); + GVFSHelpers.ModifiedPathsShouldContain(this.fileSystem, this.Enlistment.DotGVFSRoot, emptyFileName); + this.Enlistment.GetVirtualPathTo(fileName).ShouldBeAFile(this.fileSystem); } [TestCase, Order(2)] diff --git a/GVFS/GVFS.FunctionalTests/Tests/EnlistmentPerFixture/GitMoveRenameTests.cs b/GVFS/GVFS.FunctionalTests/Tests/EnlistmentPerFixture/GitMoveRenameTests.cs index 7ec766bb8..d0b1a09e4 100644 --- a/GVFS/GVFS.FunctionalTests/Tests/EnlistmentPerFixture/GitMoveRenameTests.cs +++ b/GVFS/GVFS.FunctionalTests/Tests/EnlistmentPerFixture/GitMoveRenameTests.cs @@ -32,14 +32,14 @@ public void GitStatus() } [TestCase, Order(2)] - [Category(Categories.Mac.M2TODO)] public void GitStatusAfterNewFile() { string filename = "new.cs"; + string filePath = this.Enlistment.GetVirtualPathTo(filename); - this.fileSystem.WriteAllText(this.Enlistment.GetVirtualPathTo(filename), this.testFileContents); + this.fileSystem.WriteAllText(filePath, this.testFileContents); - this.Enlistment.GetVirtualPathTo(filename).ShouldBeAFile(this.fileSystem).WithContents(this.testFileContents); + filePath.ShouldBeAFile(this.fileSystem).WithContents(this.testFileContents); GitHelpers.CheckGitCommandAgainstGVFSRepo( this.Enlistment.RepoRoot, @@ -47,6 +47,8 @@ public void GitStatusAfterNewFile() "On branch " + Properties.Settings.Default.Commitish, "Untracked files:", filename); + + this.fileSystem.DeleteFile(filePath); } [TestCase, Order(3)] @@ -54,7 +56,7 @@ public void GitStatusAfterNewFile() public void GitStatusAfterFileNameCaseChange() { string oldFilename = "new.cs"; - this.Enlistment.GetVirtualPathTo(oldFilename).ShouldBeAFile(this.fileSystem); + this.EnsureTestFileExists(oldFilename); string newFilename = "New.cs"; this.fileSystem.MoveFile(this.Enlistment.GetVirtualPathTo(oldFilename), this.Enlistment.GetVirtualPathTo(newFilename)); diff --git a/GVFS/GVFS.FunctionalTests/Tests/EnlistmentPerTestCase/PersistedModifiedPathsTests.cs b/GVFS/GVFS.FunctionalTests/Tests/EnlistmentPerTestCase/PersistedModifiedPathsTests.cs index 57ecef8d1..dfc1055c7 100644 --- a/GVFS/GVFS.FunctionalTests/Tests/EnlistmentPerTestCase/PersistedModifiedPathsTests.cs +++ b/GVFS/GVFS.FunctionalTests/Tests/EnlistmentPerTestCase/PersistedModifiedPathsTests.cs @@ -10,8 +10,7 @@ namespace GVFS.FunctionalTests.Tests.EnlistmentPerTestCase { [TestFixture] - //// TODO(Mac): Enable for M2 once notifications are working - //// [Category(Categories.Mac.M2)] + [Category(Categories.Mac.M2TODO)] public class PersistedModifiedPathsTests : TestsWithEnlistmentPerTestCase { private static readonly string FileToAdd = Path.Combine("GVFS", "TestAddFile.txt"); diff --git a/GVFS/GVFS.FunctionalTests/Tests/GitCommands/AddStageTests.cs b/GVFS/GVFS.FunctionalTests/Tests/GitCommands/AddStageTests.cs index eb9f74ab2..f079abe29 100644 --- a/GVFS/GVFS.FunctionalTests/Tests/GitCommands/AddStageTests.cs +++ b/GVFS/GVFS.FunctionalTests/Tests/GitCommands/AddStageTests.cs @@ -6,6 +6,7 @@ namespace GVFS.FunctionalTests.Tests.GitCommands { [TestFixture] [Category(Categories.GitCommands)] + [Category(Categories.Mac.M3)] public class AddStageTests : GitRepoTests { public AddStageTests() : base(enlistmentPerTest: false) diff --git a/GVFS/GVFS.FunctionalTests/Tests/GitCommands/GitCommandsTests.cs b/GVFS/GVFS.FunctionalTests/Tests/GitCommands/GitCommandsTests.cs index 1aac419aa..81383b9e5 100644 --- a/GVFS/GVFS.FunctionalTests/Tests/GitCommands/GitCommandsTests.cs +++ b/GVFS/GVFS.FunctionalTests/Tests/GitCommands/GitCommandsTests.cs @@ -88,48 +88,56 @@ public void RenameCurrentBranchTest() } [TestCase] + [Category(Categories.Mac.M2)] public void UntrackedFileTest() { this.BasicCommit(this.CreateFile, addCommand: "add ."); } [TestCase] + [Category(Categories.Mac.M2)] public void UntrackedEmptyFileTest() { this.BasicCommit(this.CreateEmptyFile, addCommand: "add ."); } [TestCase] + [Category(Categories.Mac.M2)] public void UntrackedFileAddAllTest() { this.BasicCommit(this.CreateFile, addCommand: "add --all"); } [TestCase] + [Category(Categories.Mac.M2)] public void UntrackedEmptyFileAddAllTest() { this.BasicCommit(this.CreateEmptyFile, addCommand: "add --all"); } [TestCase] + [Category(Categories.Mac.M2)] public void StageUntrackedFileTest() { this.BasicCommit(this.CreateFile, addCommand: "stage ."); } [TestCase] + [Category(Categories.Mac.M2)] public void StageUntrackedEmptyFileTest() { this.BasicCommit(this.CreateEmptyFile, addCommand: "stage ."); } [TestCase] + [Category(Categories.Mac.M2)] public void StageUntrackedFileAddAllTest() { this.BasicCommit(this.CreateFile, addCommand: "stage --all"); } [TestCase] + [Category(Categories.Mac.M2)] public void StageUntrackedEmptyFileAddAllTest() { this.BasicCommit(this.CreateEmptyFile, addCommand: "stage --all"); @@ -234,27 +242,33 @@ public void DeleteFolderCommitChangesSwitchBranchSwitchBackTest() } [TestCase] + [Category(Categories.Mac.M2)] public void DeleteFilesWithNameAheadOfDot() { - this.FolderShouldExistAndHaveFile("GitCommandsTests\\DeleteFileTests\\1", "#test"); - this.DeleteFile("GitCommandsTests\\DeleteFileTests\\1\\#test"); - this.FolderShouldExistAndBeEmpty("GitCommandsTests\\DeleteFileTests\\1"); - - this.FolderShouldExistAndHaveFile("GitCommandsTests\\DeleteFileTests\\2", "$test"); - this.DeleteFile("GitCommandsTests\\DeleteFileTests\\2\\$test"); - this.FolderShouldExistAndBeEmpty("GitCommandsTests\\DeleteFileTests\\2"); - - this.FolderShouldExistAndHaveFile("GitCommandsTests\\DeleteFileTests\\3", ")"); - this.DeleteFile("GitCommandsTests\\DeleteFileTests\\3\\)"); - this.FolderShouldExistAndBeEmpty("GitCommandsTests\\DeleteFileTests\\3"); - - this.FolderShouldExistAndHaveFile("GitCommandsTests\\DeleteFileTests\\4", "+.test"); - this.DeleteFile("GitCommandsTests\\DeleteFileTests\\4\\+.test"); - this.FolderShouldExistAndBeEmpty("GitCommandsTests\\DeleteFileTests\\4"); - - this.FolderShouldExistAndHaveFile("GitCommandsTests\\DeleteFileTests\\5", "-.test"); - this.DeleteFile("GitCommandsTests\\DeleteFileTests\\5\\-.test"); - this.FolderShouldExistAndBeEmpty("GitCommandsTests\\DeleteFileTests\\5"); + string folder = Path.Combine("GitCommandsTests", "DeleteFileTests", "1"); + this.FolderShouldExistAndHaveFile(folder, "#test"); + this.DeleteFile(Path.Combine(folder, "#test")); + this.FolderShouldExistAndBeEmpty(folder); + + folder = Path.Combine("GitCommandsTests", "DeleteFileTests", "2"); + this.FolderShouldExistAndHaveFile(folder, "$test"); + this.DeleteFile(Path.Combine(folder, "$test")); + this.FolderShouldExistAndBeEmpty(folder); + + folder = Path.Combine("GitCommandsTests", "DeleteFileTests", "3"); + this.FolderShouldExistAndHaveFile(folder, ")"); + this.DeleteFile(Path.Combine(folder, ")")); + this.FolderShouldExistAndBeEmpty(folder); + + folder = Path.Combine("GitCommandsTests", "DeleteFileTests", "4"); + this.FolderShouldExistAndHaveFile(folder, "+.test"); + this.DeleteFile(Path.Combine(folder, "+.test")); + this.FolderShouldExistAndBeEmpty(folder); + + folder = Path.Combine("GitCommandsTests", "DeleteFileTests", "5"); + this.FolderShouldExistAndHaveFile(folder, "-.test"); + this.DeleteFile(Path.Combine(folder, "-.test")); + this.FolderShouldExistAndBeEmpty(folder); this.ValidateGitCommand("status"); } diff --git a/GVFS/GVFS.FunctionalTests/Tests/GitCommands/HashObjectTests.cs b/GVFS/GVFS.FunctionalTests/Tests/GitCommands/HashObjectTests.cs index aa5d018ae..3c2bc5d86 100644 --- a/GVFS/GVFS.FunctionalTests/Tests/GitCommands/HashObjectTests.cs +++ b/GVFS/GVFS.FunctionalTests/Tests/GitCommands/HashObjectTests.cs @@ -5,6 +5,7 @@ namespace GVFS.FunctionalTests.Tests.GitCommands { [TestFixture] + [Category(Categories.Mac.M3)] public class HashObjectTests : GitRepoTests { public HashObjectTests() : base(enlistmentPerTest: false) diff --git a/GVFS/GVFS.Platform.Mac/MacFileSystemVirtualizer.cs b/GVFS/GVFS.Platform.Mac/MacFileSystemVirtualizer.cs index 0ffc5f495..66a7a83c9 100644 --- a/GVFS/GVFS.Platform.Mac/MacFileSystemVirtualizer.cs +++ b/GVFS/GVFS.Platform.Mac/MacFileSystemVirtualizer.cs @@ -101,6 +101,7 @@ protected override bool TryStart(out string error) this.virtualizationInstance.OnGetFileStream = this.OnGetFileStream; this.virtualizationInstance.OnFileModified = this.OnFileModified; this.virtualizationInstance.OnPreDelete = this.OnPreDelete; + this.virtualizationInstance.OnNewFileCreated = this.OnNewFileCreated; uint threadCount = (uint)Environment.ProcessorCount * 2; @@ -312,6 +313,41 @@ private Result OnPreDelete(string relativePath, bool isDirectory) return Result.Success; } + private void OnNewFileCreated(string relativePath, bool isDirectory) + { + try + { + if (!Virtualization.FileSystemCallbacks.IsPathInsideDotGit(relativePath)) + { + if (isDirectory) + { + GitCommandLineParser gitCommand = new GitCommandLineParser(this.Context.Repository.GVFSLock.GetLockedGitCommand()); + if (gitCommand.IsValidGitCommand) + { + // TODO(Mac): Ensure that when git creates a folder all files\folders within that folder are written to disk + EventMetadata metadata = this.CreateEventMetadata(relativePath); + metadata.Add("isDirectory", isDirectory); + this.Context.Tracer.RelatedWarning(metadata, $"{nameof(this.OnNewFileCreated)}: Git created a folder, currently an unsupported scenario on Mac"); + } + else + { + this.FileSystemCallbacks.OnFolderCreated(relativePath); + } + } + else + { + this.FileSystemCallbacks.OnFileCreated(relativePath); + } + } + } + catch (Exception e) + { + EventMetadata metadata = this.CreateEventMetadata(relativePath, e); + metadata.Add("isDirectory", isDirectory); + this.LogUnhandledExceptionAndExit(nameof(this.OnNewFileCreated), metadata); + } + } + private Result OnEnumerateDirectory( ulong commandId, string relativePath, diff --git a/ProjFS.Mac/PrjFSKext/PrjFSKext/KauthHandler.cpp b/ProjFS.Mac/PrjFSKext/PrjFSKext/KauthHandler.cpp index 3288854ad..db177bb4a 100644 --- a/ProjFS.Mac/PrjFSKext/PrjFSKext/KauthHandler.cpp +++ b/ProjFS.Mac/PrjFSKext/PrjFSKext/KauthHandler.cpp @@ -35,9 +35,10 @@ static int HandleFileOpOperation( static int GetPid(vfs_context_t context); static uint32_t ReadVNodeFileFlags(vnode_t vn, vfs_context_t context); -static bool FileFlagsBitIsSet(uint32_t fileFlags, uint32_t bit); -static bool ActionBitIsSet(kauth_action_t action, kauth_action_t mask); -static bool ActionBitsNotSet(kauth_action_t action, kauth_action_t mask); +static inline bool HasAncestorFlaggedAsInRoot(vnode_t vnode, vfs_context_t context); +static inline bool FileFlagsBitIsSet(uint32_t fileFlags, uint32_t bit); +static inline bool FileIsFlaggedAsInRoot(vnode_t vnode, vfs_context_t context); +static inline bool ActionBitIsSet(kauth_action_t action, kauth_action_t mask); static bool IsFileSystemCrawler(char* procname); @@ -225,6 +226,7 @@ void KauthHandler_HandleKernelMessageResponse(uint64_t messageId, MessageType re case MessageType_KtoU_NotifyFileModified: case MessageType_KtoU_NotifyFilePreDelete: case MessageType_KtoU_NotifyDirectoryPreDelete: + case MessageType_KtoU_NotifyFileCreated: KextLog_Error("KauthHandler_HandleKernelMessageResponse: Unexpected responseType: %d", responseType); break; } @@ -382,28 +384,66 @@ static int HandleFileOpOperation( // arg1 is the (const char *) path int closeFlags = static_cast(arg2); - if (KAUTH_FILEOP_CLOSE_MODIFIED == closeFlags) + vtype vnodeType = vnode_vtype(currentVnode); + if (ShouldIgnoreVnodeType(vnodeType, currentVnode)) { - VirtualizationRoot* root = nullptr; - int pid; - if (!ShouldHandleFileOpEvent( - context, + goto CleanupAndReturn; + } + + if (vnode_isdir(currentVnode)) + { + goto CleanupAndReturn; + } + + bool fileFlaggedInRoot = FileIsFlaggedAsInRoot(currentVnode, context); + if (fileFlaggedInRoot && KAUTH_FILEOP_CLOSE_MODIFIED != closeFlags) + { + goto CleanupAndReturn; + } + + if (!fileFlaggedInRoot && !HasAncestorFlaggedAsInRoot(currentVnode, context)) + { + goto CleanupAndReturn; + } + + VirtualizationRoot* root = nullptr; + int pid; + if (!ShouldHandleFileOpEvent( + context, + currentVnode, + action, + &root, + &pid)) + { + goto CleanupAndReturn; + } + + char procname[MAXCOMLEN + 1]; + proc_name(pid, procname, MAXCOMLEN + 1); + + if (fileFlaggedInRoot) + { + int kauthResult; + int kauthError; + if (!TrySendRequestAndWaitForResponse( + root, + MessageType_KtoU_NotifyFileModified, currentVnode, - action, - &root, - &pid)) + pid, + procname, + &kauthResult, + &kauthError)) { goto CleanupAndReturn; } - - char procname[MAXCOMLEN + 1]; - proc_name(pid, procname, MAXCOMLEN + 1); - + } + else + { int kauthResult; int kauthError; if (!TrySendRequestAndWaitForResponse( root, - MessageType_KtoU_NotifyFileModified, + MessageType_KtoU_NotifyFileCreated, currentVnode, pid, procname, @@ -484,7 +524,6 @@ static bool ShouldHandleVnodeOpEvent( } *root = VirtualizationRoots_FindForVnode(vnode); - int16_t rootIndex = nullptr == *root ? -1 : (*root)->index; if (nullptr == *root) { @@ -497,78 +536,8 @@ static bool ShouldHandleVnodeOpEvent( { // There is no registered provider for this root - // TODO(Mac): why do we need to check rootIndex > 0 here? - // If root was null, we would have already exited. And if not null, it can't be < 0. - if (rootIndex >= 0) - { - bool vnodeIsDir = (*vnodeType == VDIR); - - // File/directory is within an offline virtualization root (no provider) - // Allow read-only access to hydrated files, deny any writes except - // deletions, and prevent most read accesses to empty files. - // Empty directories need to be read/searched in order for them to - // be deleted by rm -r - if (ActionBitsNotSet(action, KAUTH_VNODE_ACCESS) && - ActionBitIsSet( - action, - KAUTH_VNODE_WRITE_ATTRIBUTES | - KAUTH_VNODE_WRITE_EXTATTRIBUTES | - KAUTH_VNODE_WRITE_DATA | - KAUTH_VNODE_APPEND_DATA | - KAUTH_VNODE_WRITE_SECURITY | - KAUTH_VNODE_LINKTARGET)) - { - KextLog_FileNote( - vnode, - "ShouldHandleEvent - write action 0x%x by process %u (%s) DENIED on %s with offline provider.", - action, - *pid, - procname, - vnodeIsDir ? "directory" : "file"); - - *kauthResult = KAUTH_RESULT_DENY; - return false; - } - - if (FileFlagsBitIsSet(*vnodeFileFlags, FileFlags_IsEmpty)) - { - // Empty files/directories with offline provider may only be queried or deleted - if (ActionBitIsSet( - action, - KAUTH_VNODE_ACCESS | - KAUTH_VNODE_DELETE_CHILD | - KAUTH_VNODE_DELETE | - KAUTH_VNODE_READ_EXTATTRIBUTES)) - { - *kauthResult = KAUTH_RESULT_DEFER; - return false; - } - - // Empty directories may additionally have their attributes and security read, and contents listed/searched (otherwise rm -r doesn't work) - if (vnodeIsDir && - ActionBitIsSet( - action, - KAUTH_VNODE_READ_ATTRIBUTES | - KAUTH_VNODE_READ_SECURITY | - KAUTH_VNODE_LIST_DIRECTORY | - KAUTH_VNODE_SEARCH)) - { - *kauthResult = KAUTH_RESULT_DEFER; - return false; - } - - // Disallow any other operations on empty placeholders - KextLog_FileNote( - vnode, - "ShouldHandleEvent - action 0x%x by process %u (%s) DENIED on empty %s with offline provider.", - action, - *pid, - procname, - vnodeIsDir ? "directory" : "file"); - *kauthResult = KAUTH_RESULT_DENY; - return false; - } - } + // TODO(Mac): Protect files in the worktree from modification (and prevent + // the creation of new files) when the provider is offline *kauthResult = KAUTH_RESULT_DEFER; return false; @@ -594,22 +563,6 @@ static bool ShouldHandleFileOpEvent( VirtualizationRoot** root, int* pid) { - vtype vnodeType = vnode_vtype(vnode); - if (ShouldIgnoreVnodeType(vnodeType, vnode)) - { - return false; - } - - // TODO(Mac): We will still want to handle renames into a root, and those vnodes would - // not yet have the FileFlags_IsInVirtualizationRoot set - uint32_t vnodeFileFlags = ReadVNodeFileFlags(vnode, context); - if (!FileFlagsBitIsSet(vnodeFileFlags, FileFlags_IsInVirtualizationRoot)) - { - // This vnode is not part of ANY virtualization root, so exit now before doing any more work. - // This gives us a cheap way to avoid adding overhead to IO outside of a virtualization root. - return false; - } - *root = VirtualizationRoots_FindForVnode(vnode); if (nullptr == *root) { @@ -782,20 +735,49 @@ static uint32_t ReadVNodeFileFlags(vnode_t vn, vfs_context_t context) return attributes.va_flags; } -static bool FileFlagsBitIsSet(uint32_t fileFlags, uint32_t bit) +static bool HasAncestorFlaggedAsInRoot(vnode_t vnode, vfs_context_t context) +{ + vnode_get(vnode); + + bool inRoot = false; + + // Search up the tree until we hit a folder know to be inside a virtualization root + // or the root of the file system + while (NULLVP != vnode && !vnode_isvroot(vnode)) + { + uint32_t vnodeFileFlags = ReadVNodeFileFlags(vnode, context); + if (FileFlagsBitIsSet(vnodeFileFlags, FileFlags_IsInVirtualizationRoot)) + { + inRoot = true; + break; + } + vnode_t parent = vnode_getparent(vnode); + vnode_put(vnode); + vnode = parent; + } + + if (NULLVP != vnode) + { + vnode_put(vnode); + } + + return inRoot; +} + +static inline bool FileFlagsBitIsSet(uint32_t fileFlags, uint32_t bit) { // Note: if multiple bits are set in 'bit', this will return true if ANY are set in fileFlags return 0 != (fileFlags & bit); } -static bool ActionBitIsSet(kauth_action_t action, kauth_action_t mask) +static inline bool FileIsFlaggedAsInRoot(vnode_t vnode, vfs_context_t context) { - return action & mask; + uint32_t vnodeFileFlags = ReadVNodeFileFlags(vnode, context); + return FileFlagsBitIsSet(vnodeFileFlags, FileFlags_IsInVirtualizationRoot); } - -static bool ActionBitsNotSet(kauth_action_t action, kauth_action_t mask) +static inline bool ActionBitIsSet(kauth_action_t action, kauth_action_t mask) { - return 0 == (action & mask); + return action & mask; } static bool IsFileSystemCrawler(char* procname) diff --git a/ProjFS.Mac/PrjFSKext/public/Message.h b/ProjFS.Mac/PrjFSKext/public/Message.h index 5817586f2..723303327 100644 --- a/ProjFS.Mac/PrjFSKext/public/Message.h +++ b/ProjFS.Mac/PrjFSKext/public/Message.h @@ -19,6 +19,7 @@ typedef enum MessageType_KtoU_NotifyFileModified, MessageType_KtoU_NotifyFilePreDelete, MessageType_KtoU_NotifyDirectoryPreDelete, + MessageType_KtoU_NotifyFileCreated, // Responses MessageType_Response_Success, diff --git a/ProjFS.Mac/PrjFSLib.Mac.Managed/VirtualizationInstance.cs b/ProjFS.Mac/PrjFSLib.Mac.Managed/VirtualizationInstance.cs index 0fd6b161a..40638e54b 100644 --- a/ProjFS.Mac/PrjFSLib.Mac.Managed/VirtualizationInstance.cs +++ b/ProjFS.Mac/PrjFSLib.Mac.Managed/VirtualizationInstance.cs @@ -15,7 +15,8 @@ public class VirtualizationInstance public virtual GetFileStreamCallback OnGetFileStream { get; set; } public virtual NotifyFileModified OnFileModified { get; set; } - public virtual NotifyPreDeleteEvent OnPreDelete { get; set; } + public virtual NotifyPreDeleteEvent OnPreDelete { get; set; } + public virtual NotifyNewFileCreatedEvent OnNewFileCreated { get; set; } public static Result ConvertDirectoryToVirtualizationRoot(string fullPath) { @@ -141,6 +142,10 @@ private Result OnNotifyOperation( case NotificationType.FileModified: this.OnFileModified(relativePath); return Result.Success; + + case NotificationType.NewFileCreated: + this.OnNewFileCreated(relativePath, isDirectory); + return Result.Success; } return Result.ENotYetImplemented; diff --git a/ProjFS.Mac/PrjFSLib/PrjFSLib.cpp b/ProjFS.Mac/PrjFSLib/PrjFSLib.cpp index 21b13c5d2..9956c0324 100644 --- a/ProjFS.Mac/PrjFSLib/PrjFSLib.cpp +++ b/ProjFS.Mac/PrjFSLib/PrjFSLib.cpp @@ -450,6 +450,20 @@ static void HandleKernelRequest(Message request, void* messageMemory) PrjFS_NotificationType_PreDelete); break; } + + case MessageType_KtoU_NotifyFileCreated: + { + char fullPath[PrjFSMaxPath]; + CombinePaths(s_virtualizationRootFullPath.c_str(), request.path, fullPath); + SetBitInFileFlags(fullPath, FileFlags_IsInVirtualizationRoot, true); + + result = HandleFileNotification( + requestHeader, + request.path, + false, // isDirectory + PrjFS_NotificationType_NewFileCreated); + break; + } } // async callbacks are not yet implemented From 04a5ef916c41a14a687b56a545ce88c90411c211 Mon Sep 17 00:00:00 2001 From: William Baker Date: Mon, 20 Aug 2018 10:04:00 -0700 Subject: [PATCH 043/272] Update MirrorProvider --- .../MacFileSystemVirtualizer.cs | 8 +++++++- .../WindowsFileSystemVirtualizer.cs | 19 ++++++++++++++++++- 2 files changed, 25 insertions(+), 2 deletions(-) diff --git a/MirrorProvider/MirrorProvider.Mac/MacFileSystemVirtualizer.cs b/MirrorProvider/MirrorProvider.Mac/MacFileSystemVirtualizer.cs index 3e2567548..5f20521bb 100644 --- a/MirrorProvider/MirrorProvider.Mac/MacFileSystemVirtualizer.cs +++ b/MirrorProvider/MirrorProvider.Mac/MacFileSystemVirtualizer.cs @@ -21,7 +21,8 @@ public override bool TryStartVirtualizationInstance(Enlistment enlistment, out s this.virtualizationInstance.OnEnumerateDirectory = this.OnEnumerateDirectory; this.virtualizationInstance.OnGetFileStream = this.OnGetFileStream; this.virtualizationInstance.OnFileModified = this.OnFileModified; - this.virtualizationInstance.OnPreDelete = this.OnPreDelete; + this.virtualizationInstance.OnPreDelete = this.OnPreDelete; + this.virtualizationInstance.OnNewFileCreated = this.OnNewFileCreated; Result result = this.virtualizationInstance.StartVirtualizationInstance( enlistment.SrcRoot, @@ -157,6 +158,11 @@ private Result OnPreDelete(string relativePath, bool isDirectory) { Console.WriteLine($"OnPreDelete (isDirectory: {isDirectory}): {relativePath}"); return Result.Success; + } + + private void OnNewFileCreated(string relativePath, bool isDirectory) + { + Console.WriteLine($"OnNewFileCreated (isDirectory: {isDirectory}): {relativePath}"); } private static byte[] ToVersionIdByteArray(byte version) diff --git a/MirrorProvider/MirrorProvider.Windows/WindowsFileSystemVirtualizer.cs b/MirrorProvider/MirrorProvider.Windows/WindowsFileSystemVirtualizer.cs index 46956a836..cecea9dce 100644 --- a/MirrorProvider/MirrorProvider.Windows/WindowsFileSystemVirtualizer.cs +++ b/MirrorProvider/MirrorProvider.Windows/WindowsFileSystemVirtualizer.cs @@ -35,13 +35,18 @@ public override bool TryStartVirtualizationInstance(Enlistment enlistment, out s this.virtualizationInstance.OnGetPlaceholderInformation = this.GetPlaceholderInformation; this.virtualizationInstance.OnGetFileStream = this.GetFileStream; this.virtualizationInstance.OnNotifyPreDelete = this.OnPreDelete; + this.virtualizationInstance.OnNotifyNewFileCreated = this.OnNewFileCreated; this.virtualizationInstance.OnNotifyFileHandleClosedFileModifiedOrDeleted = this.OnFileModifiedOrDeleted; uint threadCount = (uint)Environment.ProcessorCount * 2; NotificationMapping[] notificationMappings = new NotificationMapping[] { - new NotificationMapping(NotificationType.PreDelete | NotificationType.FileHandleClosedFileModified, string.Empty), + new NotificationMapping( + NotificationType.NewFileCreated | + NotificationType.PreDelete | + NotificationType.FileHandleClosedFileModified, + string.Empty), }; HResult result = this.virtualizationInstance.StartVirtualizationInstance( @@ -290,6 +295,18 @@ private HResult OnPreDelete(string relativePath, bool isDirectory) return HResult.Ok; } + private void OnNewFileCreated( + string relativePath, + bool isDirectory, + uint desiredAccess, + uint shareMode, + uint createDisposition, + uint createOptions, + ref NotificationType notificationMask) + { + Console.WriteLine($"OnNewFileCreated (isDirectory: {isDirectory}): {relativePath}"); + } + private void OnFileModifiedOrDeleted(string relativePath, bool isDirectory, bool isFileModified, bool isFileDeleted) { // To keep WindowsFileSystemVirtualizer in sync with MacFileSystemVirtualizer we're only registering for From 7b16fc66e176d79edbd469f084ad0038d0e80bae Mon Sep 17 00:00:00 2001 From: Kyle Rader Date: Fri, 17 Aug 2018 10:13:22 -0700 Subject: [PATCH 044/272] Enable runFastFetch to expect an error --- GVFS/GVFS.FunctionalTests/Tests/FastFetchTests.cs | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/GVFS/GVFS.FunctionalTests/Tests/FastFetchTests.cs b/GVFS/GVFS.FunctionalTests/Tests/FastFetchTests.cs index 69e1bdc5c..562be2e1c 100644 --- a/GVFS/GVFS.FunctionalTests/Tests/FastFetchTests.cs +++ b/GVFS/GVFS.FunctionalTests/Tests/FastFetchTests.cs @@ -142,7 +142,7 @@ public void ForceCheckoutRequiresCheckout() this.RunFastFetch($"--checkout --folders \"/Scripts\" -b {Settings.Default.Commitish}"); // Run a second time in the same repo on the same branch with more folders. - string result = this.RunFastFetch($"--force-checkout --folders \"/GVFS;/Scripts\" -b {Settings.Default.Commitish}"); + string result = this.RunFastFetch($"--force-checkout --folders \"/GVFS;/Scripts\" -b {Settings.Default.Commitish}", expectError: true); string[] expectedResults = new string[] { "Cannot use --force-checkout option without --checkout option." }; result.ShouldContain(expectedResults); @@ -471,7 +471,7 @@ private string GetRefTreeSha(string refName) return headTreeSha; } - private string RunFastFetch(string args) + private string RunFastFetch(string args, bool expectError = false) { args = args + " --verbose"; @@ -491,9 +491,12 @@ private string RunFastFetch(string args) processInfo.RedirectStandardError = true; ProcessResult result = ProcessHelper.Run(processInfo); - result.Output.Contains("Error").ShouldEqual(false, result.Output); - result.Errors.ShouldBeEmpty(result.Errors); - result.ExitCode.ShouldEqual(0); + result.Output.Contains("Error").ShouldEqual(expectError, result.Output); + if (!expectError) + { + result.Errors.ShouldBeEmpty(result.Errors); + result.ExitCode.ShouldEqual(0); + } return result.Output; } From 096a1b928262c3114d0e0d91c576d65f5ff84d01 Mon Sep 17 00:00:00 2001 From: Kyle Rader Date: Fri, 17 Aug 2018 10:30:37 -0700 Subject: [PATCH 045/272] Fix style --- GVFS/GVFS.FunctionalTests/Tests/FastFetchTests.cs | 1 + 1 file changed, 1 insertion(+) diff --git a/GVFS/GVFS.FunctionalTests/Tests/FastFetchTests.cs b/GVFS/GVFS.FunctionalTests/Tests/FastFetchTests.cs index 562be2e1c..0e3459ef7 100644 --- a/GVFS/GVFS.FunctionalTests/Tests/FastFetchTests.cs +++ b/GVFS/GVFS.FunctionalTests/Tests/FastFetchTests.cs @@ -497,6 +497,7 @@ private string RunFastFetch(string args, bool expectError = false) result.Errors.ShouldBeEmpty(result.Errors); result.ExitCode.ShouldEqual(0); } + return result.Output; } From cd4814ab2aba662aee1cd3855bea4766b8023549 Mon Sep 17 00:00:00 2001 From: Kyle Rader Date: Fri, 17 Aug 2018 11:31:23 -0700 Subject: [PATCH 046/272] Add expect error assertions --- GVFS/GVFS.FunctionalTests/Tests/FastFetchTests.cs | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/GVFS/GVFS.FunctionalTests/Tests/FastFetchTests.cs b/GVFS/GVFS.FunctionalTests/Tests/FastFetchTests.cs index 0e3459ef7..bed379738 100644 --- a/GVFS/GVFS.FunctionalTests/Tests/FastFetchTests.cs +++ b/GVFS/GVFS.FunctionalTests/Tests/FastFetchTests.cs @@ -492,7 +492,12 @@ private string RunFastFetch(string args, bool expectError = false) ProcessResult result = ProcessHelper.Run(processInfo); result.Output.Contains("Error").ShouldEqual(expectError, result.Output); - if (!expectError) + if (expectError) + { + Assert.IsTrue(result.Errors.Length > 0); + result.ExitCode.ShouldBeAtLeast(1, $"Exit code should be a failure (> 0) but was {result.ExitCode}"); + } + else { result.Errors.ShouldBeEmpty(result.Errors); result.ExitCode.ShouldEqual(0); From b0417de215c5a412c0a43492a33af1bdec56f7e8 Mon Sep 17 00:00:00 2001 From: Kyle Rader Date: Fri, 17 Aug 2018 11:51:03 -0700 Subject: [PATCH 047/272] Use ShouldBe... syntax --- GVFS/GVFS.FunctionalTests/Tests/FastFetchTests.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/GVFS/GVFS.FunctionalTests/Tests/FastFetchTests.cs b/GVFS/GVFS.FunctionalTests/Tests/FastFetchTests.cs index bed379738..6b2daa3b0 100644 --- a/GVFS/GVFS.FunctionalTests/Tests/FastFetchTests.cs +++ b/GVFS/GVFS.FunctionalTests/Tests/FastFetchTests.cs @@ -494,7 +494,7 @@ private string RunFastFetch(string args, bool expectError = false) result.Output.Contains("Error").ShouldEqual(expectError, result.Output); if (expectError) { - Assert.IsTrue(result.Errors.Length > 0); + result.Errors.Length.ShouldBeAtLeast(1, "Expected at lest 1 error"); result.ExitCode.ShouldBeAtLeast(1, $"Exit code should be a failure (> 0) but was {result.ExitCode}"); } else From f33a734af08e1025c02aa76a4e22900f9c358434 Mon Sep 17 00:00:00 2001 From: Kyle Rader Date: Mon, 20 Aug 2018 08:45:13 -0700 Subject: [PATCH 048/272] Remove extra assertion for expected error --- GVFS/GVFS.FunctionalTests/Tests/FastFetchTests.cs | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/GVFS/GVFS.FunctionalTests/Tests/FastFetchTests.cs b/GVFS/GVFS.FunctionalTests/Tests/FastFetchTests.cs index 6b2daa3b0..2078ace42 100644 --- a/GVFS/GVFS.FunctionalTests/Tests/FastFetchTests.cs +++ b/GVFS/GVFS.FunctionalTests/Tests/FastFetchTests.cs @@ -141,11 +141,8 @@ public void ForceCheckoutRequiresCheckout() { this.RunFastFetch($"--checkout --folders \"/Scripts\" -b {Settings.Default.Commitish}"); - // Run a second time in the same repo on the same branch with more folders. + // Run a second time in the same repo on the same branch with more folders but expect an error. string result = this.RunFastFetch($"--force-checkout --folders \"/GVFS;/Scripts\" -b {Settings.Default.Commitish}", expectError: true); - - string[] expectedResults = new string[] { "Cannot use --force-checkout option without --checkout option." }; - result.ShouldContain(expectedResults); } [TestCase] From e514b3be6721304fea7f9b916b4902b150a6c796 Mon Sep 17 00:00:00 2001 From: Kyle Rader Date: Mon, 20 Aug 2018 10:59:54 -0700 Subject: [PATCH 049/272] Remove assertions hidden within runFastFetch method --- .../Tests/FastFetchTests.cs | 30 +++++++------------ 1 file changed, 11 insertions(+), 19 deletions(-) diff --git a/GVFS/GVFS.FunctionalTests/Tests/FastFetchTests.cs b/GVFS/GVFS.FunctionalTests/Tests/FastFetchTests.cs index 2078ace42..54db8a59a 100644 --- a/GVFS/GVFS.FunctionalTests/Tests/FastFetchTests.cs +++ b/GVFS/GVFS.FunctionalTests/Tests/FastFetchTests.cs @@ -142,7 +142,10 @@ public void ForceCheckoutRequiresCheckout() this.RunFastFetch($"--checkout --folders \"/Scripts\" -b {Settings.Default.Commitish}"); // Run a second time in the same repo on the same branch with more folders but expect an error. - string result = this.RunFastFetch($"--force-checkout --folders \"/GVFS;/Scripts\" -b {Settings.Default.Commitish}", expectError: true); + ProcessResult result = this.RunFastFetch($"--force-checkout --folders \"/GVFS;/Scripts\" -b {Settings.Default.Commitish}"); + + string[] expectedResults = new string[] { "Cannot use --force-checkout option without --checkout option." }; + result.Output.ShouldContain(expectedResults); } [TestCase] @@ -230,8 +233,8 @@ public void CanUpdateIndex(int indexVersion, bool indexSigningOff) // Reset the index and use fastfetch to update the index. Compare against 'git status' baseline. GitProcess.Invoke(this.fastFetchRepoRoot, $"-c index.version= {indexVersion} read-tree HEAD"); - string fastfetchoutput = this.RunFastFetch("--checkout --Allow-index-metadata-update-from-working-tree"); - Trace.WriteLine(fastfetchoutput); // Written to log file for manual investigation + ProcessResult fastFetchResult = this.RunFastFetch("--checkout --Allow-index-metadata-update-from-working-tree"); + Trace.WriteLine(fastFetchResult.Output); // Written to log file for manual investigation string lsfilesAfterUpdate = GitProcess.Invoke(this.fastFetchRepoRoot, "ls-files --debug"); lsfilesAfterUpdate.ShouldEqual(lsfilesAfterStatus, "git status and fastfetch didn't result in the same index"); @@ -258,7 +261,7 @@ public void IncrementalChangesLeaveGoodStatus() string status = GitProcess.Invoke(this.fastFetchRepoRoot, "status --porcelain"); status.ShouldBeEmpty("Status shows unexpected files changed"); - string output = this.RunFastFetch($"--checkout -c {UpdateCommit}"); + this.RunFastFetch($"--checkout -c {UpdateCommit}"); status = GitProcess.Invoke(this.fastFetchRepoRoot, "status --porcelain"); status.ShouldBeEmpty("Status shows unexpected files changed"); @@ -295,8 +298,8 @@ public void CanDetectAlreadyUpToDate() this.RunFastFetch("--checkout -b " + Settings.Default.Commitish); this.CurrentBranchShouldEqual(Settings.Default.Commitish); - this.RunFastFetch(" -b " + Settings.Default.Commitish).ShouldContain("\"TotalMissingObjects\":0"); - this.RunFastFetch("--checkout -b " + Settings.Default.Commitish).ShouldContain("\"RequiredBlobsCount\":0"); + this.RunFastFetch(" -b " + Settings.Default.Commitish).Output.ShouldContain("\"TotalMissingObjects\":0"); + this.RunFastFetch("--checkout -b " + Settings.Default.Commitish).Output.ShouldContain("\"RequiredBlobsCount\":0"); this.CurrentBranchShouldEqual(Settings.Default.Commitish); this.fastFetchRepoRoot.ShouldBeADirectory(FileSystemRunner.DefaultRunner) @@ -468,7 +471,7 @@ private string GetRefTreeSha(string refName) return headTreeSha; } - private string RunFastFetch(string args, bool expectError = false) + private ProcessResult RunFastFetch(string args) { args = args + " --verbose"; @@ -488,19 +491,8 @@ private string RunFastFetch(string args, bool expectError = false) processInfo.RedirectStandardError = true; ProcessResult result = ProcessHelper.Run(processInfo); - result.Output.Contains("Error").ShouldEqual(expectError, result.Output); - if (expectError) - { - result.Errors.Length.ShouldBeAtLeast(1, "Expected at lest 1 error"); - result.ExitCode.ShouldBeAtLeast(1, $"Exit code should be a failure (> 0) but was {result.ExitCode}"); - } - else - { - result.Errors.ShouldBeEmpty(result.Errors); - result.ExitCode.ShouldEqual(0); - } - return result.Output; + return result; } private string GetShaFromLsLine(string line) From 120eeba149a78de3ed616dc5db3e11efc974b48a Mon Sep 17 00:00:00 2001 From: Jameson Miller Date: Mon, 20 Aug 2018 16:05:47 -0400 Subject: [PATCH 050/272] Update name of file created by installer to indicate installation version Update name of file that GVFS installer creates when an on disk version 16 capable installation is first installed. --- GVFS/GVFS.Common/GVFSConstants.cs | 5 +++ GVFS/GVFS.Installer/Setup.iss | 14 ++++---- GVFS/GVFS.Service/GvfsService.cs | 53 ++++++++++++++----------------- 3 files changed, 35 insertions(+), 37 deletions(-) diff --git a/GVFS/GVFS.Common/GVFSConstants.cs b/GVFS/GVFS.Common/GVFSConstants.cs index f498773f1..ece894ef8 100644 --- a/GVFS/GVFS.Common/GVFSConstants.cs +++ b/GVFS/GVFS.Common/GVFSConstants.cs @@ -192,6 +192,11 @@ public static class Heads } } + public static class InstallationCapabilityFiles + { + public const string OnDiskVersion16CapableInstallation = "OnDiskVersion16CapableInstallation.dat"; + } + public static class VerbParameters { public static class Mount diff --git a/GVFS/GVFS.Installer/Setup.iss b/GVFS/GVFS.Installer/Setup.iss index 36bff4d3c..f2875b699 100644 --- a/GVFS/GVFS.Installer/Setup.iss +++ b/GVFS/GVFS.Installer/Setup.iss @@ -283,15 +283,15 @@ begin end; end; -procedure WriteGitStatusCacheAvailableFile(); +procedure WriteOnDiskVersion16CapableFile(); var - TokenFilePath: string; + FilePath: string; begin - TokenFilePath := ExpandConstant('{app}\GitStatusCacheAvailable'); - if not FileExists(TokenFilePath) then + FilePath := ExpandConstant('{app}\OnDiskVersion16CapableInstallation.dat'); + if not FileExists(FilePath) then begin - Log('WritingGitStatusCacheAvailableFile: Writing file ' + TokenFilePath); - SaveStringToFile(TokenFilePath, '', False); + Log('WriteOnDiskVersion16CapableFile: Writing file ' + FilePath); + SaveStringToFile(FilePath, '', False); end end; @@ -319,7 +319,7 @@ begin end; end; - WriteGitStatusCacheAvailableFile(); + WriteOnDiskVersion16CapableFile(); finally WizardForm.StatusLabel.Caption := StatusText; WizardForm.ProgressGauge.Style := npbstNormal; diff --git a/GVFS/GVFS.Service/GvfsService.cs b/GVFS/GVFS.Service/GvfsService.cs index bacab0457..0134cd244 100644 --- a/GVFS/GVFS.Service/GvfsService.cs +++ b/GVFS/GVFS.Service/GvfsService.cs @@ -287,28 +287,37 @@ private void CheckEnableGitStatusCacheTokenFile() try { string statusCacheVersionTokenPath = Path.Combine(Paths.GetServiceDataRoot(GVFSConstants.Service.ServiceName), GVFSConstants.GitStatusCache.EnableGitStatusCacheTokenFile); - - if (!File.Exists(statusCacheVersionTokenPath)) + if (File.Exists(statusCacheVersionTokenPath)) { - DateTime lastRebootTime = NativeMethods.GetLastRebootTime(); + this.tracer.RelatedInfo($"CheckEnableGitStatusCache: EnableGitStatusCacheToken file already exists at {statusCacheVersionTokenPath}."); + return; + } + + DateTime lastRebootTime = NativeMethods.GetLastRebootTime(); + + // GitStatusCache was included with GVFS on disk version 16. The 1st time GVFS that is at or above on disk version + // is installed, it will write out a file indicating that the installation is "OnDiskVersion16Capable". + // We can query the properties of this file to get the installation time, and compare this with the last reboot time for + // this machine. + string fileToCheck = Path.Combine(Configuration.AssemblyPath, GVFSConstants.InstallationCapabilityFiles.OnDiskVersion16CapableInstallation); - // When a version of GVFS that supports the GitStatusCache is installed, it will create - // the following file. By checking the time the file was created, we know when that - // version of GVFS was installed. - string fileToCheck = Path.Combine(Configuration.AssemblyPath, "GitStatusCacheAvailable"); - if (File.Exists(fileToCheck)) + if (File.Exists(fileToCheck)) + { + DateTime installTime = File.GetCreationTime(fileToCheck); + if (lastRebootTime > installTime) { - DateTime installTime = File.GetCreationTime(fileToCheck); - if (lastRebootTime > installTime) - { - File.WriteAllText(statusCacheVersionTokenPath, string.Empty); - } + this.tracer.RelatedInfo($"CheckEnableGitStatusCache: Writing out EnableGitStatusCacheToken file. GVFS installation time: {installTime}, last Reboot time: {lastRebootTime}."); + File.WriteAllText(statusCacheVersionTokenPath, string.Empty); } else { - this.tracer.RelatedError($"Unable to determine GVFS installation time: {fileToCheck} does not exist."); + this.tracer.RelatedInfo($"CheckEnableGitStatusCache: Not writing EnableGitStatusCacheToken file - machine has not been rebooted since OnDiskVersion16Capable installation. GVFS installation time: {installTime}, last reboot time: {lastRebootTime}"); } } + else + { + this.tracer.RelatedError($"Unable to determine GVFS installation time: {fileToCheck} does not exist."); + } } catch (Exception ex) { @@ -318,22 +327,6 @@ private void CheckEnableGitStatusCacheTokenFile() } } - private bool TryGetGVFSInstallTime(out DateTime installTime) - { - installTime = DateTime.Now; - - // Get the time of a file that was created by the GVFS installer (for a version of GVFS that supports the - // GitStatusCache). The expected path is written by the installer. - string fileToCheck = Path.Combine(Configuration.AssemblyPath, "GitStatusCacheAvailable"); - if (File.Exists(fileToCheck)) - { - installTime = File.GetCreationTime(fileToCheck); - return true; - } - - return false; - } - private void LogExceptionAndExit(Exception e, string method) { EventMetadata metadata = new EventMetadata(); From 68f6f69b3b1c399fff4ea5d3607bef9da79849f9 Mon Sep 17 00:00:00 2001 From: William Baker Date: Tue, 21 Aug 2018 11:55:43 -0700 Subject: [PATCH 051/272] PR Feeback: Remove HasAncestorFlaggedAsInRoot and add TODO --- .../PrjFSKext/PrjFSKext/KauthHandler.cpp | 36 ------------------- ProjFS.Mac/PrjFSLib/PrjFSLib.cpp | 2 ++ 2 files changed, 2 insertions(+), 36 deletions(-) diff --git a/ProjFS.Mac/PrjFSKext/PrjFSKext/KauthHandler.cpp b/ProjFS.Mac/PrjFSKext/PrjFSKext/KauthHandler.cpp index db177bb4a..7dc73f78f 100644 --- a/ProjFS.Mac/PrjFSKext/PrjFSKext/KauthHandler.cpp +++ b/ProjFS.Mac/PrjFSKext/PrjFSKext/KauthHandler.cpp @@ -35,7 +35,6 @@ static int HandleFileOpOperation( static int GetPid(vfs_context_t context); static uint32_t ReadVNodeFileFlags(vnode_t vn, vfs_context_t context); -static inline bool HasAncestorFlaggedAsInRoot(vnode_t vnode, vfs_context_t context); static inline bool FileFlagsBitIsSet(uint32_t fileFlags, uint32_t bit); static inline bool FileIsFlaggedAsInRoot(vnode_t vnode, vfs_context_t context); static inline bool ActionBitIsSet(kauth_action_t action, kauth_action_t mask); @@ -400,11 +399,6 @@ static int HandleFileOpOperation( { goto CleanupAndReturn; } - - if (!fileFlaggedInRoot && !HasAncestorFlaggedAsInRoot(currentVnode, context)) - { - goto CleanupAndReturn; - } VirtualizationRoot* root = nullptr; int pid; @@ -566,7 +560,6 @@ static bool ShouldHandleFileOpEvent( *root = VirtualizationRoots_FindForVnode(vnode); if (nullptr == *root) { - KextLog_FileNote(vnode, "ShouldHandleFileOpEvent(%d): No virtualization root found for file with set flag.", action); return false; } else if (nullptr == (*root)->providerUserClient) @@ -735,35 +728,6 @@ static uint32_t ReadVNodeFileFlags(vnode_t vn, vfs_context_t context) return attributes.va_flags; } -static bool HasAncestorFlaggedAsInRoot(vnode_t vnode, vfs_context_t context) -{ - vnode_get(vnode); - - bool inRoot = false; - - // Search up the tree until we hit a folder know to be inside a virtualization root - // or the root of the file system - while (NULLVP != vnode && !vnode_isvroot(vnode)) - { - uint32_t vnodeFileFlags = ReadVNodeFileFlags(vnode, context); - if (FileFlagsBitIsSet(vnodeFileFlags, FileFlags_IsInVirtualizationRoot)) - { - inRoot = true; - break; - } - vnode_t parent = vnode_getparent(vnode); - vnode_put(vnode); - vnode = parent; - } - - if (NULLVP != vnode) - { - vnode_put(vnode); - } - - return inRoot; -} - static inline bool FileFlagsBitIsSet(uint32_t fileFlags, uint32_t bit) { // Note: if multiple bits are set in 'bit', this will return true if ANY are set in fileFlags diff --git a/ProjFS.Mac/PrjFSLib/PrjFSLib.cpp b/ProjFS.Mac/PrjFSLib/PrjFSLib.cpp index 9956c0324..48c602ced 100644 --- a/ProjFS.Mac/PrjFSLib/PrjFSLib.cpp +++ b/ProjFS.Mac/PrjFSLib/PrjFSLib.cpp @@ -455,6 +455,8 @@ static void HandleKernelRequest(Message request, void* messageMemory) { char fullPath[PrjFSMaxPath]; CombinePaths(s_virtualizationRootFullPath.c_str(), request.path, fullPath); + + // TODO(Mac): Handle SetBitInFileFlags failures SetBitInFileFlags(fullPath, FileFlags_IsInVirtualizationRoot, true); result = HandleFileNotification( From 72033a856659e1d086f7712e0552079ccc23f7cb Mon Sep 17 00:00:00 2001 From: William Baker Date: Tue, 21 Aug 2018 11:57:46 -0700 Subject: [PATCH 052/272] Convert tabs to spaces --- ProjFS.Mac/PrjFSLib/PrjFSLib.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ProjFS.Mac/PrjFSLib/PrjFSLib.cpp b/ProjFS.Mac/PrjFSLib/PrjFSLib.cpp index 48c602ced..5e18ffec9 100644 --- a/ProjFS.Mac/PrjFSLib/PrjFSLib.cpp +++ b/ProjFS.Mac/PrjFSLib/PrjFSLib.cpp @@ -456,7 +456,7 @@ static void HandleKernelRequest(Message request, void* messageMemory) char fullPath[PrjFSMaxPath]; CombinePaths(s_virtualizationRootFullPath.c_str(), request.path, fullPath); - // TODO(Mac): Handle SetBitInFileFlags failures + // TODO(Mac): Handle SetBitInFileFlags failures SetBitInFileFlags(fullPath, FileFlags_IsInVirtualizationRoot, true); result = HandleFileNotification( From 9088c0466efd6d78186938802248fc9b97b62e44 Mon Sep 17 00:00:00 2001 From: William Baker Date: Tue, 14 Aug 2018 08:13:34 -0700 Subject: [PATCH 053/272] Mac: Add file renamed notification --- .../EnlistmentPerFixture/GitFilesTests.cs | 20 +++--- .../GitMoveRenameTests.cs | 31 ++++++--- .../MoveRenameFileTests.cs | 12 ++-- .../MoveRenameFileTests_2.cs | 20 +++--- .../MacFileSystemVirtualizer.cs | 19 ++++-- .../WindowsFileSystemVirtualizer.cs | 30 +-------- .../FileSystem/FileSystemVirtualizer.cs | 33 ++++++++++ .../PrjFSKext/PrjFSKext/KauthHandler.cpp | 65 ++++++++++++++++++- .../PrjFSKext/VirtualizationRoots.hpp | 2 +- ProjFS.Mac/PrjFSKext/public/Message.h | 2 + .../PrjFSLib.Mac.Managed/CallbackDelegates.cs | 4 +- .../VirtualizationInstance.cs | 12 ++-- ProjFS.Mac/PrjFSLib/PrjFSLib.cpp | 22 ++++--- 13 files changed, 187 insertions(+), 85 deletions(-) diff --git a/GVFS/GVFS.FunctionalTests/Tests/EnlistmentPerFixture/GitFilesTests.cs b/GVFS/GVFS.FunctionalTests/Tests/EnlistmentPerFixture/GitFilesTests.cs index c2fac6926..1f8d91d38 100644 --- a/GVFS/GVFS.FunctionalTests/Tests/EnlistmentPerFixture/GitFilesTests.cs +++ b/GVFS/GVFS.FunctionalTests/Tests/EnlistmentPerFixture/GitFilesTests.cs @@ -187,18 +187,17 @@ public void ModifiedFileWillGetAddedToModifiedPathsFile() } [TestCase, Order(8)] - [Category(Categories.Mac.M2TODO)] - public void RenamedFileAddedToSparseCheckoutAndSkipWorktreeBitCleared() + public void RenamedFileAddedToModifiedPathsFile() { + GitMoveRenameTests.IgnoreSystemIOMoveOnMac(this.fileSystem); + string fileToRenameEntry = "Test_EPF_MoveRenameFileTests/ChangeUnhydratedFileName/Program.cs"; string fileToRenameTargetEntry = "Test_EPF_MoveRenameFileTests/ChangeUnhydratedFileName/Program2.cs"; - string fileToRenameRelativePath = "Test_EPF_MoveRenameFileTests\\ChangeUnhydratedFileName\\Program.cs"; - string fileToRenameTargetRelativePath = "Test_EPF_MoveRenameFileTests\\ChangeUnhydratedFileName\\Program2.cs"; this.VerifyWorktreeBit(fileToRenameEntry, LsFilesStatus.SkipWorktree); this.fileSystem.MoveFile( - this.Enlistment.GetVirtualPathTo(fileToRenameRelativePath), - this.Enlistment.GetVirtualPathTo(fileToRenameTargetRelativePath)); + this.Enlistment.GetVirtualPathTo(fileToRenameEntry), + this.Enlistment.GetVirtualPathTo(fileToRenameTargetEntry)); this.Enlistment.WaitForBackgroundOperations().ShouldEqual(true, "Background operations failed to complete."); GVFSHelpers.ModifiedPathsShouldContain(this.fileSystem, this.Enlistment.DotGVFSRoot, fileToRenameEntry); @@ -209,19 +208,16 @@ public void RenamedFileAddedToSparseCheckoutAndSkipWorktreeBitCleared() } [TestCase, Order(9)] - [Category(Categories.Mac.M2TODO)] - public void RenamedFileAndOverwrittenTargetAddedToSparseCheckoutAndSkipWorktreeBitCleared() + public void RenamedFileAndOverwrittenTargetAddedToModifiedPathsFile() { string fileToRenameEntry = "Test_EPF_MoveRenameFileTests_2/MoveUnhydratedFileToOverwriteUnhydratedFileAndWrite/RunUnitTests.bat"; string fileToRenameTargetEntry = "Test_EPF_MoveRenameFileTests_2/MoveUnhydratedFileToOverwriteUnhydratedFileAndWrite/RunFunctionalTests.bat"; - string fileToRenameRelativePath = "Test_EPF_MoveRenameFileTests_2\\MoveUnhydratedFileToOverwriteUnhydratedFileAndWrite\\RunUnitTests.bat"; - string fileToRenameTargetRelativePath = "Test_EPF_MoveRenameFileTests_2\\MoveUnhydratedFileToOverwriteUnhydratedFileAndWrite\\RunFunctionalTests.bat"; this.VerifyWorktreeBit(fileToRenameEntry, LsFilesStatus.SkipWorktree); this.VerifyWorktreeBit(fileToRenameTargetEntry, LsFilesStatus.SkipWorktree); this.fileSystem.ReplaceFile( - this.Enlistment.GetVirtualPathTo(fileToRenameRelativePath), - this.Enlistment.GetVirtualPathTo(fileToRenameTargetRelativePath)); + this.Enlistment.GetVirtualPathTo(fileToRenameEntry), + this.Enlistment.GetVirtualPathTo(fileToRenameTargetEntry)); this.Enlistment.WaitForBackgroundOperations().ShouldEqual(true, "Background operations failed to complete."); GVFSHelpers.ModifiedPathsShouldContain(this.fileSystem, this.Enlistment.DotGVFSRoot, fileToRenameEntry); diff --git a/GVFS/GVFS.FunctionalTests/Tests/EnlistmentPerFixture/GitMoveRenameTests.cs b/GVFS/GVFS.FunctionalTests/Tests/EnlistmentPerFixture/GitMoveRenameTests.cs index d0b1a09e4..30f1e4ae3 100644 --- a/GVFS/GVFS.FunctionalTests/Tests/EnlistmentPerFixture/GitMoveRenameTests.cs +++ b/GVFS/GVFS.FunctionalTests/Tests/EnlistmentPerFixture/GitMoveRenameTests.cs @@ -6,6 +6,7 @@ using System; using System.Collections.Generic; using System.IO; +using System.Runtime.InteropServices; namespace GVFS.FunctionalTests.Tests.EnlistmentPerFixture { @@ -21,6 +22,15 @@ public GitMoveRenameTests(FileSystemRunner fileSystem) this.fileSystem = fileSystem; } + public static void IgnoreSystemIOMoveOnMac(FileSystemRunner runner) + { + if (runner is SystemIORunner && + RuntimeInformation.IsOSPlatform(OSPlatform.OSX)) + { + Assert.Ignore("TODO(Mac): SystemIORunner rename tests require hardlink notifications"); + } + } + [TestCase, Order(1)] public void GitStatus() { @@ -37,6 +47,7 @@ public void GitStatusAfterNewFile() string filename = "new.cs"; string filePath = this.Enlistment.GetVirtualPathTo(filename); + filePath.ShouldNotExistOnDisk(this.fileSystem); this.fileSystem.WriteAllText(filePath, this.testFileContents); filePath.ShouldBeAFile(this.fileSystem).WithContents(this.testFileContents); @@ -52,14 +63,14 @@ public void GitStatusAfterNewFile() } [TestCase, Order(3)] - [Category(Categories.Mac.M2TODO)] public void GitStatusAfterFileNameCaseChange() { string oldFilename = "new.cs"; this.EnsureTestFileExists(oldFilename); string newFilename = "New.cs"; - this.fileSystem.MoveFile(this.Enlistment.GetVirtualPathTo(oldFilename), this.Enlistment.GetVirtualPathTo(newFilename)); + string newFilePath = this.Enlistment.GetVirtualPathTo(newFilename); + this.fileSystem.MoveFile(this.Enlistment.GetVirtualPathTo(oldFilename), newFilePath); GitHelpers.CheckGitCommandAgainstGVFSRepo( this.Enlistment.RepoRoot, @@ -67,17 +78,21 @@ public void GitStatusAfterFileNameCaseChange() "On branch " + Properties.Settings.Default.Commitish, "Untracked files:", newFilename); + + this.fileSystem.DeleteFile(newFilePath); } [TestCase, Order(4)] - [Category(Categories.Mac.M2TODO)] public void GitStatusAfterFileRename() { + IgnoreSystemIOMoveOnMac(this.fileSystem); + string oldFilename = "New.cs"; - this.Enlistment.GetVirtualPathTo(oldFilename).ShouldBeAFile(this.fileSystem); + this.EnsureTestFileExists(oldFilename); string newFilename = "test.cs"; - this.fileSystem.MoveFile(this.Enlistment.GetVirtualPathTo(oldFilename), this.Enlistment.GetVirtualPathTo(newFilename)); + string newFilePath = this.Enlistment.GetVirtualPathTo(newFilename); + this.fileSystem.MoveFile(this.Enlistment.GetVirtualPathTo(oldFilename), newFilePath); GitHelpers.CheckGitCommandAgainstGVFSRepo( this.Enlistment.RepoRoot, @@ -178,9 +193,10 @@ public void GitWithEnvironmentVariables() } [TestCase, Order(9)] - [Category(Categories.Mac.M2TODO)] public void GitStatusAfterRenameFileIntoRepo() { + IgnoreSystemIOMoveOnMac(this.fileSystem); + string filename = "GitStatusAfterRenameFileIntoRepo.cs"; // Create the test file in this.Enlistment.EnlistmentRoot as it's outside of src @@ -190,7 +206,7 @@ public void GitStatusAfterRenameFileIntoRepo() this.fileSystem.WriteAllText(filePath, this.testFileContents); filePath.ShouldBeAFile(this.fileSystem).WithContents(this.testFileContents); - string renamedFileName = "GVFlt_MoveFileTest\\GitStatusAfterRenameFileIntoRepo.cs"; + string renamedFileName = Path.Combine("GVFlt_MoveFileTest", "GitStatusAfterRenameFileIntoRepo.cs"); this.fileSystem.MoveFile(filePath, this.Enlistment.GetVirtualPathTo(renamedFileName)); this.Enlistment.GetVirtualPathTo(filePath).ShouldNotExistOnDisk(this.fileSystem); @@ -219,7 +235,6 @@ public void GitStatusAfterRenameFileOutOfRepo() } [TestCase, Order(11)] - [Category(Categories.Mac.M2TODO)] public void GitStatusAfterRenameFolderIntoRepo() { string folderName = "GitStatusAfterRenameFolderIntoRepo"; diff --git a/GVFS/GVFS.FunctionalTests/Tests/EnlistmentPerFixture/MoveRenameFileTests.cs b/GVFS/GVFS.FunctionalTests/Tests/EnlistmentPerFixture/MoveRenameFileTests.cs index 5ea5b7ab0..43af93dc6 100644 --- a/GVFS/GVFS.FunctionalTests/Tests/EnlistmentPerFixture/MoveRenameFileTests.cs +++ b/GVFS/GVFS.FunctionalTests/Tests/EnlistmentPerFixture/MoveRenameFileTests.cs @@ -10,7 +10,7 @@ namespace GVFS.FunctionalTests.Tests.EnlistmentPerFixture // TODO 452590 - Combine all of the MoveRenameTests into a single fixture, and have each use different // well known files [TestFixtureSource(typeof(FileSystemRunner), FileSystemRunner.TestRunners)] - [Category(Categories.Mac.M2TODO)] + [Category(Categories.Mac.M2)] public class MoveRenameFileTests : TestsWithEnlistmentPerFixture { public const string TestFileContents = @@ -53,8 +53,8 @@ public MoveRenameFileTests(FileSystemRunner fileSystem) [TestCase] public void ChangeUnhydratedFileName() { - string oldFilename = "Test_EPF_MoveRenameFileTests\\ChangeUnhydratedFileName\\Program.cs"; - string newFilename = "Test_EPF_MoveRenameFileTests\\ChangeUnhydratedFileName\\renamed_Program.cs"; + string oldFilename = Path.Combine("Test_EPF_MoveRenameFileTests", "ChangeUnhydratedFileName", "Program.cs"); + string newFilename = Path.Combine("Test_EPF_MoveRenameFileTests", "ChangeUnhydratedFileName", "renamed_Program.cs"); // Don't read oldFilename or check for its existence before calling MoveFile, because doing so // can cause the file to hydrate @@ -86,7 +86,7 @@ public void ChangeNestedUnhydratedFileNameCase() { string oldName = "Program.cs"; string newName = "program.cs"; - string folderName = "Test_EPF_MoveRenameFileTests\\ChangeNestedUnhydratedFileNameCase\\"; + string folderName = Path.Combine("Test_EPF_MoveRenameFileTests", "ChangeNestedUnhydratedFileNameCase"); string oldVirtualPath = this.Enlistment.GetVirtualPathTo(Path.Combine(folderName, oldName)); string newVirtualPath = this.Enlistment.GetVirtualPathTo(Path.Combine(folderName, newName)); @@ -101,8 +101,8 @@ public void MoveUnhydratedFileToDotGitFolder() this.Enlistment.GetVirtualPathTo(targetFolderName).ShouldBeADirectory(this.fileSystem); string testFileName = "Program.cs"; - string testFileFolder = "Test_EPF_MoveRenameFileTests\\MoveUnhydratedFileToDotGitFolder"; - string testFilePathSubPath = testFileFolder + "\\" + testFileName; + string testFileFolder = Path.Combine("Test_EPF_MoveRenameFileTests", "MoveUnhydratedFileToDotGitFolder"); + string testFilePathSubPath = Path.Combine(testFileFolder, testFileName); string newTestFileVirtualPath = Path.Combine(this.Enlistment.GetVirtualPathTo(targetFolderName), testFileName); diff --git a/GVFS/GVFS.FunctionalTests/Tests/EnlistmentPerFixture/MoveRenameFileTests_2.cs b/GVFS/GVFS.FunctionalTests/Tests/EnlistmentPerFixture/MoveRenameFileTests_2.cs index 01c2fe908..746f22561 100644 --- a/GVFS/GVFS.FunctionalTests/Tests/EnlistmentPerFixture/MoveRenameFileTests_2.cs +++ b/GVFS/GVFS.FunctionalTests/Tests/EnlistmentPerFixture/MoveRenameFileTests_2.cs @@ -8,7 +8,7 @@ namespace GVFS.FunctionalTests.Tests.EnlistmentPerFixture // TODO 452590 - Combine all of the MoveRenameTests into a single fixture, and have each use different // well known files [TestFixtureSource(typeof(FileSystemRunner), FileSystemRunner.TestRunners)] - [Category(Categories.Mac.M2TODO)] + [Category(Categories.Mac.M2)] public class MoveRenameFileTests_2 : TestsWithEnlistmentPerFixture { private const string TestFileFolder = "Test_EPF_MoveRenameFileTests_2"; @@ -44,8 +44,8 @@ public void MoveUnhydratedFileToUnhydratedFolderAndWrite() // Assume there will always be a GVFS folder when running tests string testFolderName = "GVFS"; - string oldTestFileVirtualPath = this.Enlistment.GetVirtualPathTo(TestFileFolder + "\\" + testFileName); - string newTestFileVirtualPath = this.Enlistment.GetVirtualPathTo(testFolderName + "\\" + testFileName); + string oldTestFileVirtualPath = this.Enlistment.GetVirtualPathTo(TestFileFolder, testFileName); + string newTestFileVirtualPath = this.Enlistment.GetVirtualPathTo(testFolderName, testFileName); this.fileSystem.MoveFile(oldTestFileVirtualPath, newTestFileVirtualPath); oldTestFileVirtualPath.ShouldNotExistOnDisk(this.fileSystem); @@ -72,8 +72,8 @@ public void MoveUnhydratedFileToNewFolderAndWrite() string newTestFileVirtualPath = Path.Combine(this.Enlistment.GetVirtualPathTo(testFolderName), testFolderName); - this.fileSystem.MoveFile(this.Enlistment.GetVirtualPathTo(TestFileFolder + "\\" + testFileName), newTestFileVirtualPath); - this.Enlistment.GetVirtualPathTo(TestFileFolder + "\\" + testFileName).ShouldNotExistOnDisk(this.fileSystem); + this.fileSystem.MoveFile(this.Enlistment.GetVirtualPathTo(TestFileFolder, testFileName), newTestFileVirtualPath); + this.Enlistment.GetVirtualPathTo(TestFileFolder, testFileName).ShouldNotExistOnDisk(this.fileSystem); newTestFileVirtualPath.ShouldBeAFile(this.fileSystem).WithContents(testFileContents); // Writing after the move should succeed @@ -92,8 +92,8 @@ public void MoveUnhydratedFileToNewFolderAndWrite() [TestCase, Order(3)] public void MoveUnhydratedFileToOverwriteUnhydratedFileAndWrite() { - string targetFilename = TestFileFolder + "\\MoveUnhydratedFileToOverwriteUnhydratedFileAndWrite\\RunFunctionalTests.bat"; - string sourceFilename = TestFileFolder + "\\MoveUnhydratedFileToOverwriteUnhydratedFileAndWrite\\RunUnitTests.bat"; + string targetFilename = Path.Combine(TestFileFolder, "MoveUnhydratedFileToOverwriteUnhydratedFileAndWrite", "RunFunctionalTests.bat"); + string sourceFilename = Path.Combine(TestFileFolder, "MoveUnhydratedFileToOverwriteUnhydratedFileAndWrite", "RunUnitTests.bat"); string sourceFileContents = RunUnitTestsContents; // Overwriting one unhydrated file with another should create a file at the target @@ -130,7 +130,11 @@ public void MoveUnhydratedFileToOverwriteFullFileAndWrite() string targetFilename = "TargetFile.txt"; string targetFileContents = "The Target"; - string sourceFilename = TestFileFolder + "\\MoveUnhydratedFileToOverwriteFullFileAndWrite\\MoveUnhydratedFileToOverwriteFullFileAndWrite.txt"; + string sourceFilename = Path.Combine( + TestFileFolder, + "MoveUnhydratedFileToOverwriteFullFileAndWrite", + "MoveUnhydratedFileToOverwriteFullFileAndWrite.txt"); + string sourceFileContents = @" diff --git a/GVFS/GVFS.Platform.Mac/MacFileSystemVirtualizer.cs b/GVFS/GVFS.Platform.Mac/MacFileSystemVirtualizer.cs index 66a7a83c9..2e54d0284 100644 --- a/GVFS/GVFS.Platform.Mac/MacFileSystemVirtualizer.cs +++ b/GVFS/GVFS.Platform.Mac/MacFileSystemVirtualizer.cs @@ -102,6 +102,7 @@ protected override bool TryStart(out string error) this.virtualizationInstance.OnFileModified = this.OnFileModified; this.virtualizationInstance.OnPreDelete = this.OnPreDelete; this.virtualizationInstance.OnNewFileCreated = this.OnNewFileCreated; + this.virtualizationInstance.OnFileRenamed = this.OnFileRenamed; uint threadCount = (uint)Environment.ProcessorCount * 2; @@ -118,8 +119,8 @@ protected override bool TryStart(out string error) this.Context.Tracer.RelatedEvent(EventLevel.Informational, $"{nameof(this.TryStart)}_StartedVirtualization", metadata: null); return true; - } - + } + private static byte[] ToVersionIdByteArray(byte[] version) { byte[] bytes = new byte[VirtualizationInstance.PlaceholderIdLength]; @@ -189,8 +190,8 @@ private Result OnGetFileStream( (stream, blobLength) => { // TODO(Mac): Find a better solution than reading from the stream one byte at at time - byte[] buffer = new byte[4096]; - uint bufferIndex = 0; + byte[] buffer = new byte[4096]; + uint bufferIndex = 0; int nextByte = stream.ReadByte(); while (nextByte != -1) { @@ -347,6 +348,16 @@ private void OnNewFileCreated(string relativePath, bool isDirectory) this.LogUnhandledExceptionAndExit(nameof(this.OnNewFileCreated), metadata); } } + + private void OnFileRenamed(string relativeDestinationPath, bool isDirectory) + { + // relativeSourcePath is handled in the OnPreDelete callback that's triggered + // prior to OnFileRenamed + this.OnFileRenamed( + relativeSourcePath: string.Empty, + relativeDestinationPath: relativeDestinationPath, + isDirectory: isDirectory); + } private Result OnEnumerateDirectory( ulong commandId, diff --git a/GVFS/GVFS.Platform.Windows/WindowsFileSystemVirtualizer.cs b/GVFS/GVFS.Platform.Windows/WindowsFileSystemVirtualizer.cs index 02167bd3e..6fb439171 100644 --- a/GVFS/GVFS.Platform.Windows/WindowsFileSystemVirtualizer.cs +++ b/GVFS/GVFS.Platform.Windows/WindowsFileSystemVirtualizer.cs @@ -1223,35 +1223,7 @@ private void NotifyFileRenamedHandler( bool isDirectory, ref NotificationType notificationMask) { - try - { - bool srcPathInDotGit = FileSystemCallbacks.IsPathInsideDotGit(virtualPath); - bool dstPathInDotGit = FileSystemCallbacks.IsPathInsideDotGit(destinationPath); - - if (dstPathInDotGit) - { - this.OnDotGitFileOrFolderChanged(destinationPath); - } - - if (!(srcPathInDotGit && dstPathInDotGit)) - { - if (isDirectory) - { - this.FileSystemCallbacks.OnFolderRenamed(virtualPath, destinationPath); - } - else - { - this.FileSystemCallbacks.OnFileRenamed(virtualPath, destinationPath); - } - } - } - catch (Exception e) - { - EventMetadata metadata = this.CreateEventMetadata(virtualPath, e); - metadata.Add("destinationPath", destinationPath); - metadata.Add("isDirectory", isDirectory); - this.LogUnhandledExceptionAndExit(nameof(this.NotifyFileRenamedHandler), metadata); - } + this.OnFileRenamed(virtualPath, destinationPath, isDirectory); } private void NotifyFileHandleClosedFileModifiedOrDeletedHandler( diff --git a/GVFS/GVFS.Virtualization/FileSystem/FileSystemVirtualizer.cs b/GVFS/GVFS.Virtualization/FileSystem/FileSystemVirtualizer.cs index 33c90c988..acd21633b 100644 --- a/GVFS/GVFS.Virtualization/FileSystem/FileSystemVirtualizer.cs +++ b/GVFS/GVFS.Virtualization/FileSystem/FileSystemVirtualizer.cs @@ -208,6 +208,39 @@ protected void OnWorkingDirectoryFileOrFolderDeleted(string relativePath, bool i this.FileSystemCallbacks.InvalidateGitStatusCache(); } + + protected void OnFileRenamed(string relativeSourcePath, string relativeDestinationPath, bool isDirectory) + { + try + { + bool srcPathInDotGit = FileSystemCallbacks.IsPathInsideDotGit(relativeSourcePath); + bool dstPathInDotGit = FileSystemCallbacks.IsPathInsideDotGit(relativeDestinationPath); + + if (dstPathInDotGit) + { + this.OnDotGitFileOrFolderChanged(relativeDestinationPath); + } + + if (!(srcPathInDotGit && dstPathInDotGit)) + { + if (isDirectory) + { + this.FileSystemCallbacks.OnFolderRenamed(relativeSourcePath, relativeDestinationPath); + } + else + { + this.FileSystemCallbacks.OnFileRenamed(relativeSourcePath, relativeDestinationPath); + } + } + } + catch (Exception e) + { + EventMetadata metadata = this.CreateEventMetadata(relativeSourcePath, e); + metadata.Add("destinationPath", relativeDestinationPath); + metadata.Add("isDirectory", isDirectory); + this.LogUnhandledExceptionAndExit(nameof(this.OnFileRenamed), metadata); + } + } protected EventMetadata CreateEventMetadata( Guid enumerationId, diff --git a/ProjFS.Mac/PrjFSKext/PrjFSKext/KauthHandler.cpp b/ProjFS.Mac/PrjFSKext/PrjFSKext/KauthHandler.cpp index 7dc73f78f..1f9e4b8d6 100644 --- a/ProjFS.Mac/PrjFSKext/PrjFSKext/KauthHandler.cpp +++ b/ProjFS.Mac/PrjFSKext/PrjFSKext/KauthHandler.cpp @@ -226,6 +226,8 @@ void KauthHandler_HandleKernelMessageResponse(uint64_t messageId, MessageType re case MessageType_KtoU_NotifyFilePreDelete: case MessageType_KtoU_NotifyDirectoryPreDelete: case MessageType_KtoU_NotifyFileCreated: + case MessageType_KtoU_NotifyFileRenamed: + case MessageType_KtoU_NotifyDirectoryRenamed: KextLog_Error("KauthHandler_HandleKernelMessageResponse: Unexpected responseType: %d", responseType); break; } @@ -376,8 +378,61 @@ static int HandleFileOpOperation( atomic_fetch_add(&s_numActiveKauthEvents, 1); vfs_context_t context = vfs_context_create(NULL); + vnode_t currentVnodeFromPath = NULLVP; - if (KAUTH_FILEOP_CLOSE == action) + if (KAUTH_FILEOP_RENAME == action) + { + // arg0 is the (const char *) fromPath + const char* toPath = (const char*)arg1; + + // TODO(Mac): Improve error handling + errno_t toErr = vnode_lookup(toPath, 0 /* flags */, ¤tVnodeFromPath, context); + if (0 != toErr) + { + goto CleanupAndReturn; + } + + vtype vnodeType = vnode_vtype(currentVnodeFromPath); + if (ShouldIgnoreVnodeType(vnodeType, currentVnodeFromPath)) + { + goto CleanupAndReturn; + } + + if (!HasAncestorFlaggedAsInRoot(currentVnodeFromPath, context)) + { + goto CleanupAndReturn; + } + + VirtualizationRoot* root = nullptr; + int pid; + if (!ShouldHandleFileOpEvent( + context, + currentVnodeFromPath, + action, + &root, + &pid)) + { + goto CleanupAndReturn; + } + + char procname[MAXCOMLEN + 1]; + proc_name(pid, procname, MAXCOMLEN + 1); + + int kauthResult; + int kauthError; + if (!TrySendRequestAndWaitForResponse( + root, + vnode_isdir(currentVnodeFromPath) ? MessageType_KtoU_NotifyDirectoryRenamed : MessageType_KtoU_NotifyFileRenamed, + currentVnodeFromPath, + pid, + procname, + &kauthResult, + &kauthError)) + { + goto CleanupAndReturn; + } + } + else if (KAUTH_FILEOP_CLOSE == action) { vnode_t currentVnode = reinterpret_cast(arg0); // arg1 is the (const char *) path @@ -449,7 +504,12 @@ static int HandleFileOpOperation( } } -CleanupAndReturn: +CleanupAndReturn: + if (NULLVP != currentVnodeFromPath) + { + vnode_put(currentVnodeFromPath); + } + vfs_context_rele(context); atomic_fetch_sub(&s_numActiveKauthEvents, 1); @@ -472,6 +532,7 @@ static bool ShouldHandleVnodeOpEvent( char procname[MAXCOMLEN + 1], int* kauthResult) { + *root = nullptr; *kauthResult = KAUTH_RESULT_DEFER; if (!VirtualizationRoot_VnodeIsOnAllowedFilesystem(vnode)) diff --git a/ProjFS.Mac/PrjFSKext/PrjFSKext/VirtualizationRoots.hpp b/ProjFS.Mac/PrjFSKext/PrjFSKext/VirtualizationRoots.hpp index a74d33e74..49389c10c 100644 --- a/ProjFS.Mac/PrjFSKext/PrjFSKext/VirtualizationRoots.hpp +++ b/ProjFS.Mac/PrjFSKext/PrjFSKext/VirtualizationRoots.hpp @@ -18,7 +18,7 @@ struct VirtualizationRoot fsid_t rootFsid; uint64_t rootInode; - // TODO: this should eventually be entirely diagnostic and not used for decisions + // TODO(Mac): this should eventually be entirely diagnostic and not used for decisions char path[PrjFSMaxPath]; int32_t index; diff --git a/ProjFS.Mac/PrjFSKext/public/Message.h b/ProjFS.Mac/PrjFSKext/public/Message.h index 723303327..ffedf4175 100644 --- a/ProjFS.Mac/PrjFSKext/public/Message.h +++ b/ProjFS.Mac/PrjFSKext/public/Message.h @@ -20,6 +20,8 @@ typedef enum MessageType_KtoU_NotifyFilePreDelete, MessageType_KtoU_NotifyDirectoryPreDelete, MessageType_KtoU_NotifyFileCreated, + MessageType_KtoU_NotifyFileRenamed, + MessageType_KtoU_NotifyDirectoryRenamed, // Responses MessageType_Response_Success, diff --git a/ProjFS.Mac/PrjFSLib.Mac.Managed/CallbackDelegates.cs b/ProjFS.Mac/PrjFSLib.Mac.Managed/CallbackDelegates.cs index bb70f658e..26254b2dc 100644 --- a/ProjFS.Mac/PrjFSLib.Mac.Managed/CallbackDelegates.cs +++ b/ProjFS.Mac/PrjFSLib.Mac.Managed/CallbackDelegates.cs @@ -31,8 +31,7 @@ public delegate Result NotifyOperationCallback( int triggeringProcessId, string triggeringProcessName, bool isDirectory, - NotificationType notificationType, - string destinationRelativePath); + NotificationType notificationType); // Pre-event notifications public delegate Result NotifyPreDeleteEvent( @@ -51,7 +50,6 @@ public delegate void NotifyNewFileCreatedEvent( bool isDirectory); public delegate void NotifyFileRenamedEvent( - string relativeSourcePath, string relativeDestinationPath, bool isDirectory); diff --git a/ProjFS.Mac/PrjFSLib.Mac.Managed/VirtualizationInstance.cs b/ProjFS.Mac/PrjFSLib.Mac.Managed/VirtualizationInstance.cs index 40638e54b..587822842 100644 --- a/ProjFS.Mac/PrjFSLib.Mac.Managed/VirtualizationInstance.cs +++ b/ProjFS.Mac/PrjFSLib.Mac.Managed/VirtualizationInstance.cs @@ -16,8 +16,9 @@ public class VirtualizationInstance public virtual NotifyFileModified OnFileModified { get; set; } public virtual NotifyPreDeleteEvent OnPreDelete { get; set; } - public virtual NotifyNewFileCreatedEvent OnNewFileCreated { get; set; } - + public virtual NotifyNewFileCreatedEvent OnNewFileCreated { get; set; } + public virtual NotifyFileRenamedEvent OnFileRenamed { get; set; } + public static Result ConvertDirectoryToVirtualizationRoot(string fullPath) { return Interop.PrjFSLib.ConvertDirectoryToVirtualizationRoot(fullPath); @@ -131,8 +132,7 @@ private Result OnNotifyOperation( int triggeringProcessId, string triggeringProcessName, bool isDirectory, - NotificationType notificationType, - string destinationRelativePath) + NotificationType notificationType) { switch (notificationType) { @@ -146,6 +146,10 @@ private Result OnNotifyOperation( case NotificationType.NewFileCreated: this.OnNewFileCreated(relativePath, isDirectory); return Result.Success; + + case NotificationType.FileRenamed: + this.OnFileRenamed(relativePath, isDirectory); + return Result.Success; } return Result.ENotYetImplemented; diff --git a/ProjFS.Mac/PrjFSLib/PrjFSLib.cpp b/ProjFS.Mac/PrjFSLib/PrjFSLib.cpp index 5e18ffec9..5312f0d1f 100644 --- a/ProjFS.Mac/PrjFSLib/PrjFSLib.cpp +++ b/ProjFS.Mac/PrjFSLib/PrjFSLib.cpp @@ -432,38 +432,44 @@ static void HandleKernelRequest(Message request, void* messageMemory) } case MessageType_KtoU_NotifyFilePreDelete: + case MessageType_KtoU_NotifyDirectoryPreDelete: { result = HandleFileNotification( requestHeader, request.path, - false, // isDirectory + requestHeader->messageType == MessageType_KtoU_NotifyDirectoryPreDelete, // isDirectory PrjFS_NotificationType_PreDelete); break; } - case MessageType_KtoU_NotifyDirectoryPreDelete: + case MessageType_KtoU_NotifyFileCreated: { + char fullPath[PrjFSMaxPath]; + CombinePaths(s_virtualizationRootFullPath.c_str(), request.path, fullPath); + SetBitInFileFlags(fullPath, FileFlags_IsInVirtualizationRoot, true); + result = HandleFileNotification( requestHeader, request.path, - true, // isDirectory - PrjFS_NotificationType_PreDelete); + false, // isDirectory + PrjFS_NotificationType_NewFileCreated); break; } - case MessageType_KtoU_NotifyFileCreated: + case MessageType_KtoU_NotifyFileRenamed: + case MessageType_KtoU_NotifyDirectoryRenamed: { char fullPath[PrjFSMaxPath]; CombinePaths(s_virtualizationRootFullPath.c_str(), request.path, fullPath); // TODO(Mac): Handle SetBitInFileFlags failures SetBitInFileFlags(fullPath, FileFlags_IsInVirtualizationRoot, true); - + result = HandleFileNotification( requestHeader, request.path, - false, // isDirectory - PrjFS_NotificationType_NewFileCreated); + requestHeader->messageType == MessageType_KtoU_NotifyDirectoryRenamed, // isDirectory + PrjFS_NotificationType_FileRenamed); break; } } From d3e24b31a61cdc0cb73d675e684e627430acdc0b Mon Sep 17 00:00:00 2001 From: William Baker Date: Mon, 20 Aug 2018 15:04:26 -0700 Subject: [PATCH 054/272] Update MirrorProvider for rename notification --- .../MirrorProvider.Mac/MacFileSystemVirtualizer.cs | 8 +++++++- .../WindowsFileSystemVirtualizer.cs | 13 ++++++++++++- 2 files changed, 19 insertions(+), 2 deletions(-) diff --git a/MirrorProvider/MirrorProvider.Mac/MacFileSystemVirtualizer.cs b/MirrorProvider/MirrorProvider.Mac/MacFileSystemVirtualizer.cs index 5f20521bb..5bd11e6e9 100644 --- a/MirrorProvider/MirrorProvider.Mac/MacFileSystemVirtualizer.cs +++ b/MirrorProvider/MirrorProvider.Mac/MacFileSystemVirtualizer.cs @@ -22,7 +22,8 @@ public override bool TryStartVirtualizationInstance(Enlistment enlistment, out s this.virtualizationInstance.OnGetFileStream = this.OnGetFileStream; this.virtualizationInstance.OnFileModified = this.OnFileModified; this.virtualizationInstance.OnPreDelete = this.OnPreDelete; - this.virtualizationInstance.OnNewFileCreated = this.OnNewFileCreated; + this.virtualizationInstance.OnNewFileCreated = this.OnNewFileCreated; + this.virtualizationInstance.OnFileRenamed = this.OnFileRenamed; Result result = this.virtualizationInstance.StartVirtualizationInstance( enlistment.SrcRoot, @@ -164,6 +165,11 @@ private void OnNewFileCreated(string relativePath, bool isDirectory) { Console.WriteLine($"OnNewFileCreated (isDirectory: {isDirectory}): {relativePath}"); } + + private void OnFileRenamed(string relativeDestinationPath, bool isDirectory) + { + Console.WriteLine($"OnFileRenamed (isDirectory: {isDirectory}) destination: {relativeDestinationPath}"); + } private static byte[] ToVersionIdByteArray(byte version) { diff --git a/MirrorProvider/MirrorProvider.Windows/WindowsFileSystemVirtualizer.cs b/MirrorProvider/MirrorProvider.Windows/WindowsFileSystemVirtualizer.cs index cecea9dce..b78cc2391 100644 --- a/MirrorProvider/MirrorProvider.Windows/WindowsFileSystemVirtualizer.cs +++ b/MirrorProvider/MirrorProvider.Windows/WindowsFileSystemVirtualizer.cs @@ -37,6 +37,7 @@ public override bool TryStartVirtualizationInstance(Enlistment enlistment, out s this.virtualizationInstance.OnNotifyPreDelete = this.OnPreDelete; this.virtualizationInstance.OnNotifyNewFileCreated = this.OnNewFileCreated; this.virtualizationInstance.OnNotifyFileHandleClosedFileModifiedOrDeleted = this.OnFileModifiedOrDeleted; + this.virtualizationInstance.OnNotifyFileRenamed = this.OnFileRenamed; uint threadCount = (uint)Environment.ProcessorCount * 2; @@ -44,7 +45,8 @@ public override bool TryStartVirtualizationInstance(Enlistment enlistment, out s { new NotificationMapping( NotificationType.NewFileCreated | - NotificationType.PreDelete | + NotificationType.PreDelete | + NotificationType.FileRenamed | NotificationType.FileHandleClosedFileModified, string.Empty), }; @@ -316,6 +318,15 @@ private void OnFileModifiedOrDeleted(string relativePath, bool isDirectory, bool Console.WriteLine($"OnFileModifiedOrDeleted: `{relativePath}`, isDirectory: {isDirectory}, isModfied: {isFileDeleted}, isDeleted: {isFileDeleted}"); } + private void OnFileRenamed( + string relativePath, + string relativeDestinationPath, + bool isDirectory, + ref NotificationType notificationMask) + { + Console.WriteLine($"OnFileRenamed (isDirectory: {isDirectory}), relativePath: {relativePath}, relativeDestinationPath: {relativeDestinationPath}"); + } + // TODO: Add this to the ProjFS API private static HResult HResultFromWin32(int win32error) { From 1918f53b1b6b9329d6e18258b0423f744235636c Mon Sep 17 00:00:00 2001 From: William Baker Date: Tue, 21 Aug 2018 09:09:26 -0700 Subject: [PATCH 055/272] Mac: Hardlink created notification --- .../EnlistmentPerFixture/GitFilesTests.cs | 2 -- .../GitMoveRenameTests.cs | 13 ----------- .../MacFileSystemVirtualizer.cs | 23 +++++++++++++++++++ .../FileSystemCallbacksTests.cs | 6 +++++ .../Background/FileSystemTask.cs | 8 ++++++- .../FileSystemCallbacks.cs | 6 +++++ .../PrjFSKext/PrjFSKext/KauthHandler.cpp | 22 ++++++++++++++---- ProjFS.Mac/PrjFSKext/public/Message.h | 1 + .../PrjFSLib.Mac.Managed/CallbackDelegates.cs | 3 +++ .../PrjFSLib.Mac.Managed/NotificationType.cs | 1 + .../VirtualizationInstance.cs | 5 ++++ ProjFS.Mac/PrjFSLib/PrjFSLib.cpp | 18 +++++++++++++-- ProjFS.Mac/PrjFSLib/PrjFSLib.h | 1 + 13 files changed, 86 insertions(+), 23 deletions(-) diff --git a/GVFS/GVFS.FunctionalTests/Tests/EnlistmentPerFixture/GitFilesTests.cs b/GVFS/GVFS.FunctionalTests/Tests/EnlistmentPerFixture/GitFilesTests.cs index 1f8d91d38..7abf164af 100644 --- a/GVFS/GVFS.FunctionalTests/Tests/EnlistmentPerFixture/GitFilesTests.cs +++ b/GVFS/GVFS.FunctionalTests/Tests/EnlistmentPerFixture/GitFilesTests.cs @@ -189,8 +189,6 @@ public void ModifiedFileWillGetAddedToModifiedPathsFile() [TestCase, Order(8)] public void RenamedFileAddedToModifiedPathsFile() { - GitMoveRenameTests.IgnoreSystemIOMoveOnMac(this.fileSystem); - string fileToRenameEntry = "Test_EPF_MoveRenameFileTests/ChangeUnhydratedFileName/Program.cs"; string fileToRenameTargetEntry = "Test_EPF_MoveRenameFileTests/ChangeUnhydratedFileName/Program2.cs"; this.VerifyWorktreeBit(fileToRenameEntry, LsFilesStatus.SkipWorktree); diff --git a/GVFS/GVFS.FunctionalTests/Tests/EnlistmentPerFixture/GitMoveRenameTests.cs b/GVFS/GVFS.FunctionalTests/Tests/EnlistmentPerFixture/GitMoveRenameTests.cs index 30f1e4ae3..fa0fbe74f 100644 --- a/GVFS/GVFS.FunctionalTests/Tests/EnlistmentPerFixture/GitMoveRenameTests.cs +++ b/GVFS/GVFS.FunctionalTests/Tests/EnlistmentPerFixture/GitMoveRenameTests.cs @@ -22,15 +22,6 @@ public GitMoveRenameTests(FileSystemRunner fileSystem) this.fileSystem = fileSystem; } - public static void IgnoreSystemIOMoveOnMac(FileSystemRunner runner) - { - if (runner is SystemIORunner && - RuntimeInformation.IsOSPlatform(OSPlatform.OSX)) - { - Assert.Ignore("TODO(Mac): SystemIORunner rename tests require hardlink notifications"); - } - } - [TestCase, Order(1)] public void GitStatus() { @@ -85,8 +76,6 @@ public void GitStatusAfterFileNameCaseChange() [TestCase, Order(4)] public void GitStatusAfterFileRename() { - IgnoreSystemIOMoveOnMac(this.fileSystem); - string oldFilename = "New.cs"; this.EnsureTestFileExists(oldFilename); @@ -195,8 +184,6 @@ public void GitWithEnvironmentVariables() [TestCase, Order(9)] public void GitStatusAfterRenameFileIntoRepo() { - IgnoreSystemIOMoveOnMac(this.fileSystem); - string filename = "GitStatusAfterRenameFileIntoRepo.cs"; // Create the test file in this.Enlistment.EnlistmentRoot as it's outside of src diff --git a/GVFS/GVFS.Platform.Mac/MacFileSystemVirtualizer.cs b/GVFS/GVFS.Platform.Mac/MacFileSystemVirtualizer.cs index 2e54d0284..081eef322 100644 --- a/GVFS/GVFS.Platform.Mac/MacFileSystemVirtualizer.cs +++ b/GVFS/GVFS.Platform.Mac/MacFileSystemVirtualizer.cs @@ -103,6 +103,7 @@ protected override bool TryStart(out string error) this.virtualizationInstance.OnPreDelete = this.OnPreDelete; this.virtualizationInstance.OnNewFileCreated = this.OnNewFileCreated; this.virtualizationInstance.OnFileRenamed = this.OnFileRenamed; + this.virtualizationInstance.OnHardLinkCreated = this.OnHardLinkCreated; uint threadCount = (uint)Environment.ProcessorCount * 2; @@ -359,6 +360,28 @@ private void OnFileRenamed(string relativeDestinationPath, bool isDirectory) isDirectory: isDirectory); } + private void OnHardLinkCreated(string relativeNewLinkPath) + { + try + { + bool pathInDotGit = Virtualization.FileSystemCallbacks.IsPathInsideDotGit(relativeNewLinkPath); + + if (pathInDotGit) + { + this.OnDotGitFileOrFolderChanged(relativeNewLinkPath); + } + else + { + this.FileSystemCallbacks.OnFileHardLinkCreated(relativeNewLinkPath); + } + } + catch (Exception e) + { + EventMetadata metadata = this.CreateEventMetadata(relativeNewLinkPath, e); + this.LogUnhandledExceptionAndExit(nameof(this.OnHardLinkCreated), metadata); + } + } + private Result OnEnumerateDirectory( ulong commandId, string relativePath, diff --git a/GVFS/GVFS.UnitTests/Virtualization/FileSystemCallbacksTests.cs b/GVFS/GVFS.UnitTests/Virtualization/FileSystemCallbacksTests.cs index 55c1ad296..7585b6179 100644 --- a/GVFS/GVFS.UnitTests/Virtualization/FileSystemCallbacksTests.cs +++ b/GVFS/GVFS.UnitTests/Virtualization/FileSystemCallbacksTests.cs @@ -275,6 +275,12 @@ public void FileAndFolderCallbacksScheduleBackgroundTasks() "OnFileRenamed2.txt", FileSystemTask.OperationType.OnFileRenamed); + this.CallbackSchedulesBackgroundTask( + backgroundTaskRunner, + (path) => fileSystemCallbacks.OnFileHardLinkCreated(path), + "OnFileHardLinkCreated.txt", + FileSystemTask.OperationType.OnFileHardLinkCreated); + this.CallbackSchedulesBackgroundTask( backgroundTaskRunner, (path) => fileSystemCallbacks.OnFileSuperseded(path), diff --git a/GVFS/GVFS.Virtualization/Background/FileSystemTask.cs b/GVFS/GVFS.Virtualization/Background/FileSystemTask.cs index a88d18e6a..b5644bf86 100644 --- a/GVFS/GVFS.Virtualization/Background/FileSystemTask.cs +++ b/GVFS/GVFS.Virtualization/Background/FileSystemTask.cs @@ -28,7 +28,8 @@ public enum OperationType OnFolderDeleted, OnFolderFirstWrite, OnIndexWriteWithoutProjectionChange, - OnPlaceholderCreationsBlockedForGit + OnPlaceholderCreationsBlockedForGit, + OnFileHardLinkCreated } public OperationType Operation { get; } @@ -46,6 +47,11 @@ public static FileSystemTask OnFileRenamed(string oldVirtualPath, string newVirt return new FileSystemTask(OperationType.OnFileRenamed, newVirtualPath, oldVirtualPath); } + public static FileSystemTask OnFileHardLinkCreated(string newLinkRelativePath) + { + return new FileSystemTask(OperationType.OnFileHardLinkCreated, newLinkRelativePath, oldVirtualPath: null); + } + public static FileSystemTask OnFileDeleted(string virtualPath) { return new FileSystemTask(OperationType.OnFileDeleted, virtualPath, oldVirtualPath: null); diff --git a/GVFS/GVFS.Virtualization/FileSystemCallbacks.cs b/GVFS/GVFS.Virtualization/FileSystemCallbacks.cs index 906bad018..980ab2b87 100644 --- a/GVFS/GVFS.Virtualization/FileSystemCallbacks.cs +++ b/GVFS/GVFS.Virtualization/FileSystemCallbacks.cs @@ -423,6 +423,11 @@ public virtual void OnFileRenamed(string oldRelativePath, string newRelativePath this.backgroundFileSystemTaskRunner.Enqueue(FileSystemTask.OnFileRenamed(oldRelativePath, newRelativePath)); } + public virtual void OnFileHardLinkCreated(string newLinkRelativePath) + { + this.backgroundFileSystemTaskRunner.Enqueue(FileSystemTask.OnFileHardLinkCreated(newLinkRelativePath)); + } + public void OnFileDeleted(string relativePath) { this.backgroundFileSystemTaskRunner.Enqueue(FileSystemTask.OnFileDeleted(relativePath)); @@ -606,6 +611,7 @@ private FileSystemTaskResult ExecuteBackgroundOperation(FileSystemTask gitUpdate { case FileSystemTask.OperationType.OnFileCreated: case FileSystemTask.OperationType.OnFailedPlaceholderDelete: + case FileSystemTask.OperationType.OnFileHardLinkCreated: metadata.Add("virtualPath", gitUpdate.VirtualPath); result = this.AddModifiedPathAndRemoveFromPlaceholderList(gitUpdate.VirtualPath); break; diff --git a/ProjFS.Mac/PrjFSKext/PrjFSKext/KauthHandler.cpp b/ProjFS.Mac/PrjFSKext/PrjFSKext/KauthHandler.cpp index 1f9e4b8d6..f7f9fc6ff 100644 --- a/ProjFS.Mac/PrjFSKext/PrjFSKext/KauthHandler.cpp +++ b/ProjFS.Mac/PrjFSKext/PrjFSKext/KauthHandler.cpp @@ -228,6 +228,7 @@ void KauthHandler_HandleKernelMessageResponse(uint64_t messageId, MessageType re case MessageType_KtoU_NotifyFileCreated: case MessageType_KtoU_NotifyFileRenamed: case MessageType_KtoU_NotifyDirectoryRenamed: + case MessageType_KtoU_NotifyFileHardLinkCreated: KextLog_Error("KauthHandler_HandleKernelMessageResponse: Unexpected responseType: %d", responseType); break; } @@ -380,13 +381,14 @@ static int HandleFileOpOperation( vfs_context_t context = vfs_context_create(NULL); vnode_t currentVnodeFromPath = NULLVP; - if (KAUTH_FILEOP_RENAME == action) + if (KAUTH_FILEOP_RENAME == action || + KAUTH_FILEOP_LINK == action) { - // arg0 is the (const char *) fromPath - const char* toPath = (const char*)arg1; + // arg0 is the (const char *) fromPath (or the file being linked to) + const char* newPath = (const char*)arg1; // TODO(Mac): Improve error handling - errno_t toErr = vnode_lookup(toPath, 0 /* flags */, ¤tVnodeFromPath, context); + errno_t toErr = vnode_lookup(newPath, 0 /* flags */, ¤tVnodeFromPath, context); if (0 != toErr) { goto CleanupAndReturn; @@ -418,11 +420,21 @@ static int HandleFileOpOperation( char procname[MAXCOMLEN + 1]; proc_name(pid, procname, MAXCOMLEN + 1); + MessageType messageType; + if (KAUTH_FILEOP_RENAME == action) + { + messageType = vnode_isdir(currentVnodeFromPath) ? MessageType_KtoU_NotifyDirectoryRenamed : MessageType_KtoU_NotifyFileRenamed; + } + else + { + messageType = MessageType_KtoU_NotifyFileHardLinkCreated; + } + int kauthResult; int kauthError; if (!TrySendRequestAndWaitForResponse( root, - vnode_isdir(currentVnodeFromPath) ? MessageType_KtoU_NotifyDirectoryRenamed : MessageType_KtoU_NotifyFileRenamed, + messageType, currentVnodeFromPath, pid, procname, diff --git a/ProjFS.Mac/PrjFSKext/public/Message.h b/ProjFS.Mac/PrjFSKext/public/Message.h index ffedf4175..a4a200fb8 100644 --- a/ProjFS.Mac/PrjFSKext/public/Message.h +++ b/ProjFS.Mac/PrjFSKext/public/Message.h @@ -22,6 +22,7 @@ typedef enum MessageType_KtoU_NotifyFileCreated, MessageType_KtoU_NotifyFileRenamed, MessageType_KtoU_NotifyDirectoryRenamed, + MessageType_KtoU_NotifyFileHardLinkCreated, // Responses MessageType_Response_Success, diff --git a/ProjFS.Mac/PrjFSLib.Mac.Managed/CallbackDelegates.cs b/ProjFS.Mac/PrjFSLib.Mac.Managed/CallbackDelegates.cs index 26254b2dc..769a9743e 100644 --- a/ProjFS.Mac/PrjFSLib.Mac.Managed/CallbackDelegates.cs +++ b/ProjFS.Mac/PrjFSLib.Mac.Managed/CallbackDelegates.cs @@ -53,6 +53,9 @@ public delegate void NotifyFileRenamedEvent( string relativeDestinationPath, bool isDirectory); + public delegate void NotifyHardLinkCreatedEvent( + string relativeNewLinkPath); + public delegate void NotifyFileModified( string relativePath); diff --git a/ProjFS.Mac/PrjFSLib.Mac.Managed/NotificationType.cs b/ProjFS.Mac/PrjFSLib.Mac.Managed/NotificationType.cs index 7e3c6b084..08b780d72 100644 --- a/ProjFS.Mac/PrjFSLib.Mac.Managed/NotificationType.cs +++ b/ProjFS.Mac/PrjFSLib.Mac.Managed/NotificationType.cs @@ -11,6 +11,7 @@ public enum NotificationType NewFileCreated = 0x00000004, PreDelete = 0x00000010, FileRenamed = 0x00000080, + HardLinkCreated = 0x00000100, PreConvertToFull = 0x00001000, PreModify = 0x10000001, diff --git a/ProjFS.Mac/PrjFSLib.Mac.Managed/VirtualizationInstance.cs b/ProjFS.Mac/PrjFSLib.Mac.Managed/VirtualizationInstance.cs index 587822842..b2ddce178 100644 --- a/ProjFS.Mac/PrjFSLib.Mac.Managed/VirtualizationInstance.cs +++ b/ProjFS.Mac/PrjFSLib.Mac.Managed/VirtualizationInstance.cs @@ -18,6 +18,7 @@ public class VirtualizationInstance public virtual NotifyPreDeleteEvent OnPreDelete { get; set; } public virtual NotifyNewFileCreatedEvent OnNewFileCreated { get; set; } public virtual NotifyFileRenamedEvent OnFileRenamed { get; set; } + public virtual NotifyHardLinkCreatedEvent OnHardLinkCreated { get; set; } public static Result ConvertDirectoryToVirtualizationRoot(string fullPath) { @@ -150,6 +151,10 @@ private Result OnNotifyOperation( case NotificationType.FileRenamed: this.OnFileRenamed(relativePath, isDirectory); return Result.Success; + + case NotificationType.HardLinkCreated: + this.OnHardLinkCreated(relativePath); + return Result.Success; } return Result.ENotYetImplemented; diff --git a/ProjFS.Mac/PrjFSLib/PrjFSLib.cpp b/ProjFS.Mac/PrjFSLib/PrjFSLib.cpp index 5312f0d1f..36be88ef3 100644 --- a/ProjFS.Mac/PrjFSLib/PrjFSLib.cpp +++ b/ProjFS.Mac/PrjFSLib/PrjFSLib.cpp @@ -458,6 +458,7 @@ static void HandleKernelRequest(Message request, void* messageMemory) case MessageType_KtoU_NotifyFileRenamed: case MessageType_KtoU_NotifyDirectoryRenamed: + case MessageType_KtoU_NotifyFileHardLinkCreated: { char fullPath[PrjFSMaxPath]; CombinePaths(s_virtualizationRootFullPath.c_str(), request.path, fullPath); @@ -465,11 +466,22 @@ static void HandleKernelRequest(Message request, void* messageMemory) // TODO(Mac): Handle SetBitInFileFlags failures SetBitInFileFlags(fullPath, FileFlags_IsInVirtualizationRoot, true); + bool isDirectory = requestHeader->messageType == MessageType_KtoU_NotifyDirectoryRenamed; + PrjFS_NotificationType notificationType; + if (requestHeader->messageType == MessageType_KtoU_NotifyFileHardLinkCreated) + { + notificationType = PrjFS_NotificationType_HardLinkCreated; + } + else + { + notificationType = PrjFS_NotificationType_FileRenamed; + } + result = HandleFileNotification( requestHeader, request.path, - requestHeader->messageType == MessageType_KtoU_NotifyDirectoryRenamed, // isDirectory - PrjFS_NotificationType_FileRenamed); + isDirectory, + notificationType); break; } } @@ -783,6 +795,8 @@ static const char* NotificationTypeToString(PrjFS_NotificationType notificationT return STRINGIFY(PrjFS_NotificationType_PreDelete); case PrjFS_NotificationType_FileRenamed: return STRINGIFY(PrjFS_NotificationType_FileRenamed); + case PrjFS_NotificationType_HardLinkCreated: + return STRINGIFY(PrjFS_NotificationType_HardLinkCreated); case PrjFS_NotificationType_PreConvertToFull: return STRINGIFY(PrjFS_NotificationType_PreConvertToFull); diff --git a/ProjFS.Mac/PrjFSLib/PrjFSLib.h b/ProjFS.Mac/PrjFSLib/PrjFSLib.h index 98b512dc0..ceddbb578 100644 --- a/ProjFS.Mac/PrjFSLib/PrjFSLib.h +++ b/ProjFS.Mac/PrjFSLib/PrjFSLib.h @@ -46,6 +46,7 @@ typedef enum PrjFS_NotificationType_NewFileCreated = 0x00000004, PrjFS_NotificationType_PreDelete = 0x00000010, PrjFS_NotificationType_FileRenamed = 0x00000080, + PrjFS_NotificationType_HardLinkCreated = 0x00000100, PrjFS_NotificationType_PreConvertToFull = 0x00001000, PrjFS_NotificationType_PreModify = 0x10000001, From 690c1d484f5e2f0b345f5bc6bc2f0cd89124e267 Mon Sep 17 00:00:00 2001 From: William Baker Date: Tue, 21 Aug 2018 09:53:53 -0700 Subject: [PATCH 056/272] Update MirrorProvider for hardlink notification --- .../MirrorProvider.Mac/MacFileSystemVirtualizer.cs | 8 +++++++- .../WindowsFileSystemVirtualizer.cs | 9 +++++++++ 2 files changed, 16 insertions(+), 1 deletion(-) diff --git a/MirrorProvider/MirrorProvider.Mac/MacFileSystemVirtualizer.cs b/MirrorProvider/MirrorProvider.Mac/MacFileSystemVirtualizer.cs index 5bd11e6e9..3d9396c82 100644 --- a/MirrorProvider/MirrorProvider.Mac/MacFileSystemVirtualizer.cs +++ b/MirrorProvider/MirrorProvider.Mac/MacFileSystemVirtualizer.cs @@ -23,7 +23,8 @@ public override bool TryStartVirtualizationInstance(Enlistment enlistment, out s this.virtualizationInstance.OnFileModified = this.OnFileModified; this.virtualizationInstance.OnPreDelete = this.OnPreDelete; this.virtualizationInstance.OnNewFileCreated = this.OnNewFileCreated; - this.virtualizationInstance.OnFileRenamed = this.OnFileRenamed; + this.virtualizationInstance.OnFileRenamed = this.OnFileRenamed; + this.virtualizationInstance.OnHardLinkCreated = this.OnHardLinkCreated; Result result = this.virtualizationInstance.StartVirtualizationInstance( enlistment.SrcRoot, @@ -170,6 +171,11 @@ private void OnFileRenamed(string relativeDestinationPath, bool isDirectory) { Console.WriteLine($"OnFileRenamed (isDirectory: {isDirectory}) destination: {relativeDestinationPath}"); } + + private void OnHardLinkCreated(string relativeNewLinkPath) + { + Console.WriteLine($"OnHardLinkCreated: {relativeNewLinkPath}"); + } private static byte[] ToVersionIdByteArray(byte version) { diff --git a/MirrorProvider/MirrorProvider.Windows/WindowsFileSystemVirtualizer.cs b/MirrorProvider/MirrorProvider.Windows/WindowsFileSystemVirtualizer.cs index b78cc2391..edfd0acf1 100644 --- a/MirrorProvider/MirrorProvider.Windows/WindowsFileSystemVirtualizer.cs +++ b/MirrorProvider/MirrorProvider.Windows/WindowsFileSystemVirtualizer.cs @@ -38,6 +38,7 @@ public override bool TryStartVirtualizationInstance(Enlistment enlistment, out s this.virtualizationInstance.OnNotifyNewFileCreated = this.OnNewFileCreated; this.virtualizationInstance.OnNotifyFileHandleClosedFileModifiedOrDeleted = this.OnFileModifiedOrDeleted; this.virtualizationInstance.OnNotifyFileRenamed = this.OnFileRenamed; + this.virtualizationInstance.OnNotifyHardlinkCreated = this.OnHardlinkCreated; uint threadCount = (uint)Environment.ProcessorCount * 2; @@ -47,6 +48,7 @@ public override bool TryStartVirtualizationInstance(Enlistment enlistment, out s NotificationType.NewFileCreated | NotificationType.PreDelete | NotificationType.FileRenamed | + NotificationType.HardlinkCreated | NotificationType.FileHandleClosedFileModified, string.Empty), }; @@ -327,6 +329,13 @@ private void OnFileRenamed( Console.WriteLine($"OnFileRenamed (isDirectory: {isDirectory}), relativePath: {relativePath}, relativeDestinationPath: {relativeDestinationPath}"); } + private void OnHardlinkCreated( + string relativePath, + string relativeDestinationPath) + { + Console.WriteLine($"OnHardlinkCreated, relativePath: {relativePath}, relativeDestinationPath: {relativeDestinationPath}"); + } + // TODO: Add this to the ProjFS API private static HResult HResultFromWin32(int win32error) { From c144821f42996126bd39f9249e90bd43e4287170 Mon Sep 17 00:00:00 2001 From: William Baker Date: Tue, 21 Aug 2018 10:22:14 -0700 Subject: [PATCH 057/272] Update Windows code to handle hardlink creation --- .../MacFileSystemVirtualizer.cs | 21 +++-------------- .../WindowsFileSystemVirtualizer.cs | 9 +++++++- .../FileSystem/FileSystemVirtualizer.cs | 23 +++++++++++++++++++ 3 files changed, 34 insertions(+), 19 deletions(-) diff --git a/GVFS/GVFS.Platform.Mac/MacFileSystemVirtualizer.cs b/GVFS/GVFS.Platform.Mac/MacFileSystemVirtualizer.cs index 081eef322..330dad5ad 100644 --- a/GVFS/GVFS.Platform.Mac/MacFileSystemVirtualizer.cs +++ b/GVFS/GVFS.Platform.Mac/MacFileSystemVirtualizer.cs @@ -362,24 +362,9 @@ private void OnFileRenamed(string relativeDestinationPath, bool isDirectory) private void OnHardLinkCreated(string relativeNewLinkPath) { - try - { - bool pathInDotGit = Virtualization.FileSystemCallbacks.IsPathInsideDotGit(relativeNewLinkPath); - - if (pathInDotGit) - { - this.OnDotGitFileOrFolderChanged(relativeNewLinkPath); - } - else - { - this.FileSystemCallbacks.OnFileHardLinkCreated(relativeNewLinkPath); - } - } - catch (Exception e) - { - EventMetadata metadata = this.CreateEventMetadata(relativeNewLinkPath, e); - this.LogUnhandledExceptionAndExit(nameof(this.OnHardLinkCreated), metadata); - } + this.OnHardLinkCreated( + relativeTargetPath: string.Empty, + relativeNewLinkPath: relativeNewLinkPath); } private Result OnEnumerateDirectory( diff --git a/GVFS/GVFS.Platform.Windows/WindowsFileSystemVirtualizer.cs b/GVFS/GVFS.Platform.Windows/WindowsFileSystemVirtualizer.cs index 6fb439171..65f5d67ad 100644 --- a/GVFS/GVFS.Platform.Windows/WindowsFileSystemVirtualizer.cs +++ b/GVFS/GVFS.Platform.Windows/WindowsFileSystemVirtualizer.cs @@ -159,7 +159,7 @@ protected override bool TryStart(out string error) this.virtualizationInstance.OnNotifyPreRename = this.NotifyPreRenameHandler; this.virtualizationInstance.OnNotifyPreSetHardlink = null; this.virtualizationInstance.OnNotifyFileRenamed = this.NotifyFileRenamedHandler; - this.virtualizationInstance.OnNotifyHardlinkCreated = null; + this.virtualizationInstance.OnNotifyHardlinkCreated = this.NotifyHardlinkCreated; this.virtualizationInstance.OnNotifyFileHandleClosedNoModification = null; this.virtualizationInstance.OnNotifyFileHandleClosedFileModifiedOrDeleted = this.NotifyFileHandleClosedFileModifiedOrDeletedHandler; this.virtualizationInstance.OnNotifyFilePreConvertToFull = this.NotifyFilePreConvertToFullHandler; @@ -1226,6 +1226,13 @@ private void NotifyFileRenamedHandler( this.OnFileRenamed(virtualPath, destinationPath, isDirectory); } + private void NotifyHardlinkCreated( + string relativePath, + string destinationPath) + { + this.OnHardLinkCreated(relativePath, destinationPath); + } + private void NotifyFileHandleClosedFileModifiedOrDeletedHandler( string virtualPath, bool isDirectory, diff --git a/GVFS/GVFS.Virtualization/FileSystem/FileSystemVirtualizer.cs b/GVFS/GVFS.Virtualization/FileSystem/FileSystemVirtualizer.cs index acd21633b..739c91f54 100644 --- a/GVFS/GVFS.Virtualization/FileSystem/FileSystemVirtualizer.cs +++ b/GVFS/GVFS.Virtualization/FileSystem/FileSystemVirtualizer.cs @@ -242,6 +242,29 @@ protected void OnFileRenamed(string relativeSourcePath, string relativeDestinati } } + protected void OnHardLinkCreated(string relativeTargetPath, string relativeNewLinkPath) + { + try + { + bool pathInDotGit = FileSystemCallbacks.IsPathInsideDotGit(relativeNewLinkPath); + + if (pathInDotGit) + { + this.OnDotGitFileOrFolderChanged(relativeNewLinkPath); + } + else + { + this.FileSystemCallbacks.OnFileHardLinkCreated(relativeNewLinkPath); + } + } + catch (Exception e) + { + EventMetadata metadata = this.CreateEventMetadata(relativeNewLinkPath, e); + metadata.Add(nameof(relativeTargetPath), relativeTargetPath); + this.LogUnhandledExceptionAndExit(nameof(this.OnHardLinkCreated), metadata); + } + } + protected EventMetadata CreateEventMetadata( Guid enumerationId, string relativePath = null, From 67eebcd051efd0cbf7a321c890bce090f8b6ae83 Mon Sep 17 00:00:00 2001 From: William Baker Date: Tue, 21 Aug 2018 10:49:24 -0700 Subject: [PATCH 058/272] Add hardlink creation functional tests --- .../FileSystemRunners/BashRunner.cs | 13 +++++ .../FileSystemRunners/CmdRunner.cs | 10 ++++ .../FileSystemRunners/FileSystemRunner.cs | 4 ++ .../FileSystemRunners/PowerShellRunner.cs | 11 ++++ .../FileSystemRunners/SystemIORunner.cs | 10 ++++ .../EnlistmentPerFixture/GitFilesTests.cs | 51 ++++++++++++++----- .../Tests/GitCommands/AddStageTests.cs | 20 +++++++- .../Tests/GitCommands/GitRepoTests.cs | 10 ++++ 8 files changed, 115 insertions(+), 14 deletions(-) diff --git a/GVFS/GVFS.FunctionalTests/FileSystemRunners/BashRunner.cs b/GVFS/GVFS.FunctionalTests/FileSystemRunners/BashRunner.cs index 6b2310a70..3a33258e0 100644 --- a/GVFS/GVFS.FunctionalTests/FileSystemRunners/BashRunner.cs +++ b/GVFS/GVFS.FunctionalTests/FileSystemRunners/BashRunner.cs @@ -47,6 +47,11 @@ public BashRunner() } } + public override bool SupportsHardlinkCreation + { + get { return true; } + } + protected override string FileName { get @@ -158,6 +163,14 @@ public override void CreateEmptyFile(string path) this.RunProcess(string.Format("-c \"touch {0}\"", bashPath)); } + public override void CreateHardLink(string existingPath, string newLinkPath) + { + string existingFileBashPath = this.ConvertWinPathToBashPath(existingPath); + string newLinkBashPath = this.ConvertWinPathToBashPath(existingPath); + + this.RunProcess(string.Format("-c \"ln {0} {1}\"", existingFileBashPath, newLinkBashPath)); + } + public override void WriteAllText(string path, string contents) { string bashPath = this.ConvertWinPathToBashPath(path); diff --git a/GVFS/GVFS.FunctionalTests/FileSystemRunners/CmdRunner.cs b/GVFS/GVFS.FunctionalTests/FileSystemRunners/CmdRunner.cs index cb14a0bec..cf66c5f87 100644 --- a/GVFS/GVFS.FunctionalTests/FileSystemRunners/CmdRunner.cs +++ b/GVFS/GVFS.FunctionalTests/FileSystemRunners/CmdRunner.cs @@ -27,6 +27,11 @@ public class CmdRunner : ShellRunner "The process cannot access the file because it is being used by another process" }; + public override bool SupportsHardlinkCreation + { + get { return true; } + } + protected override string FileName { get @@ -108,6 +113,11 @@ public override void CreateEmptyFile(string path) this.RunProcess(string.Format("/C type NUL > \"{0}\"", path)); } + public override void CreateHardLink(string targetPath, string newLinkPath) + { + this.RunProcess(string.Format("/C mklink /H \"{0}\" \"{1}\"", newLinkPath, targetPath)); + } + public override void AppendAllText(string path, string contents) { // Use echo|set /p with "" to avoid adding any trailing whitespace or newline diff --git a/GVFS/GVFS.FunctionalTests/FileSystemRunners/FileSystemRunner.cs b/GVFS/GVFS.FunctionalTests/FileSystemRunners/FileSystemRunner.cs index 7679f4fe7..b7c99068e 100644 --- a/GVFS/GVFS.FunctionalTests/FileSystemRunners/FileSystemRunner.cs +++ b/GVFS/GVFS.FunctionalTests/FileSystemRunners/FileSystemRunner.cs @@ -46,6 +46,8 @@ public static FileSystemRunner DefaultRunner get { return defaultRunner; } } + public abstract bool SupportsHardlinkCreation { get; } + // File methods public abstract bool FileExists(string path); public abstract string MoveFile(string sourcePath, string targetPath); @@ -69,6 +71,8 @@ public static FileSystemRunner DefaultRunner public abstract void CreateEmptyFile(string path); + public abstract void CreateHardLink(string targetPath, string newLinkPath); + /// /// Write the specified contents to the specified file. By calling this method the caller is /// indicating that they expect the write to succeed. However, the caller is responsible for verifying that diff --git a/GVFS/GVFS.FunctionalTests/FileSystemRunners/PowerShellRunner.cs b/GVFS/GVFS.FunctionalTests/FileSystemRunners/PowerShellRunner.cs index 9d6a1da97..cd0652f9a 100644 --- a/GVFS/GVFS.FunctionalTests/FileSystemRunners/PowerShellRunner.cs +++ b/GVFS/GVFS.FunctionalTests/FileSystemRunners/PowerShellRunner.cs @@ -1,4 +1,5 @@ using GVFS.Tests.Should; +using NUnit.Framework; using System.IO; namespace GVFS.FunctionalTests.FileSystemRunners @@ -32,6 +33,11 @@ public class PowerShellRunner : ShellRunner "PermissionDenied" }; + public override bool SupportsHardlinkCreation + { + get { return false; } + } + protected override string FileName { get @@ -105,6 +111,11 @@ public override void CreateEmptyFile(string path) this.RunProcess(string.Format("-Command \"&{{ New-Item -ItemType file {0}}}\"", path)); } + public override void CreateHardLink(string targetPath, string newLinkPath) + { + Assert.Fail($"{nameof(PowerShellRunner)} does not support {nameof(this.CreateHardLink)}"); + } + public override void WriteAllText(string path, string contents) { this.RunProcess(string.Format("-Command \"&{{ Out-File -FilePath {0} -InputObject '{1}' -Encoding ascii -NoNewline}}\"", path, contents)); diff --git a/GVFS/GVFS.FunctionalTests/FileSystemRunners/SystemIORunner.cs b/GVFS/GVFS.FunctionalTests/FileSystemRunners/SystemIORunner.cs index 8ad8bdffe..53325a8e9 100644 --- a/GVFS/GVFS.FunctionalTests/FileSystemRunners/SystemIORunner.cs +++ b/GVFS/GVFS.FunctionalTests/FileSystemRunners/SystemIORunner.cs @@ -9,6 +9,11 @@ namespace GVFS.FunctionalTests.FileSystemRunners { public class SystemIORunner : FileSystemRunner { + public override bool SupportsHardlinkCreation + { + get { return false; } + } + public override bool FileExists(string path) { return File.Exists(path); @@ -75,6 +80,11 @@ public override void CreateEmptyFile(string path) } } + public override void CreateHardLink(string targetPath, string newLinkPath) + { + Assert.Fail($"{nameof(SystemIORunner)} does not support {nameof(this.CreateHardLink)}"); + } + public override void WriteAllText(string path, string contents) { File.WriteAllText(path, contents); diff --git a/GVFS/GVFS.FunctionalTests/Tests/EnlistmentPerFixture/GitFilesTests.cs b/GVFS/GVFS.FunctionalTests/Tests/EnlistmentPerFixture/GitFilesTests.cs index 7abf164af..e18ef52db 100644 --- a/GVFS/GVFS.FunctionalTests/Tests/EnlistmentPerFixture/GitFilesTests.cs +++ b/GVFS/GVFS.FunctionalTests/Tests/EnlistmentPerFixture/GitFilesTests.cs @@ -38,10 +38,35 @@ public void CreateFileTest() this.fileSystem.CreateEmptyFile(this.Enlistment.GetVirtualPathTo(emptyFileName)); this.Enlistment.WaitForBackgroundOperations().ShouldEqual(true, "Background operations failed to complete."); GVFSHelpers.ModifiedPathsShouldContain(this.fileSystem, this.Enlistment.DotGVFSRoot, emptyFileName); - this.Enlistment.GetVirtualPathTo(fileName).ShouldBeAFile(this.fileSystem); + this.Enlistment.GetVirtualPathTo(emptyFileName).ShouldBeAFile(this.fileSystem); } [TestCase, Order(2)] + public void CreateHardLinkTest() + { + if (!this.fileSystem.SupportsHardlinkCreation) + { + return; + } + + string targetFileName = "hardLinkTarget.txt"; + string targetFilePath = this.Enlistment.GetVirtualPathTo(targetFileName); + GVFSHelpers.ModifiedPathsShouldNotContain(this.fileSystem, this.Enlistment.DotGVFSRoot, targetFileName); + this.fileSystem.WriteAllText(targetFilePath, "Some content here"); + this.Enlistment.WaitForBackgroundOperations().ShouldEqual(true, "Background operations failed to complete."); + GVFSHelpers.ModifiedPathsShouldContain(this.fileSystem, this.Enlistment.DotGVFSRoot, targetFileName); + targetFilePath.ShouldBeAFile(this.fileSystem).WithContents("Some content here"); + + string linkFileName = "hardLink.txt"; + string linkFilePath = this.Enlistment.GetVirtualPathTo(linkFileName); + GVFSHelpers.ModifiedPathsShouldNotContain(this.fileSystem, this.Enlistment.DotGVFSRoot, linkFileName); + this.fileSystem.CreateHardLink(targetFilePath, linkFilePath); + this.Enlistment.WaitForBackgroundOperations().ShouldEqual(true, "Background operations failed to complete."); + GVFSHelpers.ModifiedPathsShouldContain(this.fileSystem, this.Enlistment.DotGVFSRoot, linkFileName); + linkFilePath.ShouldBeAFile(this.fileSystem).WithContents("Some content here"); + } + + [TestCase, Order(3)] [Category(Categories.Mac.M2TODO)] public void CreateFileInFolderTest() { @@ -62,7 +87,7 @@ public void CreateFileInFolderTest() GVFSHelpers.ModifiedPathsShouldContain(this.fileSystem, this.Enlistment.DotGVFSRoot, folderName + "/" + fileName); } - [TestCase, Order(3)] + [TestCase, Order(4)] [Category(Categories.Mac.M2TODO)] public void RenameEmptyFolderTest() { @@ -83,7 +108,7 @@ public void RenameEmptyFolderTest() GVFSHelpers.ModifiedPathsShouldContain(this.fileSystem, this.Enlistment.DotGVFSRoot, expectedModifiedEntries); } - [TestCase, Order(4)] + [TestCase, Order(5)] [Category(Categories.Mac.M2TODO)] public void RenameFolderTest() { @@ -116,7 +141,7 @@ public void RenameFolderTest() GVFSHelpers.ModifiedPathsShouldContain(this.fileSystem, this.Enlistment.DotGVFSRoot, expectedModifiedEntries); } - [TestCase, Order(5)] + [TestCase, Order(6)] [Category(Categories.Mac.M2TODO)] public void CaseOnlyRenameOfNewFolderKeepsExcludeEntries() { @@ -138,7 +163,7 @@ public void CaseOnlyRenameOfNewFolderKeepsExcludeEntries() GVFSHelpers.ModifiedPathsShouldContain(this.fileSystem, this.Enlistment.DotGVFSRoot, expectedModifiedPathsEntries); } - [TestCase, Order(6)] + [TestCase, Order(7)] public void ReadingFileDoesNotUpdateIndexOrSparseCheckout() { string gitFileToCheck = "GVFS/GVFS.FunctionalTests/Category/CategoryConstants.cs"; @@ -166,7 +191,7 @@ public void ReadingFileDoesNotUpdateIndexOrSparseCheckout() } // TODO(Mac): Enable this test once the LockHolder is converted to .NET Core - [TestCase, Order(7)] + [TestCase, Order(8)] [Category(Categories.Mac.M2TODO)] public void ModifiedFileWillGetAddedToModifiedPathsFile() { @@ -186,7 +211,7 @@ public void ModifiedFileWillGetAddedToModifiedPathsFile() this.VerifyWorktreeBit(gitFileToTest, LsFilesStatus.Cached); } - [TestCase, Order(8)] + [TestCase, Order(9)] public void RenamedFileAddedToModifiedPathsFile() { string fileToRenameEntry = "Test_EPF_MoveRenameFileTests/ChangeUnhydratedFileName/Program.cs"; @@ -205,7 +230,7 @@ public void RenamedFileAddedToModifiedPathsFile() this.VerifyWorktreeBit(fileToRenameEntry, LsFilesStatus.Cached); } - [TestCase, Order(9)] + [TestCase, Order(10)] public void RenamedFileAndOverwrittenTargetAddedToModifiedPathsFile() { string fileToRenameEntry = "Test_EPF_MoveRenameFileTests_2/MoveUnhydratedFileToOverwriteUnhydratedFileAndWrite/RunUnitTests.bat"; @@ -226,7 +251,7 @@ public void RenamedFileAndOverwrittenTargetAddedToModifiedPathsFile() this.VerifyWorktreeBit(fileToRenameTargetEntry, LsFilesStatus.Cached); } - [TestCase, Order(10)] + [TestCase, Order(11)] public void DeletedFileAddedToModifiedPathsFile() { string fileToDeleteEntry = "GVFlt_DeleteFileTest/GVFlt_DeleteFullFileWithoutFileContext_DeleteOnClose/a.txt"; @@ -241,7 +266,7 @@ public void DeletedFileAddedToModifiedPathsFile() this.VerifyWorktreeBit(fileToDeleteEntry, LsFilesStatus.Cached); } - [TestCase, Order(11)] + [TestCase, Order(12)] public void DeletedFolderAndChildrenAddedToToModifiedPathsFile() { string folderToDelete = "Scripts"; @@ -273,7 +298,7 @@ public void DeletedFolderAndChildrenAddedToToModifiedPathsFile() } } - [TestCase, Order(12)] + [TestCase, Order(13)] public void FileRenamedOutOfRepoAddedToModifiedPathsFile() { string fileToRenameEntry = "GVFlt_MoveFileTest/PartialToOutside/from/lessInFrom.txt"; @@ -292,7 +317,7 @@ public void FileRenamedOutOfRepoAddedToModifiedPathsFile() this.VerifyWorktreeBit(fileToRenameEntry, LsFilesStatus.Cached); } - [TestCase, Order(13)] + [TestCase, Order(14)] public void OverwrittenFileAddedToSparseCheckoutAndSkipWorktreeBitCleared() { string fileToOverwriteEntry = "Test_EPF_WorkingDirectoryTests/1/2/3/4/ReadDeepProjectedFile.cpp"; @@ -312,7 +337,7 @@ public void OverwrittenFileAddedToSparseCheckoutAndSkipWorktreeBitCleared() this.VerifyWorktreeBit(fileToOverwriteEntry, LsFilesStatus.Cached); } - [TestCase, Order(14)] + [TestCase, Order(15)] [Category(Categories.Mac.M2TODO)] public void SupersededFileAddedToSparseCheckoutAndSkipWorktreeBitCleared() { diff --git a/GVFS/GVFS.FunctionalTests/Tests/GitCommands/AddStageTests.cs b/GVFS/GVFS.FunctionalTests/Tests/GitCommands/AddStageTests.cs index f079abe29..9e9e755bc 100644 --- a/GVFS/GVFS.FunctionalTests/Tests/GitCommands/AddStageTests.cs +++ b/GVFS/GVFS.FunctionalTests/Tests/GitCommands/AddStageTests.cs @@ -32,12 +32,30 @@ public void StageBasicTest() } [TestCase, Order(3)] + [Category(Categories.Mac.M2)] + public void AddAndStageHardLinksTest() + { + if (!this.FileSystem.SupportsHardlinkCreation) + { + return; + } + + this.CreateHardLink("Readme.md", "ReadmeLink.md"); + this.ValidateGitCommand("add ReadmeLink.md"); + this.RunGitCommand("commit -m \"Created ReadmeLink.md\""); + + this.CreateHardLink("AuthoringTests.md", "AuthoringTestsLink.md"); + this.ValidateGitCommand("stage AuthoringTestsLink.md"); + this.RunGitCommand("commit -m \"Created AuthoringTestsLink.md\""); + } + + [TestCase, Order(4)] public void AddAllowsPlaceholderCreation() { this.CommandAllowsPlaceholderCreation("add", @"GVFS\GVFS\Program.cs"); } - [TestCase, Order(4)] + [TestCase, Order(5)] public void StageAllowsPlaceholderCreation() { this.CommandAllowsPlaceholderCreation("stage", @"GVFS\GVFS\App.config"); diff --git a/GVFS/GVFS.FunctionalTests/Tests/GitCommands/GitRepoTests.cs b/GVFS/GVFS.FunctionalTests/Tests/GitCommands/GitRepoTests.cs index 6dd862117..29c15d888 100644 --- a/GVFS/GVFS.FunctionalTests/Tests/GitCommands/GitRepoTests.cs +++ b/GVFS/GVFS.FunctionalTests/Tests/GitCommands/GitRepoTests.cs @@ -215,6 +215,16 @@ protected void EditFile(string filePath, string content) this.FileSystem.AppendAllText(controlFile, content); } + protected void CreateHardLink(string targetPath, string newLinkPath) + { + string virtualTargetFile = Path.Combine(this.Enlistment.RepoRoot, targetPath); + string controlTargetFile = Path.Combine(this.ControlGitRepo.RootPath, targetPath); + string virtualNewLinkFile = Path.Combine(this.Enlistment.RepoRoot, newLinkPath); + string controlNewLinkFile = Path.Combine(this.ControlGitRepo.RootPath, newLinkPath); + this.FileSystem.CreateHardLink(virtualTargetFile, virtualNewLinkFile); + this.FileSystem.CreateHardLink(controlTargetFile, controlNewLinkFile); + } + protected void SetFileAsReadOnly(string filePath) { string virtualFile = Path.Combine(this.Enlistment.RepoRoot, filePath); From 635f4d86376f352ca98f170016b26bfb105532ea Mon Sep 17 00:00:00 2001 From: William Baker Date: Tue, 21 Aug 2018 15:50:52 -0700 Subject: [PATCH 059/272] Cleanup after rebasing --- .../PrjFSKext/PrjFSKext/KauthHandler.cpp | 29 +++----- ProjFS.Mac/PrjFSLib/PrjFSLib.cpp | 71 ++++++++++--------- 2 files changed, 46 insertions(+), 54 deletions(-) diff --git a/ProjFS.Mac/PrjFSKext/PrjFSKext/KauthHandler.cpp b/ProjFS.Mac/PrjFSKext/PrjFSKext/KauthHandler.cpp index f7f9fc6ff..ab288448d 100644 --- a/ProjFS.Mac/PrjFSKext/PrjFSKext/KauthHandler.cpp +++ b/ProjFS.Mac/PrjFSKext/PrjFSKext/KauthHandler.cpp @@ -35,9 +35,9 @@ static int HandleFileOpOperation( static int GetPid(vfs_context_t context); static uint32_t ReadVNodeFileFlags(vnode_t vn, vfs_context_t context); -static inline bool FileFlagsBitIsSet(uint32_t fileFlags, uint32_t bit); -static inline bool FileIsFlaggedAsInRoot(vnode_t vnode, vfs_context_t context); -static inline bool ActionBitIsSet(kauth_action_t action, kauth_action_t mask); +static bool FileFlagsBitIsSet(uint32_t fileFlags, uint32_t bit); +static bool FileIsFlaggedAsInRoot(vnode_t vnode, vfs_context_t context); +static bool ActionBitIsSet(kauth_action_t action, kauth_action_t mask); static bool IsFileSystemCrawler(char* procname); @@ -394,17 +394,6 @@ static int HandleFileOpOperation( goto CleanupAndReturn; } - vtype vnodeType = vnode_vtype(currentVnodeFromPath); - if (ShouldIgnoreVnodeType(vnodeType, currentVnodeFromPath)) - { - goto CleanupAndReturn; - } - - if (!HasAncestorFlaggedAsInRoot(currentVnodeFromPath, context)) - { - goto CleanupAndReturn; - } - VirtualizationRoot* root = nullptr; int pid; if (!ShouldHandleFileOpEvent( @@ -450,12 +439,6 @@ static int HandleFileOpOperation( // arg1 is the (const char *) path int closeFlags = static_cast(arg2); - vtype vnodeType = vnode_vtype(currentVnode); - if (ShouldIgnoreVnodeType(vnodeType, currentVnode)) - { - goto CleanupAndReturn; - } - if (vnode_isdir(currentVnode)) { goto CleanupAndReturn; @@ -630,6 +613,12 @@ static bool ShouldHandleFileOpEvent( VirtualizationRoot** root, int* pid) { + vtype vnodeType = vnode_vtype(vnode); + if (ShouldIgnoreVnodeType(vnodeType, vnode)) + { + return false; + } + *root = VirtualizationRoots_FindForVnode(vnode); if (nullptr == *root) { diff --git a/ProjFS.Mac/PrjFSLib/PrjFSLib.cpp b/ProjFS.Mac/PrjFSLib/PrjFSLib.cpp index 36be88ef3..79da6c482 100644 --- a/ProjFS.Mac/PrjFSLib/PrjFSLib.cpp +++ b/ProjFS.Mac/PrjFSLib/PrjFSLib.cpp @@ -44,6 +44,8 @@ template static bool InitializeEmptyPlaceholder(const cha static bool AddXAttr(const char* path, const char* name, const void* value, size_t size); static bool GetXAttr(const char* path, const char* name, size_t size, _Out_ void* value); +static PrjFS_NotificationType KUMessageTypeToNotificationType(MessageType kuNotificationType); + static bool IsVirtualizationRoot(const char* path); static void CombinePaths(const char* root, const char* relative, char (&combined)[PrjFSMaxPath]); @@ -422,15 +424,6 @@ static void HandleKernelRequest(Message request, void* messageMemory) } case MessageType_KtoU_NotifyFileModified: - { - result = HandleFileNotification( - requestHeader, - request.path, - false, // isDirectory - PrjFS_NotificationType_FileModified); - break; - } - case MessageType_KtoU_NotifyFilePreDelete: case MessageType_KtoU_NotifyDirectoryPreDelete: { @@ -438,24 +431,11 @@ static void HandleKernelRequest(Message request, void* messageMemory) requestHeader, request.path, requestHeader->messageType == MessageType_KtoU_NotifyDirectoryPreDelete, // isDirectory - PrjFS_NotificationType_PreDelete); + KUMessageTypeToNotificationType(static_cast(requestHeader->messageType))); break; } case MessageType_KtoU_NotifyFileCreated: - { - char fullPath[PrjFSMaxPath]; - CombinePaths(s_virtualizationRootFullPath.c_str(), request.path, fullPath); - SetBitInFileFlags(fullPath, FileFlags_IsInVirtualizationRoot, true); - - result = HandleFileNotification( - requestHeader, - request.path, - false, // isDirectory - PrjFS_NotificationType_NewFileCreated); - break; - } - case MessageType_KtoU_NotifyFileRenamed: case MessageType_KtoU_NotifyDirectoryRenamed: case MessageType_KtoU_NotifyFileHardLinkCreated: @@ -467,21 +447,11 @@ static void HandleKernelRequest(Message request, void* messageMemory) SetBitInFileFlags(fullPath, FileFlags_IsInVirtualizationRoot, true); bool isDirectory = requestHeader->messageType == MessageType_KtoU_NotifyDirectoryRenamed; - PrjFS_NotificationType notificationType; - if (requestHeader->messageType == MessageType_KtoU_NotifyFileHardLinkCreated) - { - notificationType = PrjFS_NotificationType_HardLinkCreated; - } - else - { - notificationType = PrjFS_NotificationType_FileRenamed; - } - result = HandleFileNotification( requestHeader, request.path, isDirectory, - notificationType); + KUMessageTypeToNotificationType(static_cast(requestHeader->messageType))); break; } } @@ -744,6 +714,39 @@ static bool GetXAttr(const char* path, const char* name, size_t size, _Out_ void return false; } +static inline PrjFS_NotificationType KUMessageTypeToNotificationType(MessageType kuNotificationType) +{ + switch(kuNotificationType) + { + case MessageType_KtoU_NotifyFileModified: + return PrjFS_NotificationType_FileModified; + + case MessageType_KtoU_NotifyFilePreDelete: + case MessageType_KtoU_NotifyDirectoryPreDelete: + return PrjFS_NotificationType_PreDelete; + + case MessageType_KtoU_NotifyFileCreated: + return PrjFS_NotificationType_NewFileCreated; + + case MessageType_KtoU_NotifyFileRenamed: + case MessageType_KtoU_NotifyDirectoryRenamed: + return PrjFS_NotificationType_FileRenamed; + + case MessageType_KtoU_NotifyFileHardLinkCreated: + return PrjFS_NotificationType_HardLinkCreated; + + // Non-notification types + case MessageType_Invalid: + case MessageType_UtoK_StartVirtualizationInstance: + case MessageType_UtoK_StopVirtualizationInstance: + case MessageType_KtoU_EnumerateDirectory: + case MessageType_KtoU_HydrateFile: + case MessageType_Response_Success: + case MessageType_Response_Fail: + return PrjFS_NotificationType_Invalid; + } +} + static errno_t SendKernelMessageResponse(uint64_t messageId, MessageType responseType) { const uint64_t inputs[] = { messageId, responseType }; From a5bea1ef291f8cf318d61a907d771b939f42c47b Mon Sep 17 00:00:00 2001 From: William Baker Date: Wed, 22 Aug 2018 11:56:21 -0700 Subject: [PATCH 060/272] Register for HardLinkCreated on Windows --- GVFS/GVFS.Platform.Windows/WindowsFileSystemVirtualizer.cs | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/GVFS/GVFS.Platform.Windows/WindowsFileSystemVirtualizer.cs b/GVFS/GVFS.Platform.Windows/WindowsFileSystemVirtualizer.cs index 65f5d67ad..5654a6a69 100644 --- a/GVFS/GVFS.Platform.Windows/WindowsFileSystemVirtualizer.cs +++ b/GVFS/GVFS.Platform.Windows/WindowsFileSystemVirtualizer.cs @@ -1365,19 +1365,23 @@ private class Notifications NotificationType.PreRename | NotificationType.PreDelete | NotificationType.FileRenamed | + NotificationType.HardlinkCreated | NotificationType.FileHandleClosedFileModified; public const NotificationType LogsHeadFile = - NotificationType.FileRenamed | + NotificationType.FileRenamed | + NotificationType.HardlinkCreated | NotificationType.FileHandleClosedFileModified; public const NotificationType ExcludeAndHeadFile = NotificationType.FileRenamed | + NotificationType.HardlinkCreated | NotificationType.FileHandleClosedFileDeleted | NotificationType.FileHandleClosedFileModified; public const NotificationType FilesAndFoldersInRefsHeads = NotificationType.FileRenamed | + NotificationType.HardlinkCreated | NotificationType.FileHandleClosedFileDeleted | NotificationType.FileHandleClosedFileModified; @@ -1385,6 +1389,7 @@ private class Notifications NotificationType.NewFileCreated | NotificationType.FileSupersededOrOverwritten | NotificationType.FileRenamed | + NotificationType.HardlinkCreated | NotificationType.FileHandleClosedFileDeleted | NotificationType.FilePreConvertToFull | NotificationType.FileHandleClosedFileModified; From b4fb4498270ca54b837ab0a73406af5285a03883 Mon Sep 17 00:00:00 2001 From: William Baker Date: Wed, 22 Aug 2018 14:16:18 -0700 Subject: [PATCH 061/272] PR Feedback: Code cleanup and test fixes --- .../FileSystem/IPlatformFileSystem.cs | 2 +- GVFS/GVFS.Common/NativeMethods.cs | 8 +-- .../FileSystemRunners/BashRunner.cs | 2 +- .../FileSystemRunners/FileSystemRunner.cs | 13 +++-- .../FileSystemRunners/PowerShellRunner.cs | 11 ---- .../FileSystemRunners/SystemIORunner.cs | 10 ---- .../EnlistmentPerFixture/GitFilesTests.cs | 51 +++++-------------- .../Tests/GitCommands/GitRepoTests.cs | 12 ++++- .../MacFileSystemVirtualizer.cs | 7 +-- .../WindowsFileSystemVirtualizer.cs | 6 +-- .../FileSystem/FileSystemVirtualizer.cs | 4 +- .../WindowsFileSystemVirtualizer.cs | 10 ++-- .../PrjFSKext/PrjFSKext/KauthHandler.cpp | 26 +++++----- ProjFS.Mac/PrjFSLib/PrjFSLib.cpp | 2 +- 14 files changed, 68 insertions(+), 96 deletions(-) diff --git a/GVFS/GVFS.Common/FileSystem/IPlatformFileSystem.cs b/GVFS/GVFS.Common/FileSystem/IPlatformFileSystem.cs index fd7f5ec37..93c8cb2e9 100644 --- a/GVFS/GVFS.Common/FileSystem/IPlatformFileSystem.cs +++ b/GVFS/GVFS.Common/FileSystem/IPlatformFileSystem.cs @@ -5,7 +5,7 @@ public interface IPlatformFileSystem bool SupportsFileMode { get; } void FlushFileBuffers(string path); void MoveAndOverwriteFile(string sourceFileName, string destinationFilename); - void CreateHardLink(string newFileName, string existingFileName); + void CreateHardLink(string newLinkFileName, string existingFileName); bool TryGetNormalizedPath(string path, out string normalizedPath, out string errorMessage); void ChangeMode(string path, int mode); } diff --git a/GVFS/GVFS.Common/NativeMethods.cs b/GVFS/GVFS.Common/NativeMethods.cs index c4564e44d..4f19f1e25 100644 --- a/GVFS/GVFS.Common/NativeMethods.cs +++ b/GVFS/GVFS.Common/NativeMethods.cs @@ -98,11 +98,11 @@ public static void MoveFile(string existingFileName, string newFileName, MoveFil } } - public static void CreateHardLink(string newFileName, string existingFileName) + public static void CreateHardLink(string newLinkFileName, string existingFileName) { - if (!CreateHardLink(newFileName, existingFileName, IntPtr.Zero)) + if (!CreateHardLink(newLinkFileName, existingFileName, IntPtr.Zero)) { - ThrowLastWin32Exception($"Failed to create hard link from '{newFileName}' to '{existingFileName}'"); + ThrowLastWin32Exception($"Failed to create hard link from '{newLinkFileName}' to '{existingFileName}'"); } } @@ -172,7 +172,7 @@ private static extern bool MoveFileEx( [DllImport("kernel32.dll", SetLastError = true, CharSet = CharSet.Unicode)] private static extern bool CreateHardLink( - string newFileName, + string newLinkFileName, string existingFileName, IntPtr securityAttributes); diff --git a/GVFS/GVFS.FunctionalTests/FileSystemRunners/BashRunner.cs b/GVFS/GVFS.FunctionalTests/FileSystemRunners/BashRunner.cs index 3a33258e0..e02bca468 100644 --- a/GVFS/GVFS.FunctionalTests/FileSystemRunners/BashRunner.cs +++ b/GVFS/GVFS.FunctionalTests/FileSystemRunners/BashRunner.cs @@ -166,7 +166,7 @@ public override void CreateEmptyFile(string path) public override void CreateHardLink(string existingPath, string newLinkPath) { string existingFileBashPath = this.ConvertWinPathToBashPath(existingPath); - string newLinkBashPath = this.ConvertWinPathToBashPath(existingPath); + string newLinkBashPath = this.ConvertWinPathToBashPath(newLinkPath); this.RunProcess(string.Format("-c \"ln {0} {1}\"", existingFileBashPath, newLinkBashPath)); } diff --git a/GVFS/GVFS.FunctionalTests/FileSystemRunners/FileSystemRunner.cs b/GVFS/GVFS.FunctionalTests/FileSystemRunners/FileSystemRunner.cs index b7c99068e..3a4a37f0e 100644 --- a/GVFS/GVFS.FunctionalTests/FileSystemRunners/FileSystemRunner.cs +++ b/GVFS/GVFS.FunctionalTests/FileSystemRunners/FileSystemRunner.cs @@ -1,4 +1,5 @@ -using System; +using NUnit.Framework; +using System; namespace GVFS.FunctionalTests.FileSystemRunners { @@ -46,7 +47,10 @@ public static FileSystemRunner DefaultRunner get { return defaultRunner; } } - public abstract bool SupportsHardlinkCreation { get; } + public virtual bool SupportsHardlinkCreation + { + get { return false; } + } // File methods public abstract bool FileExists(string path); @@ -71,7 +75,10 @@ public static FileSystemRunner DefaultRunner public abstract void CreateEmptyFile(string path); - public abstract void CreateHardLink(string targetPath, string newLinkPath); + public virtual void CreateHardLink(string targetPath, string newLinkPath) + { + Assert.Fail($"This runner does not support {nameof(this.CreateHardLink)}"); + } /// /// Write the specified contents to the specified file. By calling this method the caller is diff --git a/GVFS/GVFS.FunctionalTests/FileSystemRunners/PowerShellRunner.cs b/GVFS/GVFS.FunctionalTests/FileSystemRunners/PowerShellRunner.cs index cd0652f9a..9d6a1da97 100644 --- a/GVFS/GVFS.FunctionalTests/FileSystemRunners/PowerShellRunner.cs +++ b/GVFS/GVFS.FunctionalTests/FileSystemRunners/PowerShellRunner.cs @@ -1,5 +1,4 @@ using GVFS.Tests.Should; -using NUnit.Framework; using System.IO; namespace GVFS.FunctionalTests.FileSystemRunners @@ -33,11 +32,6 @@ public class PowerShellRunner : ShellRunner "PermissionDenied" }; - public override bool SupportsHardlinkCreation - { - get { return false; } - } - protected override string FileName { get @@ -111,11 +105,6 @@ public override void CreateEmptyFile(string path) this.RunProcess(string.Format("-Command \"&{{ New-Item -ItemType file {0}}}\"", path)); } - public override void CreateHardLink(string targetPath, string newLinkPath) - { - Assert.Fail($"{nameof(PowerShellRunner)} does not support {nameof(this.CreateHardLink)}"); - } - public override void WriteAllText(string path, string contents) { this.RunProcess(string.Format("-Command \"&{{ Out-File -FilePath {0} -InputObject '{1}' -Encoding ascii -NoNewline}}\"", path, contents)); diff --git a/GVFS/GVFS.FunctionalTests/FileSystemRunners/SystemIORunner.cs b/GVFS/GVFS.FunctionalTests/FileSystemRunners/SystemIORunner.cs index 53325a8e9..8ad8bdffe 100644 --- a/GVFS/GVFS.FunctionalTests/FileSystemRunners/SystemIORunner.cs +++ b/GVFS/GVFS.FunctionalTests/FileSystemRunners/SystemIORunner.cs @@ -9,11 +9,6 @@ namespace GVFS.FunctionalTests.FileSystemRunners { public class SystemIORunner : FileSystemRunner { - public override bool SupportsHardlinkCreation - { - get { return false; } - } - public override bool FileExists(string path) { return File.Exists(path); @@ -80,11 +75,6 @@ public override void CreateEmptyFile(string path) } } - public override void CreateHardLink(string targetPath, string newLinkPath) - { - Assert.Fail($"{nameof(SystemIORunner)} does not support {nameof(this.CreateHardLink)}"); - } - public override void WriteAllText(string path, string contents) { File.WriteAllText(path, contents); diff --git a/GVFS/GVFS.FunctionalTests/Tests/EnlistmentPerFixture/GitFilesTests.cs b/GVFS/GVFS.FunctionalTests/Tests/EnlistmentPerFixture/GitFilesTests.cs index e18ef52db..72e3fab84 100644 --- a/GVFS/GVFS.FunctionalTests/Tests/EnlistmentPerFixture/GitFilesTests.cs +++ b/GVFS/GVFS.FunctionalTests/Tests/EnlistmentPerFixture/GitFilesTests.cs @@ -3,7 +3,6 @@ using GVFS.FunctionalTests.Tools; using GVFS.Tests.Should; using NUnit.Framework; -using System; using System.Collections.Generic; using System.IO; using System.Linq; @@ -12,7 +11,7 @@ namespace GVFS.FunctionalTests.Tests.EnlistmentPerFixture { - [TestFixtureSource(typeof(GitFilesTestsRunners), GitFilesTestsRunners.TestRunners)] + [TestFixtureSource(typeof(FileSystemRunner), FileSystemRunner.TestRunners)] [Category(Categories.Mac.M2)] public class GitFilesTests : TestsWithEnlistmentPerFixture { @@ -49,21 +48,21 @@ public void CreateHardLinkTest() return; } - string targetFileName = "hardLinkTarget.txt"; - string targetFilePath = this.Enlistment.GetVirtualPathTo(targetFileName); - GVFSHelpers.ModifiedPathsShouldNotContain(this.fileSystem, this.Enlistment.DotGVFSRoot, targetFileName); - this.fileSystem.WriteAllText(targetFilePath, "Some content here"); + string existingFileName = "fileToLinkTo.txt"; + string existingFilePath = this.Enlistment.GetVirtualPathTo(existingFileName); + GVFSHelpers.ModifiedPathsShouldNotContain(this.fileSystem, this.Enlistment.DotGVFSRoot, existingFileName); + this.fileSystem.WriteAllText(existingFilePath, "Some content here"); this.Enlistment.WaitForBackgroundOperations().ShouldEqual(true, "Background operations failed to complete."); - GVFSHelpers.ModifiedPathsShouldContain(this.fileSystem, this.Enlistment.DotGVFSRoot, targetFileName); - targetFilePath.ShouldBeAFile(this.fileSystem).WithContents("Some content here"); + GVFSHelpers.ModifiedPathsShouldContain(this.fileSystem, this.Enlistment.DotGVFSRoot, existingFileName); + existingFilePath.ShouldBeAFile(this.fileSystem).WithContents("Some content here"); - string linkFileName = "hardLink.txt"; - string linkFilePath = this.Enlistment.GetVirtualPathTo(linkFileName); - GVFSHelpers.ModifiedPathsShouldNotContain(this.fileSystem, this.Enlistment.DotGVFSRoot, linkFileName); - this.fileSystem.CreateHardLink(targetFilePath, linkFilePath); + string newLinkFileName = "newHardLink.txt"; + string newLinkFilePath = this.Enlistment.GetVirtualPathTo(newLinkFileName); + GVFSHelpers.ModifiedPathsShouldNotContain(this.fileSystem, this.Enlistment.DotGVFSRoot, newLinkFileName); + this.fileSystem.CreateHardLink(existingFilePath, newLinkFilePath); this.Enlistment.WaitForBackgroundOperations().ShouldEqual(true, "Background operations failed to complete."); - GVFSHelpers.ModifiedPathsShouldContain(this.fileSystem, this.Enlistment.DotGVFSRoot, linkFileName); - linkFilePath.ShouldBeAFile(this.fileSystem).WithContents("Some content here"); + GVFSHelpers.ModifiedPathsShouldContain(this.fileSystem, this.Enlistment.DotGVFSRoot, newLinkFileName); + newLinkFilePath.ShouldBeAFile(this.fileSystem).WithContents("Some content here"); } [TestCase, Order(3)] @@ -376,29 +375,5 @@ private static class LsFilesStatus public const char Cached = 'H'; public const char SkipWorktree = 'S'; } - - private class GitFilesTestsRunners - { - public const string TestRunners = "Runners"; - - public static object[] Runners - { - get - { - // Don't use the BashRunner for GitFilesTests as the BashRunner always strips off the last trailing newline (\n) - // and we expect there to be a trailing new line - List runners = new List(); - foreach (object[] runner in FileSystemRunner.Runners.ToList()) - { - if (!(runner.ToList().First() is BashRunner)) - { - runners.Add(new object[] { runner.ToList().First() }); - } - } - - return runners.ToArray(); - } - } - } } } diff --git a/GVFS/GVFS.FunctionalTests/Tests/GitCommands/GitRepoTests.cs b/GVFS/GVFS.FunctionalTests/Tests/GitCommands/GitRepoTests.cs index 29c15d888..f19ee2ee3 100644 --- a/GVFS/GVFS.FunctionalTests/Tests/GitCommands/GitRepoTests.cs +++ b/GVFS/GVFS.FunctionalTests/Tests/GitCommands/GitRepoTests.cs @@ -221,8 +221,16 @@ protected void CreateHardLink(string targetPath, string newLinkPath) string controlTargetFile = Path.Combine(this.ControlGitRepo.RootPath, targetPath); string virtualNewLinkFile = Path.Combine(this.Enlistment.RepoRoot, newLinkPath); string controlNewLinkFile = Path.Combine(this.ControlGitRepo.RootPath, newLinkPath); - this.FileSystem.CreateHardLink(virtualTargetFile, virtualNewLinkFile); - this.FileSystem.CreateHardLink(controlTargetFile, controlNewLinkFile); + + // GitRepoTests are only run with SystemIORunner (which does not support hardlink + // creation) so use a BashRunner instead. + this.FileSystem.ShouldBeOfType(); + this.FileSystem.SupportsHardlinkCreation.ShouldBeFalse( + "FileSystem *does* support hard link creation, CreateHardLink no longer needs to create a BashRunner"); + FileSystemRunner runner = new BashRunner(); + + runner.CreateHardLink(virtualTargetFile, virtualNewLinkFile); + runner.CreateHardLink(controlTargetFile, controlNewLinkFile); } protected void SetFileAsReadOnly(string filePath) diff --git a/GVFS/GVFS.Platform.Mac/MacFileSystemVirtualizer.cs b/GVFS/GVFS.Platform.Mac/MacFileSystemVirtualizer.cs index 330dad5ad..f358f07bb 100644 --- a/GVFS/GVFS.Platform.Mac/MacFileSystemVirtualizer.cs +++ b/GVFS/GVFS.Platform.Mac/MacFileSystemVirtualizer.cs @@ -352,8 +352,9 @@ private void OnNewFileCreated(string relativePath, bool isDirectory) private void OnFileRenamed(string relativeDestinationPath, bool isDirectory) { - // relativeSourcePath is handled in the OnPreDelete callback that's triggered - // prior to OnFileRenamed + // ProjFS for Mac *could* be updated to provide us with relativeSourcePath as well, + // but because VFSForGit doesn't need the source path on Mac for correct behavior + // the relativeSourcePath is left out of the notification to keep the kext simple this.OnFileRenamed( relativeSourcePath: string.Empty, relativeDestinationPath: relativeDestinationPath, @@ -363,7 +364,7 @@ private void OnFileRenamed(string relativeDestinationPath, bool isDirectory) private void OnHardLinkCreated(string relativeNewLinkPath) { this.OnHardLinkCreated( - relativeTargetPath: string.Empty, + relativeExistingFilePath: string.Empty, relativeNewLinkPath: relativeNewLinkPath); } diff --git a/GVFS/GVFS.Platform.Windows/WindowsFileSystemVirtualizer.cs b/GVFS/GVFS.Platform.Windows/WindowsFileSystemVirtualizer.cs index 5654a6a69..6503194c0 100644 --- a/GVFS/GVFS.Platform.Windows/WindowsFileSystemVirtualizer.cs +++ b/GVFS/GVFS.Platform.Windows/WindowsFileSystemVirtualizer.cs @@ -1227,10 +1227,10 @@ private void NotifyFileRenamedHandler( } private void NotifyHardlinkCreated( - string relativePath, - string destinationPath) + string relativeExistingFilePath, + string relativeNewLinkPath) { - this.OnHardLinkCreated(relativePath, destinationPath); + this.OnHardLinkCreated(relativeExistingFilePath, relativeNewLinkPath); } private void NotifyFileHandleClosedFileModifiedOrDeletedHandler( diff --git a/GVFS/GVFS.Virtualization/FileSystem/FileSystemVirtualizer.cs b/GVFS/GVFS.Virtualization/FileSystem/FileSystemVirtualizer.cs index 739c91f54..4357d85a4 100644 --- a/GVFS/GVFS.Virtualization/FileSystem/FileSystemVirtualizer.cs +++ b/GVFS/GVFS.Virtualization/FileSystem/FileSystemVirtualizer.cs @@ -242,7 +242,7 @@ protected void OnFileRenamed(string relativeSourcePath, string relativeDestinati } } - protected void OnHardLinkCreated(string relativeTargetPath, string relativeNewLinkPath) + protected void OnHardLinkCreated(string relativeExistingFilePath, string relativeNewLinkPath) { try { @@ -260,7 +260,7 @@ protected void OnHardLinkCreated(string relativeTargetPath, string relativeNewLi catch (Exception e) { EventMetadata metadata = this.CreateEventMetadata(relativeNewLinkPath, e); - metadata.Add(nameof(relativeTargetPath), relativeTargetPath); + metadata.Add(nameof(relativeExistingFilePath), relativeExistingFilePath); this.LogUnhandledExceptionAndExit(nameof(this.OnHardLinkCreated), metadata); } } diff --git a/MirrorProvider/MirrorProvider.Windows/WindowsFileSystemVirtualizer.cs b/MirrorProvider/MirrorProvider.Windows/WindowsFileSystemVirtualizer.cs index edfd0acf1..5100884cf 100644 --- a/MirrorProvider/MirrorProvider.Windows/WindowsFileSystemVirtualizer.cs +++ b/MirrorProvider/MirrorProvider.Windows/WindowsFileSystemVirtualizer.cs @@ -321,19 +321,19 @@ private void OnFileModifiedOrDeleted(string relativePath, bool isDirectory, bool } private void OnFileRenamed( - string relativePath, + string relativeSourcePath, string relativeDestinationPath, bool isDirectory, ref NotificationType notificationMask) { - Console.WriteLine($"OnFileRenamed (isDirectory: {isDirectory}), relativePath: {relativePath}, relativeDestinationPath: {relativeDestinationPath}"); + Console.WriteLine($"OnFileRenamed (isDirectory: {isDirectory}), relativeSourcePath: {relativeSourcePath}, relativeDestinationPath: {relativeDestinationPath}"); } private void OnHardlinkCreated( - string relativePath, - string relativeDestinationPath) + string relativeExistingFilePath, + string relativeNewLinkFilePath) { - Console.WriteLine($"OnHardlinkCreated, relativePath: {relativePath}, relativeDestinationPath: {relativeDestinationPath}"); + Console.WriteLine($"OnHardlinkCreated, relativeExistingFilePath: {relativeExistingFilePath}, relativeNewLinkFilePath: {relativeNewLinkFilePath}"); } // TODO: Add this to the ProjFS API diff --git a/ProjFS.Mac/PrjFSKext/PrjFSKext/KauthHandler.cpp b/ProjFS.Mac/PrjFSKext/PrjFSKext/KauthHandler.cpp index ab288448d..c5d749782 100644 --- a/ProjFS.Mac/PrjFSKext/PrjFSKext/KauthHandler.cpp +++ b/ProjFS.Mac/PrjFSKext/PrjFSKext/KauthHandler.cpp @@ -35,9 +35,9 @@ static int HandleFileOpOperation( static int GetPid(vfs_context_t context); static uint32_t ReadVNodeFileFlags(vnode_t vn, vfs_context_t context); -static bool FileFlagsBitIsSet(uint32_t fileFlags, uint32_t bit); -static bool FileIsFlaggedAsInRoot(vnode_t vnode, vfs_context_t context); -static bool ActionBitIsSet(kauth_action_t action, kauth_action_t mask); +static inline bool FileFlagsBitIsSet(uint32_t fileFlags, uint32_t bit); +static inline bool FileIsFlaggedAsInRoot(vnode_t vnode, vfs_context_t context); +static inline bool ActionBitIsSet(kauth_action_t action, kauth_action_t mask); static bool IsFileSystemCrawler(char* procname); @@ -341,7 +341,8 @@ static int HandleVnodeOperation( KAUTH_VNODE_WRITE_EXTATTRIBUTES | KAUTH_VNODE_READ_DATA | KAUTH_VNODE_WRITE_DATA | - KAUTH_VNODE_EXECUTE)) + KAUTH_VNODE_EXECUTE | + KAUTH_VNODE_DELETE)) // Hydrate on delete to ensure files are hydrated before rename operations { if (FileFlagsBitIsSet(currentVnodeFileFlags, FileFlags_IsEmpty)) { @@ -387,7 +388,8 @@ static int HandleFileOpOperation( // arg0 is the (const char *) fromPath (or the file being linked to) const char* newPath = (const char*)arg1; - // TODO(Mac): Improve error handling + // TODO(Mac): We need to handle failures to lookup the vnode. If we fail to lookup the vnode + // it's possible that we'll miss notifications errno_t toErr = vnode_lookup(newPath, 0 /* flags */, ¤tVnodeFromPath, context); if (0 != toErr) { @@ -422,13 +424,13 @@ static int HandleFileOpOperation( int kauthResult; int kauthError; if (!TrySendRequestAndWaitForResponse( - root, - messageType, - currentVnodeFromPath, - pid, - procname, - &kauthResult, - &kauthError)) + root, + messageType, + currentVnodeFromPath, + pid, + procname, + &kauthResult, + &kauthError)) { goto CleanupAndReturn; } diff --git a/ProjFS.Mac/PrjFSLib/PrjFSLib.cpp b/ProjFS.Mac/PrjFSLib/PrjFSLib.cpp index 79da6c482..dd8ab7cec 100644 --- a/ProjFS.Mac/PrjFSLib/PrjFSLib.cpp +++ b/ProjFS.Mac/PrjFSLib/PrjFSLib.cpp @@ -44,7 +44,7 @@ template static bool InitializeEmptyPlaceholder(const cha static bool AddXAttr(const char* path, const char* name, const void* value, size_t size); static bool GetXAttr(const char* path, const char* name, size_t size, _Out_ void* value); -static PrjFS_NotificationType KUMessageTypeToNotificationType(MessageType kuNotificationType); +static inline PrjFS_NotificationType KUMessageTypeToNotificationType(MessageType kuNotificationType); static bool IsVirtualizationRoot(const char* path); static void CombinePaths(const char* root, const char* relative, char (&combined)[PrjFSMaxPath]); From 4e90178a079d5d12409d4b75a97cbac61d82de79 Mon Sep 17 00:00:00 2001 From: William Baker Date: Thu, 23 Aug 2018 10:45:30 -0700 Subject: [PATCH 062/272] More cleanup for PR comments --- .../FileSystemRunners/BashRunner.cs | 6 +++--- .../FileSystemRunners/CmdRunner.cs | 4 ++-- .../FileSystemRunners/FileSystemRunner.cs | 2 +- .../Tests/EnlistmentPerFixture/GitFilesTests.cs | 2 +- .../Tests/GitCommands/AddStageTests.cs | 4 ++-- .../Tests/GitCommands/GitRepoTests.cs | 17 ++++++++--------- 6 files changed, 17 insertions(+), 18 deletions(-) diff --git a/GVFS/GVFS.FunctionalTests/FileSystemRunners/BashRunner.cs b/GVFS/GVFS.FunctionalTests/FileSystemRunners/BashRunner.cs index e02bca468..40f2fbc12 100644 --- a/GVFS/GVFS.FunctionalTests/FileSystemRunners/BashRunner.cs +++ b/GVFS/GVFS.FunctionalTests/FileSystemRunners/BashRunner.cs @@ -163,10 +163,10 @@ public override void CreateEmptyFile(string path) this.RunProcess(string.Format("-c \"touch {0}\"", bashPath)); } - public override void CreateHardLink(string existingPath, string newLinkPath) + public override void CreateHardLink(string newLinkFilePath, string existingFilePath) { - string existingFileBashPath = this.ConvertWinPathToBashPath(existingPath); - string newLinkBashPath = this.ConvertWinPathToBashPath(newLinkPath); + string existingFileBashPath = this.ConvertWinPathToBashPath(existingFilePath); + string newLinkBashPath = this.ConvertWinPathToBashPath(newLinkFilePath); this.RunProcess(string.Format("-c \"ln {0} {1}\"", existingFileBashPath, newLinkBashPath)); } diff --git a/GVFS/GVFS.FunctionalTests/FileSystemRunners/CmdRunner.cs b/GVFS/GVFS.FunctionalTests/FileSystemRunners/CmdRunner.cs index cf66c5f87..cb5184c06 100644 --- a/GVFS/GVFS.FunctionalTests/FileSystemRunners/CmdRunner.cs +++ b/GVFS/GVFS.FunctionalTests/FileSystemRunners/CmdRunner.cs @@ -113,9 +113,9 @@ public override void CreateEmptyFile(string path) this.RunProcess(string.Format("/C type NUL > \"{0}\"", path)); } - public override void CreateHardLink(string targetPath, string newLinkPath) + public override void CreateHardLink(string newLinkFilePath, string existingFilePath) { - this.RunProcess(string.Format("/C mklink /H \"{0}\" \"{1}\"", newLinkPath, targetPath)); + this.RunProcess(string.Format("/C mklink /H \"{0}\" \"{1}\"", newLinkFilePath, existingFilePath)); } public override void AppendAllText(string path, string contents) diff --git a/GVFS/GVFS.FunctionalTests/FileSystemRunners/FileSystemRunner.cs b/GVFS/GVFS.FunctionalTests/FileSystemRunners/FileSystemRunner.cs index 3a4a37f0e..b8a2f9cc6 100644 --- a/GVFS/GVFS.FunctionalTests/FileSystemRunners/FileSystemRunner.cs +++ b/GVFS/GVFS.FunctionalTests/FileSystemRunners/FileSystemRunner.cs @@ -75,7 +75,7 @@ public virtual bool SupportsHardlinkCreation public abstract void CreateEmptyFile(string path); - public virtual void CreateHardLink(string targetPath, string newLinkPath) + public virtual void CreateHardLink(string newLinkFilePath, string existingFilePath) { Assert.Fail($"This runner does not support {nameof(this.CreateHardLink)}"); } diff --git a/GVFS/GVFS.FunctionalTests/Tests/EnlistmentPerFixture/GitFilesTests.cs b/GVFS/GVFS.FunctionalTests/Tests/EnlistmentPerFixture/GitFilesTests.cs index 72e3fab84..5ec2ee3b3 100644 --- a/GVFS/GVFS.FunctionalTests/Tests/EnlistmentPerFixture/GitFilesTests.cs +++ b/GVFS/GVFS.FunctionalTests/Tests/EnlistmentPerFixture/GitFilesTests.cs @@ -59,7 +59,7 @@ public void CreateHardLinkTest() string newLinkFileName = "newHardLink.txt"; string newLinkFilePath = this.Enlistment.GetVirtualPathTo(newLinkFileName); GVFSHelpers.ModifiedPathsShouldNotContain(this.fileSystem, this.Enlistment.DotGVFSRoot, newLinkFileName); - this.fileSystem.CreateHardLink(existingFilePath, newLinkFilePath); + this.fileSystem.CreateHardLink(newLinkFilePath, existingFilePath); this.Enlistment.WaitForBackgroundOperations().ShouldEqual(true, "Background operations failed to complete."); GVFSHelpers.ModifiedPathsShouldContain(this.fileSystem, this.Enlistment.DotGVFSRoot, newLinkFileName); newLinkFilePath.ShouldBeAFile(this.fileSystem).WithContents("Some content here"); diff --git a/GVFS/GVFS.FunctionalTests/Tests/GitCommands/AddStageTests.cs b/GVFS/GVFS.FunctionalTests/Tests/GitCommands/AddStageTests.cs index 9e9e755bc..ac38b5e19 100644 --- a/GVFS/GVFS.FunctionalTests/Tests/GitCommands/AddStageTests.cs +++ b/GVFS/GVFS.FunctionalTests/Tests/GitCommands/AddStageTests.cs @@ -40,11 +40,11 @@ public void AddAndStageHardLinksTest() return; } - this.CreateHardLink("Readme.md", "ReadmeLink.md"); + this.CreateHardLink("ReadmeLink.md", "Readme.md"); this.ValidateGitCommand("add ReadmeLink.md"); this.RunGitCommand("commit -m \"Created ReadmeLink.md\""); - this.CreateHardLink("AuthoringTests.md", "AuthoringTestsLink.md"); + this.CreateHardLink("AuthoringTestsLink.md", "AuthoringTests.md"); this.ValidateGitCommand("stage AuthoringTestsLink.md"); this.RunGitCommand("commit -m \"Created AuthoringTestsLink.md\""); } diff --git a/GVFS/GVFS.FunctionalTests/Tests/GitCommands/GitRepoTests.cs b/GVFS/GVFS.FunctionalTests/Tests/GitCommands/GitRepoTests.cs index f19ee2ee3..89fe43289 100644 --- a/GVFS/GVFS.FunctionalTests/Tests/GitCommands/GitRepoTests.cs +++ b/GVFS/GVFS.FunctionalTests/Tests/GitCommands/GitRepoTests.cs @@ -215,22 +215,21 @@ protected void EditFile(string filePath, string content) this.FileSystem.AppendAllText(controlFile, content); } - protected void CreateHardLink(string targetPath, string newLinkPath) + protected void CreateHardLink(string newLinkFileName, string existingFileName) { - string virtualTargetFile = Path.Combine(this.Enlistment.RepoRoot, targetPath); - string controlTargetFile = Path.Combine(this.ControlGitRepo.RootPath, targetPath); - string virtualNewLinkFile = Path.Combine(this.Enlistment.RepoRoot, newLinkPath); - string controlNewLinkFile = Path.Combine(this.ControlGitRepo.RootPath, newLinkPath); + string virtualExistingFile = Path.Combine(this.Enlistment.RepoRoot, existingFileName); + string controlExistingFile = Path.Combine(this.ControlGitRepo.RootPath, existingFileName); + string virtualNewLinkFile = Path.Combine(this.Enlistment.RepoRoot, newLinkFileName); + string controlNewLinkFile = Path.Combine(this.ControlGitRepo.RootPath, newLinkFileName); // GitRepoTests are only run with SystemIORunner (which does not support hardlink // creation) so use a BashRunner instead. - this.FileSystem.ShouldBeOfType(); this.FileSystem.SupportsHardlinkCreation.ShouldBeFalse( - "FileSystem *does* support hard link creation, CreateHardLink no longer needs to create a BashRunner"); + "If this.FileSystem.SupportsHardlinkCreation is true, CreateHardLink no longer needs to create a BashRunner"); FileSystemRunner runner = new BashRunner(); - runner.CreateHardLink(virtualTargetFile, virtualNewLinkFile); - runner.CreateHardLink(controlTargetFile, controlNewLinkFile); + runner.CreateHardLink(virtualNewLinkFile, virtualExistingFile); + runner.CreateHardLink(controlNewLinkFile, controlExistingFile); } protected void SetFileAsReadOnly(string filePath) From a486ef32d430e66d1c9c3501d6c931ea8eb96ec9 Mon Sep 17 00:00:00 2001 From: Christian Allred Date: Thu, 23 Aug 2018 11:54:25 -0700 Subject: [PATCH 063/272] Updated to ProjFS package version 2018.823.1 --- GVFS/GVFS.Build/ProjFS.props | 2 +- GVFS/GVFS.Platform.Windows/packages.config | 2 +- GVFS/GVFS.UnitTests.Windows/packages.config | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/GVFS/GVFS.Build/ProjFS.props b/GVFS/GVFS.Build/ProjFS.props index 538aa0c28..ba810dca2 100644 --- a/GVFS/GVFS.Build/ProjFS.props +++ b/GVFS/GVFS.Build/ProjFS.props @@ -1,6 +1,6 @@ - GVFS.ProjFS.2018.719.1 + GVFS.ProjFS.2018.823.1 diff --git a/GVFS/GVFS.Platform.Windows/packages.config b/GVFS/GVFS.Platform.Windows/packages.config index 79bfdd0ba..2ae29a96a 100644 --- a/GVFS/GVFS.Platform.Windows/packages.config +++ b/GVFS/GVFS.Platform.Windows/packages.config @@ -1,6 +1,6 @@ - + diff --git a/GVFS/GVFS.UnitTests.Windows/packages.config b/GVFS/GVFS.UnitTests.Windows/packages.config index 8857181ac..162025ee2 100644 --- a/GVFS/GVFS.UnitTests.Windows/packages.config +++ b/GVFS/GVFS.UnitTests.Windows/packages.config @@ -1,6 +1,6 @@ - + From 2ab1b2b6d7275e0a7af42a0bf8e2bf158fb52d7f Mon Sep 17 00:00:00 2001 From: William Baker Date: Thu, 23 Aug 2018 14:41:55 -0700 Subject: [PATCH 064/272] Cleanup FileBasedLock and make it Windows specific --- GVFS/GVFS.Common/FileBasedLock.cs | 188 +++--------------- GVFS/GVFS.Common/GVFSPlatform.cs | 6 + GVFS/GVFS.Common/IFileBasedLock.cs | 11 + GVFS/GVFS.Common/LocalCacheResolver.cs | 9 +- GVFS/GVFS.Common/Prefetch/CommitPrefetcher.cs | 9 +- GVFS/GVFS.Platform.Mac/MacPlatform.cs | 9 + GVFS/GVFS.Platform.Windows/WindowsPlatform.cs | 9 + .../GVFS.UnitTests.Windows.csproj | 3 +- .../Windows/WindowsFileBasedLockTests.cs} | 9 +- .../Mock/Common/MockFileBasedLock.cs | 23 +++ .../Mock/Common/MockPlatform.cs | 5 + .../FileSystemCallbacks.cs | 7 +- 12 files changed, 107 insertions(+), 181 deletions(-) create mode 100644 GVFS/GVFS.Common/IFileBasedLock.cs rename GVFS/{GVFS.UnitTests/Common/FileBasedLockTests.cs => GVFS.UnitTests.Windows/Windows/WindowsFileBasedLockTests.cs} (79%) create mode 100644 GVFS/GVFS.UnitTests/Mock/Common/MockFileBasedLock.cs diff --git a/GVFS/GVFS.Common/FileBasedLock.cs b/GVFS/GVFS.Common/FileBasedLock.cs index 24df80941..669559383 100644 --- a/GVFS/GVFS.Common/FileBasedLock.cs +++ b/GVFS/GVFS.Common/FileBasedLock.cs @@ -1,19 +1,19 @@ -using GVFS.Common.FileSystem; +using GVFS.Common; +using GVFS.Common.FileSystem; using GVFS.Common.Tracing; using System; using System.ComponentModel; using System.IO; using System.Text; -namespace GVFS.Common +namespace GVFS.Platform.Windows { - public class FileBasedLock : IDisposable + public class WindowsFileBasedLock : IFileBasedLock { private const int HResultErrorSharingViolation = -2147024864; // -2147024864 = 0x80070020 = ERROR_SHARING_VIOLATION private const int HResultErrorFileExists = -2147024816; // -2147024816 = 0x80070050 = ERROR_FILE_EXISTS private const int DefaultStreamWriterBufferSize = 1024; // Copied from: http://referencesource.microsoft.com/#mscorlib/system/io/streamwriter.cs,5516ce201dc06b5f - private const long InvalidFileLength = -1; - private const string EtwArea = nameof(FileBasedLock); + private const string EtwArea = nameof(WindowsFileBasedLock); private static readonly Encoding UTF8NoBOM = new UTF8Encoding(false, true); // Default encoding used by StreamWriter private readonly object deleteOnCloseStreamLock = new object(); @@ -21,39 +21,30 @@ public class FileBasedLock : IDisposable private readonly string lockPath; private ITracer tracer; private Stream deleteOnCloseStream; - private bool overwriteExistingLock; + private string signature; /// /// FileBasedLock constructor /// /// Path to lock file /// Text to write in lock file - /// - /// If true, FileBasedLock will attempt to overwrite an existing lock file (if one exists on disk) when - /// acquiring the lock file. - /// /// /// GVFS keeps an exclusive write handle open to lock files that it creates with FileBasedLock. This means that - /// FileBasedLock still ensures exclusivity when "overwriteExistingLock" is true if the lock file is only used for - /// coordination between multiple GVFS processes. + /// FileBasedLock still ensures exclusivity when the lock file is used only for coordination between multiple GVFS processes. /// - public FileBasedLock( + public WindowsFileBasedLock( PhysicalFileSystem fileSystem, ITracer tracer, string lockPath, - string signature, - bool overwriteExistingLock) + string signature) { this.fileSystem = fileSystem; this.tracer = tracer; this.lockPath = lockPath; - this.Signature = signature; - this.overwriteExistingLock = overwriteExistingLock; + this.signature = signature; } - public string Signature { get; private set; } - - public bool TryAcquireLockAndDeleteOnClose() + public bool TryAcquireLock() { try { @@ -68,7 +59,7 @@ public bool TryAcquireLockAndDeleteOnClose() this.deleteOnCloseStream = this.fileSystem.OpenFileStream( this.lockPath, - this.overwriteExistingLock ? FileMode.Create : FileMode.CreateNew, + FileMode.Create, FileAccess.ReadWrite, FileShare.Read, FileOptions.DeleteOnClose, @@ -81,7 +72,7 @@ public bool TryAcquireLockAndDeleteOnClose() DefaultStreamWriterBufferSize, leaveOpen: true)) { - this.WriteSignatureAndMessage(writer, message: null); + this.WriteSignature(writer); } return true; @@ -90,13 +81,11 @@ public bool TryAcquireLockAndDeleteOnClose() catch (IOException e) { // HResultErrorFileExists is expected when the lock file exists - // HResultErrorSharingViolation is expected when the lock file exists and we're in this.overwriteExistingLock mode, as - // another GVFS process has likely acquired the lock file - if (e.HResult != HResultErrorFileExists && - !(this.overwriteExistingLock && e.HResult == HResultErrorSharingViolation)) + // HResultErrorSharingViolation is expected when the lock file exists andanother GVFS process has acquired the lock file + if (e.HResult != HResultErrorFileExists && e.HResult != HResultErrorSharingViolation) { EventMetadata metadata = this.CreateLockMetadata(e); - this.tracer.RelatedWarning(metadata, "TryAcquireLockAndDeleteOnClose: IOException caught while trying to acquire lock"); + this.tracer.RelatedWarning(metadata, $"{nameof(this.TryAcquireLock)}: IOException caught while trying to acquire lock"); } this.DisposeStream(); @@ -105,7 +94,7 @@ public bool TryAcquireLockAndDeleteOnClose() catch (UnauthorizedAccessException e) { EventMetadata metadata = this.CreateLockMetadata(e); - this.tracer.RelatedWarning(metadata, "TryAcquireLockAndDeleteOnClose: UnauthorizedAccessException caught while trying to acquire lock"); + this.tracer.RelatedWarning(metadata, $"{nameof(this.TryAcquireLock)}: UnauthorizedAccessException caught while trying to acquire lock"); this.DisposeStream(); return false; @@ -113,7 +102,7 @@ public bool TryAcquireLockAndDeleteOnClose() catch (Win32Exception e) { EventMetadata metadata = this.CreateLockMetadata(e); - this.tracer.RelatedWarning(metadata, "TryAcquireLockAndDeleteOnClose: Win32Exception caught while trying to acquire lock"); + this.tracer.RelatedWarning(metadata, $"{nameof(this.TryAcquireLock)}: Win32Exception caught while trying to acquire lock"); this.DisposeStream(); return false; @@ -121,118 +110,34 @@ public bool TryAcquireLockAndDeleteOnClose() catch (Exception e) { EventMetadata metadata = this.CreateLockMetadata(e); - this.tracer.RelatedError(metadata, "TryAcquireLockAndDeleteOnClose: Unhandled exception caught while trying to acquire lock"); + this.tracer.RelatedError(metadata, $"{nameof(this.TryAcquireLock)}: Unhandled exception caught while trying to acquire lock"); this.DisposeStream(); throw; } } - public bool TryReleaseLock() - { - if (this.DisposeStream()) - { - return true; - } - - LockData lockData = this.GetLockDataFromDisk(); - if (lockData == null || lockData.Signature != this.Signature) - { - if (lockData == null) - { - throw new LockFileDoesNotExistException(this.lockPath); - } - - throw new LockSignatureDoesNotMatchException(this.lockPath, this.Signature, lockData.Signature); - } - - try - { - this.fileSystem.DeleteFile(this.lockPath); - } - catch (IOException e) - { - EventMetadata metadata = this.CreateLockMetadata(e); - this.tracer.RelatedWarning(metadata, "TryReleaseLock: IOException caught while trying to release lock"); - - return false; - } - - return true; - } - - public bool IsOpen() - { - return this.deleteOnCloseStream != null; - } - public void Dispose() { - this.Dispose(true); - GC.SuppressFinalize(this); - } - - protected void Dispose(bool disposing) - { - if (disposing) - { - this.DisposeStream(); - } - } - - private LockData GetLockDataFromDisk() - { - if (this.LockFileExists()) - { - string existingSignature; - string existingMessage; - this.ReadLockFile(out existingSignature, out existingMessage); - return new LockData(existingSignature, existingMessage); - } - - return null; + this.DisposeStream(); } - private void ReadLockFile(out string existingSignature, out string lockerMessage) + private bool IsOpen() { - using (Stream fs = this.fileSystem.OpenFileStream(this.lockPath, FileMode.Open, FileAccess.Read, FileShare.ReadWrite | FileShare.Delete, callFlushFileBuffers: false)) - using (StreamReader reader = new StreamReader(fs, UTF8NoBOM)) - { - existingSignature = reader.ReadLine(); - lockerMessage = reader.ReadLine(); - } - - existingSignature = existingSignature ?? string.Empty; - lockerMessage = lockerMessage ?? string.Empty; - } - - private bool LockFileExists() - { - return this.fileSystem.FileExists(this.lockPath); + return this.deleteOnCloseStream != null; } - private void WriteSignatureAndMessage(StreamWriter writer, string message) + private void WriteSignature(StreamWriter writer) { - writer.WriteLine(this.Signature); - if (message != null) - { - writer.Write(message); - } + writer.WriteLine(this.signature); } - private EventMetadata CreateLockMetadata() + private EventMetadata CreateLockMetadata(Exception exception = null) { EventMetadata metadata = new EventMetadata(); metadata.Add("Area", EtwArea); metadata.Add("LockPath", this.lockPath); - metadata.Add("Signature", this.Signature); - - return metadata; - } - - private EventMetadata CreateLockMetadata(Exception exception = null) - { - EventMetadata metadata = this.CreateLockMetadata(); + metadata.Add("Signature", this.signature); if (exception != null) { metadata.Add("Exception", exception.ToString()); @@ -255,46 +160,5 @@ private bool DisposeStream() return false; } - - public class LockException : Exception - { - public LockException(string messageFormat, params string[] args) - : base(string.Format(messageFormat, args)) - { - } - } - - public class LockFileDoesNotExistException : LockException - { - public LockFileDoesNotExistException(string lockPath) - : base("Lock file {0} does not exist", lockPath) - { - } - } - - public class LockSignatureDoesNotMatchException : LockException - { - public LockSignatureDoesNotMatchException(string lockPath, string expectedSignature, string actualSignature) - : base( - "Lock file {0} does not contain expected signature '{1}' (existing signature: '{2}')", - lockPath, - expectedSignature, - actualSignature) - { - } - } - - public class LockData - { - public LockData(string signature, string message) - { - this.Signature = signature; - this.Message = message; - } - - public string Signature { get; } - - public string Message { get; } - } } } diff --git a/GVFS/GVFS.Common/GVFSPlatform.cs b/GVFS/GVFS.Common/GVFSPlatform.cs index cb0b7f2d4..f604fdad6 100644 --- a/GVFS/GVFS.Common/GVFSPlatform.cs +++ b/GVFS/GVFS.Common/GVFSPlatform.cs @@ -59,6 +59,12 @@ public static void Register(GVFSPlatform platform) public abstract bool IsGitStatusCacheSupported(); + public abstract IFileBasedLock CreateFileBasedLock( + PhysicalFileSystem fileSystem, + ITracer tracer, + string lockPath, + string signature); + public bool TryGetNormalizedPathRoot(string path, out string pathRoot, out string errorMessage) { pathRoot = null; diff --git a/GVFS/GVFS.Common/IFileBasedLock.cs b/GVFS/GVFS.Common/IFileBasedLock.cs new file mode 100644 index 000000000..44231a670 --- /dev/null +++ b/GVFS/GVFS.Common/IFileBasedLock.cs @@ -0,0 +1,11 @@ +using System; +using System.Collections.Generic; +using System.Text; + +namespace GVFS.Common +{ + public interface IFileBasedLock : IDisposable + { + bool TryAcquireLock(); + } +} diff --git a/GVFS/GVFS.Common/LocalCacheResolver.cs b/GVFS/GVFS.Common/LocalCacheResolver.cs index 76abac9ac..4ef9757e2 100644 --- a/GVFS/GVFS.Common/LocalCacheResolver.cs +++ b/GVFS/GVFS.Common/LocalCacheResolver.cs @@ -64,12 +64,11 @@ public bool TryGetLocalCacheKeyFromLocalConfigOrRemoteCacheServers( string lockPath = Path.Combine(localCacheRoot, MappingFile + ".lock"); this.fileSystem.CreateDirectory(localCacheRoot); - using (FileBasedLock mappingLock = new FileBasedLock( + using (IFileBasedLock mappingLock = GVFSPlatform.Instance.CreateFileBasedLock( this.fileSystem, tracer, lockPath, - this.enlistment.EnlistmentRoot, - overwriteExistingLock: true)) + this.enlistment.EnlistmentRoot)) { if (!this.TryAcquireLockWithRetries(tracer, mappingLock)) { @@ -272,14 +271,14 @@ private bool TryGetLocalCacheKeyFromRemoteCacheServers( return true; } - private bool TryAcquireLockWithRetries(ITracer tracer, FileBasedLock mappingLock) + private bool TryAcquireLockWithRetries(ITracer tracer, IFileBasedLock mappingLock) { const int NumRetries = 100; const int WaitTimeMs = 100; for (int i = 0; i < NumRetries; ++i) { - if (mappingLock.TryAcquireLockAndDeleteOnClose()) + if (mappingLock.TryAcquireLock()) { return true; } diff --git a/GVFS/GVFS.Common/Prefetch/CommitPrefetcher.cs b/GVFS/GVFS.Common/Prefetch/CommitPrefetcher.cs index f2ee21742..d1c2b3c6e 100644 --- a/GVFS/GVFS.Common/Prefetch/CommitPrefetcher.cs +++ b/GVFS/GVFS.Common/Prefetch/CommitPrefetcher.cs @@ -24,12 +24,11 @@ public static bool TryPrefetchCommitsAndTrees( out string error) { List packIndexes; - using (FileBasedLock prefetchLock = new FileBasedLock( + using (IFileBasedLock prefetchLock = GVFSPlatform.Instance.CreateFileBasedLock( fileSystem, tracer, Path.Combine(enlistment.GitPackRoot, PrefetchCommitsAndTreesLock), - enlistment.EnlistmentRoot, - overwriteExistingLock: true)) + enlistment.EnlistmentRoot)) { WaitUntilLockIsAcquired(tracer, prefetchLock); long maxGoodTimeStamp; @@ -210,10 +209,10 @@ private static bool TrySchedulePostFetchJob(ITracer tracer, string namedPipeName return null; } - private static void WaitUntilLockIsAcquired(ITracer tracer, FileBasedLock fileBasedLock) + private static void WaitUntilLockIsAcquired(ITracer tracer, IFileBasedLock fileBasedLock) { int attempt = 0; - while (!fileBasedLock.TryAcquireLockAndDeleteOnClose()) + while (!fileBasedLock.TryAcquireLock()) { Thread.Sleep(LockWaitTimeMs); ++attempt; diff --git a/GVFS/GVFS.Platform.Mac/MacPlatform.cs b/GVFS/GVFS.Platform.Mac/MacPlatform.cs index d11b9d5bd..3238bd7c2 100644 --- a/GVFS/GVFS.Platform.Mac/MacPlatform.cs +++ b/GVFS/GVFS.Platform.Mac/MacPlatform.cs @@ -127,5 +127,14 @@ public override bool IsGitStatusCacheSupported() // TODO(Mac): support git status cache return false; } + + public override IFileBasedLock CreateFileBasedLock( + PhysicalFileSystem fileSystem, + ITracer tracer, + string lockPath, + string signature) + { + throw new NotImplementedException(); + } } } diff --git a/GVFS/GVFS.Platform.Windows/WindowsPlatform.cs b/GVFS/GVFS.Platform.Windows/WindowsPlatform.cs index c7c0b6018..df771e63f 100644 --- a/GVFS/GVFS.Platform.Windows/WindowsPlatform.cs +++ b/GVFS/GVFS.Platform.Windows/WindowsPlatform.cs @@ -253,6 +253,15 @@ public override bool IsGitStatusCacheSupported() return File.Exists(Path.Combine(Paths.GetServiceDataRoot(GVFSConstants.Service.ServiceName), GVFSConstants.GitStatusCache.EnableGitStatusCacheTokenFile)); } + public override IFileBasedLock CreateFileBasedLock( + PhysicalFileSystem fileSystem, + ITracer tracer, + string lockPath, + string signature) + { + return new WindowsFileBasedLock(fileSystem, tracer, lockPath, signature); + } + public override bool TryGetGVFSEnlistmentRoot(string directory, out string enlistmentRoot, out string errorMessage) { return WindowsPlatform.TryGetGVFSEnlistmentRootImplementation(directory, out enlistmentRoot, out errorMessage); diff --git a/GVFS/GVFS.UnitTests.Windows/GVFS.UnitTests.Windows.csproj b/GVFS/GVFS.UnitTests.Windows/GVFS.UnitTests.Windows.csproj index 506f0bcb2..6d9597a17 100644 --- a/GVFS/GVFS.UnitTests.Windows/GVFS.UnitTests.Windows.csproj +++ b/GVFS/GVFS.UnitTests.Windows/GVFS.UnitTests.Windows.csproj @@ -1,4 +1,4 @@ - + @@ -83,6 +83,7 @@ + diff --git a/GVFS/GVFS.UnitTests/Common/FileBasedLockTests.cs b/GVFS/GVFS.UnitTests.Windows/Windows/WindowsFileBasedLockTests.cs similarity index 79% rename from GVFS/GVFS.UnitTests/Common/FileBasedLockTests.cs rename to GVFS/GVFS.UnitTests.Windows/Windows/WindowsFileBasedLockTests.cs index 3d6e0baf7..51bbe9e24 100644 --- a/GVFS/GVFS.UnitTests/Common/FileBasedLockTests.cs +++ b/GVFS/GVFS.UnitTests.Windows/Windows/WindowsFileBasedLockTests.cs @@ -1,14 +1,15 @@ using GVFS.Common; +using GVFS.Platform.Windows; using GVFS.Tests.Should; using GVFS.UnitTests.Mock.Common; using GVFS.UnitTests.Mock.FileSystem; using NUnit.Framework; using System.IO; -namespace GVFS.UnitTests.Common +namespace GVFS.UnitTests.Windows { [TestFixture] - public class FileBasedLockTests + public class WindowsFileBasedLockTests { [TestCase] public void CreateLockWhenDirectoryMissing() @@ -17,9 +18,9 @@ public void CreateLockWhenDirectoryMissing() string lockPath = Path.Combine(parentPath, "lock"); MockTracer tracer = new MockTracer(); FileBasedLockFileSystem fs = new FileBasedLockFileSystem(); - FileBasedLock fileBasedLock = new FileBasedLock(fs, tracer, lockPath, "signature", overwriteExistingLock: true); + IFileBasedLock fileBasedLock = new WindowsFileBasedLock(fs, tracer, lockPath, "signature"); - fileBasedLock.TryAcquireLockAndDeleteOnClose().ShouldBeTrue(); + fileBasedLock.TryAcquireLock().ShouldBeTrue(); fs.CreateDirectoryPath.ShouldNotBeNull(); fs.CreateDirectoryPath.ShouldEqual(parentPath); } diff --git a/GVFS/GVFS.UnitTests/Mock/Common/MockFileBasedLock.cs b/GVFS/GVFS.UnitTests/Mock/Common/MockFileBasedLock.cs new file mode 100644 index 000000000..a9ceb13a8 --- /dev/null +++ b/GVFS/GVFS.UnitTests/Mock/Common/MockFileBasedLock.cs @@ -0,0 +1,23 @@ +using GVFS.Common; +using System; +using System.Collections.Generic; +using System.Text; + +namespace GVFS.UnitTests.Mock.Common +{ + public class MockFileBasedLock : IFileBasedLock + { + public MockFileBasedLock() + { + } + + public bool TryAcquireLock() + { + return true; + } + + public void Dispose() + { + } + } +} diff --git a/GVFS/GVFS.UnitTests/Mock/Common/MockPlatform.cs b/GVFS/GVFS.UnitTests/Mock/Common/MockPlatform.cs index 973be36b2..bf5debfce 100644 --- a/GVFS/GVFS.UnitTests/Mock/Common/MockPlatform.cs +++ b/GVFS/GVFS.UnitTests/Mock/Common/MockPlatform.cs @@ -104,5 +104,10 @@ public override bool IsGitStatusCacheSupported() { return true; } + + public override IFileBasedLock CreateFileBasedLock(PhysicalFileSystem fileSystem, ITracer tracer, string lockPath, string signature) + { + return new MockFileBasedLock(); + } } } diff --git a/GVFS/GVFS.Virtualization/FileSystemCallbacks.cs b/GVFS/GVFS.Virtualization/FileSystemCallbacks.cs index 980ab2b87..d897c5320 100644 --- a/GVFS/GVFS.Virtualization/FileSystemCallbacks.cs +++ b/GVFS/GVFS.Virtualization/FileSystemCallbacks.cs @@ -524,14 +524,13 @@ private void PostFetchJob(List packIndexes) { try { - using (FileBasedLock postFetchFileLock = new FileBasedLock( + using (IFileBasedLock postFetchFileLock = GVFSPlatform.Instance.CreateFileBasedLock( this.context.FileSystem, this.context.Tracer, Path.Combine(this.context.Enlistment.GitObjectsRoot, PostFetchLock), - this.context.Enlistment.EnlistmentRoot, - overwriteExistingLock: true)) + this.context.Enlistment.EnlistmentRoot)) { - if (!postFetchFileLock.TryAcquireLockAndDeleteOnClose()) + if (!postFetchFileLock.TryAcquireLock()) { this.context.Tracer.RelatedInfo(PostFetchTelemetryKey + ": Skipping post-fetch work since another process holds the lock"); return; From ddc5f85a6e84b460caf608d4ee6ff65751e1dac2 Mon Sep 17 00:00:00 2001 From: William Baker Date: Thu, 23 Aug 2018 14:45:31 -0700 Subject: [PATCH 065/272] Rename FileBasedLock.cs to WindowsFileBasedLock.cs and move to Windows platform folder --- GVFS/GVFS.Platform.Windows/GVFS.Platform.Windows.csproj | 1 + .../WindowsFileBasedLock.cs} | 0 2 files changed, 1 insertion(+) rename GVFS/{GVFS.Common/FileBasedLock.cs => GVFS.Platform.Windows/WindowsFileBasedLock.cs} (100%) diff --git a/GVFS/GVFS.Platform.Windows/GVFS.Platform.Windows.csproj b/GVFS/GVFS.Platform.Windows/GVFS.Platform.Windows.csproj index 7a86d6226..e3deda70c 100644 --- a/GVFS/GVFS.Platform.Windows/GVFS.Platform.Windows.csproj +++ b/GVFS/GVFS.Platform.Windows/GVFS.Platform.Windows.csproj @@ -91,6 +91,7 @@ + diff --git a/GVFS/GVFS.Common/FileBasedLock.cs b/GVFS/GVFS.Platform.Windows/WindowsFileBasedLock.cs similarity index 100% rename from GVFS/GVFS.Common/FileBasedLock.cs rename to GVFS/GVFS.Platform.Windows/WindowsFileBasedLock.cs From 7b17d52cd56f78d92694c67268f600af800c37b6 Mon Sep 17 00:00:00 2001 From: William Baker Date: Thu, 23 Aug 2018 15:11:36 -0700 Subject: [PATCH 066/272] More cleanup, switch from interface to abstract class --- GVFS/GVFS.Common/FileBasedLock.cs | 30 ++++++++++++++++ GVFS/GVFS.Common/GVFSPlatform.cs | 2 +- GVFS/GVFS.Common/IFileBasedLock.cs | 11 ------ GVFS/GVFS.Common/LocalCacheResolver.cs | 4 +-- GVFS/GVFS.Common/Prefetch/CommitPrefetcher.cs | 4 +-- GVFS/GVFS.Platform.Mac/MacFileBasedLock.cs | 28 +++++++++++++++ GVFS/GVFS.Platform.Mac/MacPlatform.cs | 4 +-- .../WindowsFileBasedLock.cs | 35 ++++++++----------- GVFS/GVFS.Platform.Windows/WindowsPlatform.cs | 2 +- .../Windows/WindowsFileBasedLockTests.cs | 2 +- .../Mock/Common/MockFileBasedLock.cs | 18 ++++++---- .../Mock/Common/MockPlatform.cs | 4 +-- .../FileSystemCallbacks.cs | 2 +- 13 files changed, 95 insertions(+), 51 deletions(-) create mode 100644 GVFS/GVFS.Common/FileBasedLock.cs delete mode 100644 GVFS/GVFS.Common/IFileBasedLock.cs create mode 100644 GVFS/GVFS.Platform.Mac/MacFileBasedLock.cs diff --git a/GVFS/GVFS.Common/FileBasedLock.cs b/GVFS/GVFS.Common/FileBasedLock.cs new file mode 100644 index 000000000..a216ecc88 --- /dev/null +++ b/GVFS/GVFS.Common/FileBasedLock.cs @@ -0,0 +1,30 @@ +using GVFS.Common.FileSystem; +using GVFS.Common.Tracing; +using System; + +namespace GVFS.Common +{ + public abstract class FileBasedLock : IDisposable + { + protected readonly PhysicalFileSystem FileSystem; + protected readonly string LockPath; + protected readonly ITracer Tracer; + protected readonly string Signature; + + public FileBasedLock( + PhysicalFileSystem fileSystem, + ITracer tracer, + string lockPath, + string signature) + { + this.FileSystem = fileSystem; + this.Tracer = tracer; + this.LockPath = lockPath; + this.Signature = signature; + } + + public abstract bool TryAcquireLock(); + + public abstract void Dispose(); + } +} diff --git a/GVFS/GVFS.Common/GVFSPlatform.cs b/GVFS/GVFS.Common/GVFSPlatform.cs index f604fdad6..fe0b70255 100644 --- a/GVFS/GVFS.Common/GVFSPlatform.cs +++ b/GVFS/GVFS.Common/GVFSPlatform.cs @@ -59,7 +59,7 @@ public static void Register(GVFSPlatform platform) public abstract bool IsGitStatusCacheSupported(); - public abstract IFileBasedLock CreateFileBasedLock( + public abstract FileBasedLock CreateFileBasedLock( PhysicalFileSystem fileSystem, ITracer tracer, string lockPath, diff --git a/GVFS/GVFS.Common/IFileBasedLock.cs b/GVFS/GVFS.Common/IFileBasedLock.cs deleted file mode 100644 index 44231a670..000000000 --- a/GVFS/GVFS.Common/IFileBasedLock.cs +++ /dev/null @@ -1,11 +0,0 @@ -using System; -using System.Collections.Generic; -using System.Text; - -namespace GVFS.Common -{ - public interface IFileBasedLock : IDisposable - { - bool TryAcquireLock(); - } -} diff --git a/GVFS/GVFS.Common/LocalCacheResolver.cs b/GVFS/GVFS.Common/LocalCacheResolver.cs index 4ef9757e2..0a3fdaa73 100644 --- a/GVFS/GVFS.Common/LocalCacheResolver.cs +++ b/GVFS/GVFS.Common/LocalCacheResolver.cs @@ -64,7 +64,7 @@ public bool TryGetLocalCacheKeyFromLocalConfigOrRemoteCacheServers( string lockPath = Path.Combine(localCacheRoot, MappingFile + ".lock"); this.fileSystem.CreateDirectory(localCacheRoot); - using (IFileBasedLock mappingLock = GVFSPlatform.Instance.CreateFileBasedLock( + using (FileBasedLock mappingLock = GVFSPlatform.Instance.CreateFileBasedLock( this.fileSystem, tracer, lockPath, @@ -271,7 +271,7 @@ private bool TryGetLocalCacheKeyFromRemoteCacheServers( return true; } - private bool TryAcquireLockWithRetries(ITracer tracer, IFileBasedLock mappingLock) + private bool TryAcquireLockWithRetries(ITracer tracer, FileBasedLock mappingLock) { const int NumRetries = 100; const int WaitTimeMs = 100; diff --git a/GVFS/GVFS.Common/Prefetch/CommitPrefetcher.cs b/GVFS/GVFS.Common/Prefetch/CommitPrefetcher.cs index d1c2b3c6e..60d7a3d38 100644 --- a/GVFS/GVFS.Common/Prefetch/CommitPrefetcher.cs +++ b/GVFS/GVFS.Common/Prefetch/CommitPrefetcher.cs @@ -24,7 +24,7 @@ public static bool TryPrefetchCommitsAndTrees( out string error) { List packIndexes; - using (IFileBasedLock prefetchLock = GVFSPlatform.Instance.CreateFileBasedLock( + using (FileBasedLock prefetchLock = GVFSPlatform.Instance.CreateFileBasedLock( fileSystem, tracer, Path.Combine(enlistment.GitPackRoot, PrefetchCommitsAndTreesLock), @@ -209,7 +209,7 @@ private static bool TrySchedulePostFetchJob(ITracer tracer, string namedPipeName return null; } - private static void WaitUntilLockIsAcquired(ITracer tracer, IFileBasedLock fileBasedLock) + private static void WaitUntilLockIsAcquired(ITracer tracer, FileBasedLock fileBasedLock) { int attempt = 0; while (!fileBasedLock.TryAcquireLock()) diff --git a/GVFS/GVFS.Platform.Mac/MacFileBasedLock.cs b/GVFS/GVFS.Platform.Mac/MacFileBasedLock.cs new file mode 100644 index 000000000..c781a2e13 --- /dev/null +++ b/GVFS/GVFS.Platform.Mac/MacFileBasedLock.cs @@ -0,0 +1,28 @@ +using GVFS.Common; +using GVFS.Common.FileSystem; +using GVFS.Common.Tracing; +using System; + +namespace GVFS.Platform.Mac +{ + public class MacFileBasedLock : FileBasedLock + { + public MacFileBasedLock( + PhysicalFileSystem fileSystem, + ITracer tracer, + string lockPath, + string signature) + : base (fileSystem, tracer, lockPath, signature) + { + } + + public override bool TryAcquireLock() + { + throw new NotImplementedException(); + } + + public override void Dispose() + { + } + } +} diff --git a/GVFS/GVFS.Platform.Mac/MacPlatform.cs b/GVFS/GVFS.Platform.Mac/MacPlatform.cs index 3238bd7c2..e2c641484 100644 --- a/GVFS/GVFS.Platform.Mac/MacPlatform.cs +++ b/GVFS/GVFS.Platform.Mac/MacPlatform.cs @@ -128,13 +128,13 @@ public override bool IsGitStatusCacheSupported() return false; } - public override IFileBasedLock CreateFileBasedLock( + public override FileBasedLock CreateFileBasedLock( PhysicalFileSystem fileSystem, ITracer tracer, string lockPath, string signature) { - throw new NotImplementedException(); + return new MacFileBasedLock(fileSystem, tracer, lockPath, signature); } } } diff --git a/GVFS/GVFS.Platform.Windows/WindowsFileBasedLock.cs b/GVFS/GVFS.Platform.Windows/WindowsFileBasedLock.cs index 669559383..0dacc4172 100644 --- a/GVFS/GVFS.Platform.Windows/WindowsFileBasedLock.cs +++ b/GVFS/GVFS.Platform.Windows/WindowsFileBasedLock.cs @@ -8,7 +8,7 @@ namespace GVFS.Platform.Windows { - public class WindowsFileBasedLock : IFileBasedLock + public class WindowsFileBasedLock : FileBasedLock { private const int HResultErrorSharingViolation = -2147024864; // -2147024864 = 0x80070020 = ERROR_SHARING_VIOLATION private const int HResultErrorFileExists = -2147024816; // -2147024816 = 0x80070050 = ERROR_FILE_EXISTS @@ -17,11 +17,7 @@ public class WindowsFileBasedLock : IFileBasedLock private static readonly Encoding UTF8NoBOM = new UTF8Encoding(false, true); // Default encoding used by StreamWriter private readonly object deleteOnCloseStreamLock = new object(); - private readonly PhysicalFileSystem fileSystem; - private readonly string lockPath; - private ITracer tracer; private Stream deleteOnCloseStream; - private string signature; /// /// FileBasedLock constructor @@ -37,14 +33,11 @@ public WindowsFileBasedLock( ITracer tracer, string lockPath, string signature) + : base(fileSystem, tracer, lockPath, signature) { - this.fileSystem = fileSystem; - this.tracer = tracer; - this.lockPath = lockPath; - this.signature = signature; } - public bool TryAcquireLock() + public override bool TryAcquireLock() { try { @@ -55,10 +48,10 @@ public bool TryAcquireLock() return true; } - this.fileSystem.CreateDirectory(Path.GetDirectoryName(this.lockPath)); + this.FileSystem.CreateDirectory(Path.GetDirectoryName(this.LockPath)); - this.deleteOnCloseStream = this.fileSystem.OpenFileStream( - this.lockPath, + this.deleteOnCloseStream = this.FileSystem.OpenFileStream( + this.LockPath, FileMode.Create, FileAccess.ReadWrite, FileShare.Read, @@ -85,7 +78,7 @@ public bool TryAcquireLock() if (e.HResult != HResultErrorFileExists && e.HResult != HResultErrorSharingViolation) { EventMetadata metadata = this.CreateLockMetadata(e); - this.tracer.RelatedWarning(metadata, $"{nameof(this.TryAcquireLock)}: IOException caught while trying to acquire lock"); + this.Tracer.RelatedWarning(metadata, $"{nameof(this.TryAcquireLock)}: IOException caught while trying to acquire lock"); } this.DisposeStream(); @@ -94,7 +87,7 @@ public bool TryAcquireLock() catch (UnauthorizedAccessException e) { EventMetadata metadata = this.CreateLockMetadata(e); - this.tracer.RelatedWarning(metadata, $"{nameof(this.TryAcquireLock)}: UnauthorizedAccessException caught while trying to acquire lock"); + this.Tracer.RelatedWarning(metadata, $"{nameof(this.TryAcquireLock)}: UnauthorizedAccessException caught while trying to acquire lock"); this.DisposeStream(); return false; @@ -102,7 +95,7 @@ public bool TryAcquireLock() catch (Win32Exception e) { EventMetadata metadata = this.CreateLockMetadata(e); - this.tracer.RelatedWarning(metadata, $"{nameof(this.TryAcquireLock)}: Win32Exception caught while trying to acquire lock"); + this.Tracer.RelatedWarning(metadata, $"{nameof(this.TryAcquireLock)}: Win32Exception caught while trying to acquire lock"); this.DisposeStream(); return false; @@ -110,14 +103,14 @@ public bool TryAcquireLock() catch (Exception e) { EventMetadata metadata = this.CreateLockMetadata(e); - this.tracer.RelatedError(metadata, $"{nameof(this.TryAcquireLock)}: Unhandled exception caught while trying to acquire lock"); + this.Tracer.RelatedError(metadata, $"{nameof(this.TryAcquireLock)}: Unhandled exception caught while trying to acquire lock"); this.DisposeStream(); throw; } } - public void Dispose() + public override void Dispose() { this.DisposeStream(); } @@ -129,15 +122,15 @@ private bool IsOpen() private void WriteSignature(StreamWriter writer) { - writer.WriteLine(this.signature); + writer.WriteLine(this.Signature); } private EventMetadata CreateLockMetadata(Exception exception = null) { EventMetadata metadata = new EventMetadata(); metadata.Add("Area", EtwArea); - metadata.Add("LockPath", this.lockPath); - metadata.Add("Signature", this.signature); + metadata.Add("LockPath", this.LockPath); + metadata.Add("Signature", this.Signature); if (exception != null) { metadata.Add("Exception", exception.ToString()); diff --git a/GVFS/GVFS.Platform.Windows/WindowsPlatform.cs b/GVFS/GVFS.Platform.Windows/WindowsPlatform.cs index df771e63f..11c8708c7 100644 --- a/GVFS/GVFS.Platform.Windows/WindowsPlatform.cs +++ b/GVFS/GVFS.Platform.Windows/WindowsPlatform.cs @@ -253,7 +253,7 @@ public override bool IsGitStatusCacheSupported() return File.Exists(Path.Combine(Paths.GetServiceDataRoot(GVFSConstants.Service.ServiceName), GVFSConstants.GitStatusCache.EnableGitStatusCacheTokenFile)); } - public override IFileBasedLock CreateFileBasedLock( + public override FileBasedLock CreateFileBasedLock( PhysicalFileSystem fileSystem, ITracer tracer, string lockPath, diff --git a/GVFS/GVFS.UnitTests.Windows/Windows/WindowsFileBasedLockTests.cs b/GVFS/GVFS.UnitTests.Windows/Windows/WindowsFileBasedLockTests.cs index 51bbe9e24..82ecdfdf4 100644 --- a/GVFS/GVFS.UnitTests.Windows/Windows/WindowsFileBasedLockTests.cs +++ b/GVFS/GVFS.UnitTests.Windows/Windows/WindowsFileBasedLockTests.cs @@ -18,7 +18,7 @@ public void CreateLockWhenDirectoryMissing() string lockPath = Path.Combine(parentPath, "lock"); MockTracer tracer = new MockTracer(); FileBasedLockFileSystem fs = new FileBasedLockFileSystem(); - IFileBasedLock fileBasedLock = new WindowsFileBasedLock(fs, tracer, lockPath, "signature"); + FileBasedLock fileBasedLock = new WindowsFileBasedLock(fs, tracer, lockPath, "signature"); fileBasedLock.TryAcquireLock().ShouldBeTrue(); fs.CreateDirectoryPath.ShouldNotBeNull(); diff --git a/GVFS/GVFS.UnitTests/Mock/Common/MockFileBasedLock.cs b/GVFS/GVFS.UnitTests/Mock/Common/MockFileBasedLock.cs index a9ceb13a8..9c59d9c02 100644 --- a/GVFS/GVFS.UnitTests/Mock/Common/MockFileBasedLock.cs +++ b/GVFS/GVFS.UnitTests/Mock/Common/MockFileBasedLock.cs @@ -1,22 +1,26 @@ using GVFS.Common; -using System; -using System.Collections.Generic; -using System.Text; +using GVFS.Common.FileSystem; +using GVFS.Common.Tracing; namespace GVFS.UnitTests.Mock.Common { - public class MockFileBasedLock : IFileBasedLock + public class MockFileBasedLock : FileBasedLock { - public MockFileBasedLock() + public MockFileBasedLock( + PhysicalFileSystem fileSystem, + ITracer tracer, + string lockPath, + string signature) + : base(fileSystem, tracer, lockPath, signature) { } - public bool TryAcquireLock() + public override bool TryAcquireLock() { return true; } - public void Dispose() + public override void Dispose() { } } diff --git a/GVFS/GVFS.UnitTests/Mock/Common/MockPlatform.cs b/GVFS/GVFS.UnitTests/Mock/Common/MockPlatform.cs index bf5debfce..2aff728b5 100644 --- a/GVFS/GVFS.UnitTests/Mock/Common/MockPlatform.cs +++ b/GVFS/GVFS.UnitTests/Mock/Common/MockPlatform.cs @@ -105,9 +105,9 @@ public override bool IsGitStatusCacheSupported() return true; } - public override IFileBasedLock CreateFileBasedLock(PhysicalFileSystem fileSystem, ITracer tracer, string lockPath, string signature) + public override FileBasedLock CreateFileBasedLock(PhysicalFileSystem fileSystem, ITracer tracer, string lockPath, string signature) { - return new MockFileBasedLock(); + return new MockFileBasedLock(fileSystem, tracer, lockPath, signature); } } } diff --git a/GVFS/GVFS.Virtualization/FileSystemCallbacks.cs b/GVFS/GVFS.Virtualization/FileSystemCallbacks.cs index d897c5320..e50ad7bdc 100644 --- a/GVFS/GVFS.Virtualization/FileSystemCallbacks.cs +++ b/GVFS/GVFS.Virtualization/FileSystemCallbacks.cs @@ -524,7 +524,7 @@ private void PostFetchJob(List packIndexes) { try { - using (IFileBasedLock postFetchFileLock = GVFSPlatform.Instance.CreateFileBasedLock( + using (FileBasedLock postFetchFileLock = GVFSPlatform.Instance.CreateFileBasedLock( this.context.FileSystem, this.context.Tracer, Path.Combine(this.context.Enlistment.GitObjectsRoot, PostFetchLock), From 89d702c2c310b9c5ade0a0d40637fdb2a52d30da Mon Sep 17 00:00:00 2001 From: Saeed Noursalehi Date: Thu, 23 Aug 2018 15:13:44 -0700 Subject: [PATCH 067/272] Added a multithreaded functional test that currently fails on Mac by attempting to concurrently read a previously unhydrated file from several threads --- .../MultithreadedReadWriteTests.cs | 47 +++++++++++++++++-- 1 file changed, 44 insertions(+), 3 deletions(-) diff --git a/GVFS/GVFS.FunctionalTests/Tests/EnlistmentPerFixture/MultithreadedReadWriteTests.cs b/GVFS/GVFS.FunctionalTests/Tests/EnlistmentPerFixture/MultithreadedReadWriteTests.cs index 75f9f4936..2255e9c66 100644 --- a/GVFS/GVFS.FunctionalTests/Tests/EnlistmentPerFixture/MultithreadedReadWriteTests.cs +++ b/GVFS/GVFS.FunctionalTests/Tests/EnlistmentPerFixture/MultithreadedReadWriteTests.cs @@ -1,4 +1,4 @@ -using GVFS.FunctionalTests.FileSystemRunners; +using GVFS.FunctionalTests.FileSystemRunners; using GVFS.FunctionalTests.Should; using GVFS.Tests.Should; using NUnit.Framework; @@ -14,8 +14,48 @@ namespace GVFS.FunctionalTests.Tests.EnlistmentPerFixture [Category(Categories.Mac.M1)] public class MultithreadedReadWriteTests : TestsWithEnlistmentPerFixture { - [TestCase] - public void CanReadUnhydratedFileInParallelWithoutTearing() + [TestCase, Order(1)] + public void CanReadVirtualFileInParallel() + { + // Note: This test MUST go first, or else it needs to ensure that it is reading a unique path compared to the + // other tests in this class. That applies to every directory in the path, as well as the leaf file name. + // Otherwise, this test loses most of its value because there will be no races occurring on creating the + // placeholder directories, enumerating them, and then creating a placeholder file and hydrating it. + + string fileName = Path.Combine("GVFS", "GVFS.FunctionalTests", "Tests", "LongRunningEnlistment", "GitMoveRenameTests.cs"); + string virtualPath = this.Enlistment.GetVirtualPathTo(fileName); + + Exception readException = null; + + Thread[] threads = new Thread[32]; + for (int i = 0; i < threads.Length; ++i) + { + int myIndex = i; + threads[i] = new Thread(() => + { + try + { + FileSystemRunner.DefaultRunner.ReadAllText(virtualPath).ShouldBeNonEmpty(); + } + catch (Exception e) + { + readException = e; + } + } ); + + threads[i].Start(); + } + + for (int i = 0; i < threads.Length; ++i) + { + threads[i].Join(); + } + + readException.ShouldBeNull("At least one of the reads failed"); + } + + [TestCase, Order(2)] + public void CanReadHydratedPlaceholderInParallel() { FileSystemRunner fileSystem = FileSystemRunner.DefaultRunner; string fileName = Path.Combine("GVFS", "GVFS.FunctionalTests", "Tests", "LongRunningEnlistment", "WorkingDirectoryTests.cs"); @@ -72,6 +112,7 @@ public void CanReadUnhydratedFileInParallelWithoutTearing() } [TestCaseSource(typeof(FileSystemRunner), FileSystemRunner.TestRunners)] + [Order(3)] public void CanReadWriteAFileInParallel(FileSystemRunner fileSystem) { string fileName = @"CanReadWriteAFileInParallel"; From 9790552558daad724a5ebb5ae419d443ddce9fdd Mon Sep 17 00:00:00 2001 From: Saeed Noursalehi Date: Thu, 23 Aug 2018 15:50:52 -0700 Subject: [PATCH 068/272] Fix spacing --- .../Tests/EnlistmentPerFixture/MultithreadedReadWriteTests.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/GVFS/GVFS.FunctionalTests/Tests/EnlistmentPerFixture/MultithreadedReadWriteTests.cs b/GVFS/GVFS.FunctionalTests/Tests/EnlistmentPerFixture/MultithreadedReadWriteTests.cs index 2255e9c66..5c0167b5b 100644 --- a/GVFS/GVFS.FunctionalTests/Tests/EnlistmentPerFixture/MultithreadedReadWriteTests.cs +++ b/GVFS/GVFS.FunctionalTests/Tests/EnlistmentPerFixture/MultithreadedReadWriteTests.cs @@ -41,7 +41,7 @@ public void CanReadVirtualFileInParallel() { readException = e; } - } ); + }); threads[i].Start(); } From 3027507782f32b37e6c9676eeccc5b77a4fb180f Mon Sep 17 00:00:00 2001 From: William Baker Date: Thu, 23 Aug 2018 16:19:07 -0700 Subject: [PATCH 069/272] Initial MacFileBasedLock implementation --- GVFS/GVFS.Platform.Mac/MacFileBasedLock.cs | 95 +++++++++++++++++++++- GVFS/GVFS.Platform.Mac/MacFileSystem.cs | 3 +- 2 files changed, 94 insertions(+), 4 deletions(-) diff --git a/GVFS/GVFS.Platform.Mac/MacFileBasedLock.cs b/GVFS/GVFS.Platform.Mac/MacFileBasedLock.cs index c781a2e13..0d64ad45c 100644 --- a/GVFS/GVFS.Platform.Mac/MacFileBasedLock.cs +++ b/GVFS/GVFS.Platform.Mac/MacFileBasedLock.cs @@ -2,11 +2,23 @@ using GVFS.Common.FileSystem; using GVFS.Common.Tracing; using System; - +using System.Runtime.InteropServices; +using System.Text; + namespace GVFS.Platform.Mac { public class MacFileBasedLock : FileBasedLock { + private const int InvalidFileDescriptor = -1; + + private const int LockSh = 1; // #define LOCK_SH 1 /* shared lock */ + private const int LockEx = 2; // #define LOCK_EX 2 /* exclusive lock */ + private const int LockNb = 4; // #define LOCK_NB 4 /* don't block when locking */ + private const int LockUn = 8; // #define LOCK_UN 8 /* unlock */ + + int lockFileDescriptor; + bool lockAcquired; + public MacFileBasedLock( PhysicalFileSystem fileSystem, ITracer tracer, @@ -14,15 +26,94 @@ public MacFileBasedLock( string signature) : base (fileSystem, tracer, lockPath, signature) { + this.lockFileDescriptor = InvalidFileDescriptor; } public override bool TryAcquireLock() { - throw new NotImplementedException(); + if (this.lockAcquired) + { + return true; + } + + if (this.lockFileDescriptor == InvalidFileDescriptor) + { + this.lockFileDescriptor = Creat(this.LockPath, Convert.ToInt32("644", 8)); + if (this.lockFileDescriptor == InvalidFileDescriptor) + { + int errno = Marshal.GetLastWin32Error(); + + this.Tracer.RelatedWarning($"Failed to create lock file descriptor for '{this.LockPath}': {errno}"); + + return false; + } + } + + if (Flock(this.lockFileDescriptor, LockEx | LockNb) != 0) + { + int errno = Marshal.GetLastWin32Error(); + + // Log error if not EWOULDBLOCK + this.Tracer.RelatedInfo($"Failed to acquire lock for '{this.LockPath}': {errno}"); + + return false; + } + + byte[] signatureBytes = Encoding.UTF8.GetBytes(this.Signature); + long bytesWritten = Write( + this.lockFileDescriptor, + signatureBytes, + Convert.ToUInt64(signatureBytes.Length)); + + if (bytesWritten == -1) + { + int errno = Marshal.GetLastWin32Error(); + + this.Tracer.RelatedWarning($"Failed to write signature for '{this.LockPath}': {errno}"); + } + + this.Tracer.RelatedInfo($"Lock acquired for for '{this.LockPath}'"); + + return true; } public override void Dispose() { + if (this.lockAcquired) + { + if (Flock(this.lockFileDescriptor, LockUn) != 0) + { + this.Tracer.RelatedWarning($"Failed to release lock for: '{this.LockPath}'"); + } + + this.Tracer.RelatedInfo($"Lock released: '{this.LockPath}'"); + + this.lockAcquired = false; + } + + if (this.lockFileDescriptor != InvalidFileDescriptor) + { + if (Close(this.lockFileDescriptor) != 0) + { + int errno = Marshal.GetLastWin32Error(); + + this.Tracer.RelatedWarning($"Failed to close fd for lock: '{this.LockPath}'"); + } + + this.lockFileDescriptor = InvalidFileDescriptor; + } } + + [DllImport("libc", EntryPoint = "creat", SetLastError = true)] + private static extern int Creat(string pathname, int mode); + + [DllImport("libc", EntryPoint = "write", SetLastError = true)] + private static extern long Write(int fd, byte[] buf, ulong count); + + [DllImport("libc", EntryPoint = "close", SetLastError = true)] + private static extern int Close(int fd); + + [DllImport("libc", EntryPoint = "flock", SetLastError = true)] + private static extern int Flock(int fd, int operation); } } diff --git a/GVFS/GVFS.Platform.Mac/MacFileSystem.cs b/GVFS/GVFS.Platform.Mac/MacFileSystem.cs index 96ed8681a..4e72868a9 100644 --- a/GVFS/GVFS.Platform.Mac/MacFileSystem.cs +++ b/GVFS/GVFS.Platform.Mac/MacFileSystem.cs @@ -1,5 +1,4 @@ -using GVFS.Common; -using GVFS.Common.FileSystem; +using GVFS.Common.FileSystem; using System.IO; using System.Runtime.InteropServices; From eca1bd268e3fa0da11b04252abf0b03cad003a14 Mon Sep 17 00:00:00 2001 From: William Baker Date: Thu, 23 Aug 2018 16:40:11 -0700 Subject: [PATCH 070/272] Fix StyleCop errors --- GVFS/GVFS.Platform.Mac/MacFileBasedLock.cs | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/GVFS/GVFS.Platform.Mac/MacFileBasedLock.cs b/GVFS/GVFS.Platform.Mac/MacFileBasedLock.cs index 0d64ad45c..ea7ba63fc 100644 --- a/GVFS/GVFS.Platform.Mac/MacFileBasedLock.cs +++ b/GVFS/GVFS.Platform.Mac/MacFileBasedLock.cs @@ -16,15 +16,15 @@ public class MacFileBasedLock : FileBasedLock private const int LockNb = 4; // #define LOCK_NB 4 /* don't block when locking */ private const int LockUn = 8; // #define LOCK_UN 8 /* unlock */ - int lockFileDescriptor; - bool lockAcquired; + private int lockFileDescriptor; + private bool lockAcquired; public MacFileBasedLock( PhysicalFileSystem fileSystem, ITracer tracer, string lockPath, string signature) - : base (fileSystem, tracer, lockPath, signature) + : base(fileSystem, tracer, lockPath, signature) { this.lockFileDescriptor = InvalidFileDescriptor; } From 18611bed628d3c8333b0c719a94cd55db89c0cfc Mon Sep 17 00:00:00 2001 From: Saeed Noursalehi Date: Fri, 24 Aug 2018 11:14:24 -0700 Subject: [PATCH 071/272] Mark the new test as Windows-only --- .../Tests/EnlistmentPerFixture/MultithreadedReadWriteTests.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/GVFS/GVFS.FunctionalTests/Tests/EnlistmentPerFixture/MultithreadedReadWriteTests.cs b/GVFS/GVFS.FunctionalTests/Tests/EnlistmentPerFixture/MultithreadedReadWriteTests.cs index 5c0167b5b..9c3a21e0c 100644 --- a/GVFS/GVFS.FunctionalTests/Tests/EnlistmentPerFixture/MultithreadedReadWriteTests.cs +++ b/GVFS/GVFS.FunctionalTests/Tests/EnlistmentPerFixture/MultithreadedReadWriteTests.cs @@ -15,6 +15,7 @@ namespace GVFS.FunctionalTests.Tests.EnlistmentPerFixture public class MultithreadedReadWriteTests : TestsWithEnlistmentPerFixture { [TestCase, Order(1)] + [Category(Categories.Windows)] public void CanReadVirtualFileInParallel() { // Note: This test MUST go first, or else it needs to ensure that it is reading a unique path compared to the @@ -30,7 +31,6 @@ public void CanReadVirtualFileInParallel() Thread[] threads = new Thread[32]; for (int i = 0; i < threads.Length; ++i) { - int myIndex = i; threads[i] = new Thread(() => { try From 540aaf589705f90a66341ea3dd2d0eca808928df Mon Sep 17 00:00:00 2001 From: William Baker Date: Fri, 24 Aug 2018 11:35:28 -0700 Subject: [PATCH 072/272] Cleanup and changes for PR feedback --- GVFS/GVFS.Common/FileBasedLock.cs | 10 +- GVFS/GVFS.Platform.Mac/MacFileBasedLock.cs | 128 +++++++++++------- .../WindowsFileBasedLock.cs | 4 +- ProjFS.Mac/PrjFSLib/PrjFSLib.cpp | 1 + 4 files changed, 90 insertions(+), 53 deletions(-) diff --git a/GVFS/GVFS.Common/FileBasedLock.cs b/GVFS/GVFS.Common/FileBasedLock.cs index a216ecc88..bb086fcf1 100644 --- a/GVFS/GVFS.Common/FileBasedLock.cs +++ b/GVFS/GVFS.Common/FileBasedLock.cs @@ -6,11 +6,6 @@ namespace GVFS.Common { public abstract class FileBasedLock : IDisposable { - protected readonly PhysicalFileSystem FileSystem; - protected readonly string LockPath; - protected readonly ITracer Tracer; - protected readonly string Signature; - public FileBasedLock( PhysicalFileSystem fileSystem, ITracer tracer, @@ -23,6 +18,11 @@ public FileBasedLock( this.Signature = signature; } + protected PhysicalFileSystem FileSystem { get; } + protected string LockPath { get; } + protected ITracer Tracer { get; } + protected string Signature { get; } + public abstract bool TryAcquireLock(); public abstract void Dispose(); diff --git a/GVFS/GVFS.Platform.Mac/MacFileBasedLock.cs b/GVFS/GVFS.Platform.Mac/MacFileBasedLock.cs index ea7ba63fc..15aa6ec0d 100644 --- a/GVFS/GVFS.Platform.Mac/MacFileBasedLock.cs +++ b/GVFS/GVFS.Platform.Mac/MacFileBasedLock.cs @@ -9,15 +9,33 @@ namespace GVFS.Platform.Mac { public class MacFileBasedLock : FileBasedLock { + private static readonly ushort FileMode644 = Convert.ToUInt16("644", 8); + + // #define O_WRONLY 0x0001 /* open for writing only */ + private const int OpenWriteOnly = 0x0001; + + // #define O_CREAT 0x0200 /* create if nonexistant */ + private const int OpenCreate = 0x0200; + + // #define EINTR 4 /* Interrupted system call */ + private const int EIntr = 4; + + // #define EAGAIN 35 /* Resource temporarily unavailable */ + // #define EWOULDBLOCK EAGAIN /* Operation would block */ + private const int EWouldBlock = 35; + private const int InvalidFileDescriptor = -1; - private const int LockSh = 1; // #define LOCK_SH 1 /* shared lock */ - private const int LockEx = 2; // #define LOCK_EX 2 /* exclusive lock */ - private const int LockNb = 4; // #define LOCK_NB 4 /* don't block when locking */ - private const int LockUn = 8; // #define LOCK_UN 8 /* unlock */ + [Flags] + private enum FLockOperations + { + LockSh = 1, // #define LOCK_SH 1 /* shared lock */ + LockEx = 2, // #define LOCK_EX 2 /* exclusive lock */ + LockNb = 4, // #define LOCK_NB 4 /* don't block when locking */ + LockUn = 8 // #define LOCK_UN 8 /* unlock */ + } private int lockFileDescriptor; - private bool lockAcquired; public MacFileBasedLock( PhysicalFileSystem fileSystem, @@ -31,81 +49,99 @@ public MacFileBasedLock( public override bool TryAcquireLock() { - if (this.lockAcquired) - { - return true; - } - if (this.lockFileDescriptor == InvalidFileDescriptor) { - this.lockFileDescriptor = Creat(this.LockPath, Convert.ToInt32("644", 8)); + this.lockFileDescriptor = Open(this.LockPath, OpenCreate | OpenWriteOnly, FileMode644); if (this.lockFileDescriptor == InvalidFileDescriptor) { int errno = Marshal.GetLastWin32Error(); - - this.Tracer.RelatedWarning($"Failed to create lock file descriptor for '{this.LockPath}': {errno}"); - + EventMetadata metadata = this.CreateEventMetadata(errno); + this.Tracer.RelatedWarning( + metadata, + $"{nameof(MacFileBasedLock)}.{nameof(this.TryAcquireLock)}: Failed to open lock file"); + return false; } } - if (Flock(this.lockFileDescriptor, LockEx | LockNb) != 0) + if (FLock(this.lockFileDescriptor, (int)(FLockOperations.LockEx | FLockOperations.LockNb)) != 0) { int errno = Marshal.GetLastWin32Error(); - - // Log error if not EWOULDBLOCK - this.Tracer.RelatedInfo($"Failed to acquire lock for '{this.LockPath}': {errno}"); + if (errno != EIntr && errno != EWouldBlock) + { + EventMetadata metadata = this.CreateEventMetadata(errno); + this.Tracer.RelatedWarning( + metadata, + $"{nameof(MacFileBasedLock)}.{nameof(this.TryAcquireLock)}: Unexpected error when locking file"); + } return false; } - byte[] signatureBytes = Encoding.UTF8.GetBytes(this.Signature); - long bytesWritten = Write( - this.lockFileDescriptor, - signatureBytes, - Convert.ToUInt64(signatureBytes.Length)); - - if (bytesWritten == -1) + if (FTruncate(this.lockFileDescriptor, 0) == 0) { - int errno = Marshal.GetLastWin32Error(); + byte[] signatureBytes = Encoding.UTF8.GetBytes(this.Signature); + long bytesWritten = Write( + this.lockFileDescriptor, + signatureBytes, + Convert.ToUInt64(signatureBytes.Length)); - this.Tracer.RelatedWarning($"Failed to write signature for '{this.LockPath}': {errno}"); + if (bytesWritten == -1) + { + int errno = Marshal.GetLastWin32Error(); + EventMetadata metadata = this.CreateEventMetadata(errno); + this.Tracer.RelatedWarning( + metadata, + $"{nameof(MacFileBasedLock)}.{nameof(this.TryAcquireLock)}: Failed to write signature"); + } + } + else + { + int errno = Marshal.GetLastWin32Error(); + EventMetadata metadata = this.CreateEventMetadata(errno); + this.Tracer.RelatedWarning( + metadata, + $"{nameof(MacFileBasedLock)}.{nameof(this.TryAcquireLock)}: Failed to truncate lock file"); } - - this.Tracer.RelatedInfo($"Lock acquired for for '{this.LockPath}'"); return true; } public override void Dispose() { - if (this.lockAcquired) - { - if (Flock(this.lockFileDescriptor, LockUn) != 0) - { - this.Tracer.RelatedWarning($"Failed to release lock for: '{this.LockPath}'"); - } - - this.Tracer.RelatedInfo($"Lock released: '{this.LockPath}'"); - - this.lockAcquired = false; - } - if (this.lockFileDescriptor != InvalidFileDescriptor) { if (Close(this.lockFileDescriptor) != 0) { int errno = Marshal.GetLastWin32Error(); - - this.Tracer.RelatedWarning($"Failed to close fd for lock: '{this.LockPath}'"); + EventMetadata metadata = this.CreateEventMetadata(errno); + this.Tracer.RelatedWarning( + metadata, + $"{nameof(MacFileBasedLock)}.{nameof(this.Dispose)}: Error when closing lock fd"); } this.lockFileDescriptor = InvalidFileDescriptor; } } - [DllImport("libc", EntryPoint = "creat", SetLastError = true)] - private static extern int Creat(string pathname, int mode); + private EventMetadata CreateEventMetadata(int errno = 0) + { + EventMetadata metadata = new EventMetadata(); + metadata.Add("Area", nameof(MacFileBasedLock)); + metadata.Add(nameof(this.LockPath), this.LockPath); + metadata.Add(nameof(this.Signature), this.Signature); + if (errno != 0) + { + metadata.Add(nameof(errno), errno); + } + return metadata; + } + + [DllImport("libc", EntryPoint = "open", SetLastError = true)] + private static extern int Open(string pathname, int flags, ushort mode); + + [DllImport("libc", EntryPoint = "ftruncate", SetLastError = true)] + private static extern long FTruncate(int fd, long length); [DllImport("libc", EntryPoint = "write", SetLastError = true)] private static extern long Write(int fd, byte[] buf, ulong count); @@ -114,6 +150,6 @@ public override void Dispose() private static extern int Close(int fd); [DllImport("libc", EntryPoint = "flock", SetLastError = true)] - private static extern int Flock(int fd, int operation); + private static extern int FLock(int fd, int operation); } } diff --git a/GVFS/GVFS.Platform.Windows/WindowsFileBasedLock.cs b/GVFS/GVFS.Platform.Windows/WindowsFileBasedLock.cs index 0dacc4172..436f28b2b 100644 --- a/GVFS/GVFS.Platform.Windows/WindowsFileBasedLock.cs +++ b/GVFS/GVFS.Platform.Windows/WindowsFileBasedLock.cs @@ -129,8 +129,8 @@ private EventMetadata CreateLockMetadata(Exception exception = null) { EventMetadata metadata = new EventMetadata(); metadata.Add("Area", EtwArea); - metadata.Add("LockPath", this.LockPath); - metadata.Add("Signature", this.Signature); + metadata.Add(nameof(this.LockPath), this.LockPath); + metadata.Add(nameof(this.Signature), this.Signature); if (exception != null) { metadata.Add("Exception", exception.ToString()); diff --git a/ProjFS.Mac/PrjFSLib/PrjFSLib.cpp b/ProjFS.Mac/PrjFSLib/PrjFSLib.cpp index dd8ab7cec..524e2fe82 100644 --- a/ProjFS.Mac/PrjFSLib/PrjFSLib.cpp +++ b/ProjFS.Mac/PrjFSLib/PrjFSLib.cpp @@ -331,6 +331,7 @@ PrjFS_Result PrjFS_WritePlaceholderFile( goto CleanupAndFail; } + // TODO(Mac): Only call chmod is fileMode is different than the default file mode if (chmod(fullPath, fileMode)) { goto CleanupAndFail; From 394b337c62968227e3eb92d6927ed13bcc0397bd Mon Sep 17 00:00:00 2001 From: William Baker Date: Fri, 24 Aug 2018 11:54:11 -0700 Subject: [PATCH 073/272] Fix StyleCop errors --- GVFS/GVFS.Platform.Mac/MacFileBasedLock.cs | 64 ++++++++++++---------- 1 file changed, 34 insertions(+), 30 deletions(-) diff --git a/GVFS/GVFS.Platform.Mac/MacFileBasedLock.cs b/GVFS/GVFS.Platform.Mac/MacFileBasedLock.cs index 15aa6ec0d..58e0fc79f 100644 --- a/GVFS/GVFS.Platform.Mac/MacFileBasedLock.cs +++ b/GVFS/GVFS.Platform.Mac/MacFileBasedLock.cs @@ -9,8 +9,6 @@ namespace GVFS.Platform.Mac { public class MacFileBasedLock : FileBasedLock { - private static readonly ushort FileMode644 = Convert.ToUInt16("644", 8); - // #define O_WRONLY 0x0001 /* open for writing only */ private const int OpenWriteOnly = 0x0001; @@ -26,14 +24,7 @@ public class MacFileBasedLock : FileBasedLock private const int InvalidFileDescriptor = -1; - [Flags] - private enum FLockOperations - { - LockSh = 1, // #define LOCK_SH 1 /* shared lock */ - LockEx = 2, // #define LOCK_EX 2 /* exclusive lock */ - LockNb = 4, // #define LOCK_NB 4 /* don't block when locking */ - LockUn = 8 // #define LOCK_UN 8 /* unlock */ - } + private static readonly ushort FileMode644 = Convert.ToUInt16("644", 8); private int lockFileDescriptor; @@ -47,11 +38,20 @@ public MacFileBasedLock( this.lockFileDescriptor = InvalidFileDescriptor; } + [Flags] + private enum FLockOperations + { + LockSh = 1, // #define LOCK_SH 1 /* shared lock */ + LockEx = 2, // #define LOCK_EX 2 /* exclusive lock */ + LockNb = 4, // #define LOCK_NB 4 /* don't block when locking */ + LockUn = 8 // #define LOCK_UN 8 /* unlock */ + } + public override bool TryAcquireLock() { if (this.lockFileDescriptor == InvalidFileDescriptor) { - this.lockFileDescriptor = Open(this.LockPath, OpenCreate | OpenWriteOnly, FileMode644); + this.lockFileDescriptor = NativeMethods.Open(this.LockPath, OpenCreate | OpenWriteOnly, FileMode644); if (this.lockFileDescriptor == InvalidFileDescriptor) { int errno = Marshal.GetLastWin32Error(); @@ -64,7 +64,7 @@ public override bool TryAcquireLock() } } - if (FLock(this.lockFileDescriptor, (int)(FLockOperations.LockEx | FLockOperations.LockNb)) != 0) + if (NativeMethods.FLock(this.lockFileDescriptor, (int)(FLockOperations.LockEx | FLockOperations.LockNb)) != 0) { int errno = Marshal.GetLastWin32Error(); if (errno != EIntr && errno != EWouldBlock) @@ -78,10 +78,10 @@ public override bool TryAcquireLock() return false; } - if (FTruncate(this.lockFileDescriptor, 0) == 0) + if (NativeMethods.FTruncate(this.lockFileDescriptor, 0) == 0) { byte[] signatureBytes = Encoding.UTF8.GetBytes(this.Signature); - long bytesWritten = Write( + long bytesWritten = NativeMethods.Write( this.lockFileDescriptor, signatureBytes, Convert.ToUInt64(signatureBytes.Length)); @@ -111,7 +111,7 @@ public override void Dispose() { if (this.lockFileDescriptor != InvalidFileDescriptor) { - if (Close(this.lockFileDescriptor) != 0) + if (NativeMethods.Close(this.lockFileDescriptor) != 0) { int errno = Marshal.GetLastWin32Error(); EventMetadata metadata = this.CreateEventMetadata(errno); @@ -127,29 +127,33 @@ public override void Dispose() private EventMetadata CreateEventMetadata(int errno = 0) { EventMetadata metadata = new EventMetadata(); - metadata.Add("Area", nameof(MacFileBasedLock)); + metadata.Add("Area", "MacFileBasedLock"); metadata.Add(nameof(this.LockPath), this.LockPath); metadata.Add(nameof(this.Signature), this.Signature); if (errno != 0) { metadata.Add(nameof(errno), errno); } + return metadata; } - [DllImport("libc", EntryPoint = "open", SetLastError = true)] - private static extern int Open(string pathname, int flags, ushort mode); - - [DllImport("libc", EntryPoint = "ftruncate", SetLastError = true)] - private static extern long FTruncate(int fd, long length); - - [DllImport("libc", EntryPoint = "write", SetLastError = true)] - private static extern long Write(int fd, byte[] buf, ulong count); - - [DllImport("libc", EntryPoint = "close", SetLastError = true)] - private static extern int Close(int fd); - - [DllImport("libc", EntryPoint = "flock", SetLastError = true)] - private static extern int FLock(int fd, int operation); + private static class NativeMethods + { + [DllImport("libc", EntryPoint = "open", SetLastError = true)] + public static extern int Open(string pathname, int flags, ushort mode); + + [DllImport("libc", EntryPoint = "ftruncate", SetLastError = true)] + public static extern long FTruncate(int fd, long length); + + [DllImport("libc", EntryPoint = "write", SetLastError = true)] + public static extern long Write(int fd, byte[] buf, ulong count); + + [DllImport("libc", EntryPoint = "close", SetLastError = true)] + public static extern int Close(int fd); + + [DllImport("libc", EntryPoint = "flock", SetLastError = true)] + public static extern int FLock(int fd, int operation); + } } } From 467c6533ea341f39d35dd9c959aa37d29c768d31 Mon Sep 17 00:00:00 2001 From: William Baker Date: Fri, 24 Aug 2018 14:30:26 -0700 Subject: [PATCH 074/272] Remove code for writing signature --- GVFS/GVFS.Common/FileBasedLock.cs | 5 +- GVFS/GVFS.Common/GVFSPlatform.cs | 3 +- GVFS/GVFS.Common/LocalCacheResolver.cs | 3 +- GVFS/GVFS.Common/Prefetch/CommitPrefetcher.cs | 3 +- GVFS/GVFS.Platform.Mac/MacFileBasedLock.cs | 120 +++++++----------- GVFS/GVFS.Platform.Mac/MacPlatform.cs | 5 +- .../WindowsFileBasedLock.cs | 30 +---- GVFS/GVFS.Platform.Windows/WindowsPlatform.cs | 5 +- .../Windows/WindowsFileBasedLockTests.cs | 18 ++- .../Mock/Common/MockFileBasedLock.cs | 5 +- .../Mock/Common/MockPlatform.cs | 4 +- .../FileSystemCallbacks.cs | 3 +- ProjFS.Mac/PrjFSLib/PrjFSLib.cpp | 2 +- 13 files changed, 82 insertions(+), 124 deletions(-) diff --git a/GVFS/GVFS.Common/FileBasedLock.cs b/GVFS/GVFS.Common/FileBasedLock.cs index bb086fcf1..560ad091f 100644 --- a/GVFS/GVFS.Common/FileBasedLock.cs +++ b/GVFS/GVFS.Common/FileBasedLock.cs @@ -9,19 +9,16 @@ public abstract class FileBasedLock : IDisposable public FileBasedLock( PhysicalFileSystem fileSystem, ITracer tracer, - string lockPath, - string signature) + string lockPath) { this.FileSystem = fileSystem; this.Tracer = tracer; this.LockPath = lockPath; - this.Signature = signature; } protected PhysicalFileSystem FileSystem { get; } protected string LockPath { get; } protected ITracer Tracer { get; } - protected string Signature { get; } public abstract bool TryAcquireLock(); diff --git a/GVFS/GVFS.Common/GVFSPlatform.cs b/GVFS/GVFS.Common/GVFSPlatform.cs index fe0b70255..acb0aa2f3 100644 --- a/GVFS/GVFS.Common/GVFSPlatform.cs +++ b/GVFS/GVFS.Common/GVFSPlatform.cs @@ -62,8 +62,7 @@ public static void Register(GVFSPlatform platform) public abstract FileBasedLock CreateFileBasedLock( PhysicalFileSystem fileSystem, ITracer tracer, - string lockPath, - string signature); + string lockPath); public bool TryGetNormalizedPathRoot(string path, out string pathRoot, out string errorMessage) { diff --git a/GVFS/GVFS.Common/LocalCacheResolver.cs b/GVFS/GVFS.Common/LocalCacheResolver.cs index 0a3fdaa73..12a5a7c23 100644 --- a/GVFS/GVFS.Common/LocalCacheResolver.cs +++ b/GVFS/GVFS.Common/LocalCacheResolver.cs @@ -67,8 +67,7 @@ public bool TryGetLocalCacheKeyFromLocalConfigOrRemoteCacheServers( using (FileBasedLock mappingLock = GVFSPlatform.Instance.CreateFileBasedLock( this.fileSystem, tracer, - lockPath, - this.enlistment.EnlistmentRoot)) + lockPath)) { if (!this.TryAcquireLockWithRetries(tracer, mappingLock)) { diff --git a/GVFS/GVFS.Common/Prefetch/CommitPrefetcher.cs b/GVFS/GVFS.Common/Prefetch/CommitPrefetcher.cs index 60d7a3d38..eb2237cbc 100644 --- a/GVFS/GVFS.Common/Prefetch/CommitPrefetcher.cs +++ b/GVFS/GVFS.Common/Prefetch/CommitPrefetcher.cs @@ -27,8 +27,7 @@ public static bool TryPrefetchCommitsAndTrees( using (FileBasedLock prefetchLock = GVFSPlatform.Instance.CreateFileBasedLock( fileSystem, tracer, - Path.Combine(enlistment.GitPackRoot, PrefetchCommitsAndTreesLock), - enlistment.EnlistmentRoot)) + Path.Combine(enlistment.GitPackRoot, PrefetchCommitsAndTreesLock))) { WaitUntilLockIsAcquired(tracer, prefetchLock); long maxGoodTimeStamp; diff --git a/GVFS/GVFS.Platform.Mac/MacFileBasedLock.cs b/GVFS/GVFS.Platform.Mac/MacFileBasedLock.cs index 58e0fc79f..52157be55 100644 --- a/GVFS/GVFS.Platform.Mac/MacFileBasedLock.cs +++ b/GVFS/GVFS.Platform.Mac/MacFileBasedLock.cs @@ -2,57 +2,36 @@ using GVFS.Common.FileSystem; using GVFS.Common.Tracing; using System; -using System.Runtime.InteropServices; -using System.Text; +using System.IO; +using System.Runtime.InteropServices; namespace GVFS.Platform.Mac { public class MacFileBasedLock : FileBasedLock { - // #define O_WRONLY 0x0001 /* open for writing only */ - private const int OpenWriteOnly = 0x0001; - - // #define O_CREAT 0x0200 /* create if nonexistant */ - private const int OpenCreate = 0x0200; - - // #define EINTR 4 /* Interrupted system call */ - private const int EIntr = 4; - - // #define EAGAIN 35 /* Resource temporarily unavailable */ - // #define EWOULDBLOCK EAGAIN /* Operation would block */ - private const int EWouldBlock = 35; - - private const int InvalidFileDescriptor = -1; - - private static readonly ushort FileMode644 = Convert.ToUInt16("644", 8); - private int lockFileDescriptor; public MacFileBasedLock( PhysicalFileSystem fileSystem, ITracer tracer, - string lockPath, - string signature) - : base(fileSystem, tracer, lockPath, signature) - { - this.lockFileDescriptor = InvalidFileDescriptor; - } - - [Flags] - private enum FLockOperations + string lockPath) + : base(fileSystem, tracer, lockPath) { - LockSh = 1, // #define LOCK_SH 1 /* shared lock */ - LockEx = 2, // #define LOCK_EX 2 /* exclusive lock */ - LockNb = 4, // #define LOCK_NB 4 /* don't block when locking */ - LockUn = 8 // #define LOCK_UN 8 /* unlock */ + this.lockFileDescriptor = NativeMethods.InvalidFileDescriptor; } public override bool TryAcquireLock() { - if (this.lockFileDescriptor == InvalidFileDescriptor) + if (this.lockFileDescriptor == NativeMethods.InvalidFileDescriptor) { - this.lockFileDescriptor = NativeMethods.Open(this.LockPath, OpenCreate | OpenWriteOnly, FileMode644); - if (this.lockFileDescriptor == InvalidFileDescriptor) + this.FileSystem.CreateDirectory(Path.GetDirectoryName(this.LockPath)); + + this.lockFileDescriptor = NativeMethods.Open( + this.LockPath, + NativeMethods.OpenCreate | NativeMethods.OpenWriteOnly, + NativeMethods.FileMode644); + + if (this.lockFileDescriptor == NativeMethods.InvalidFileDescriptor) { int errno = Marshal.GetLastWin32Error(); EventMetadata metadata = this.CreateEventMetadata(errno); @@ -64,10 +43,10 @@ public override bool TryAcquireLock() } } - if (NativeMethods.FLock(this.lockFileDescriptor, (int)(FLockOperations.LockEx | FLockOperations.LockNb)) != 0) + if (NativeMethods.FLock(this.lockFileDescriptor, NativeMethods.LockEx | NativeMethods.LockNb) != 0) { int errno = Marshal.GetLastWin32Error(); - if (errno != EIntr && errno != EWouldBlock) + if (errno != NativeMethods.EIntr && errno != NativeMethods.EWouldBlock) { EventMetadata metadata = this.CreateEventMetadata(errno); this.Tracer.RelatedWarning( @@ -78,41 +57,21 @@ public override bool TryAcquireLock() return false; } - if (NativeMethods.FTruncate(this.lockFileDescriptor, 0) == 0) - { - byte[] signatureBytes = Encoding.UTF8.GetBytes(this.Signature); - long bytesWritten = NativeMethods.Write( - this.lockFileDescriptor, - signatureBytes, - Convert.ToUInt64(signatureBytes.Length)); - - if (bytesWritten == -1) - { - int errno = Marshal.GetLastWin32Error(); - EventMetadata metadata = this.CreateEventMetadata(errno); - this.Tracer.RelatedWarning( - metadata, - $"{nameof(MacFileBasedLock)}.{nameof(this.TryAcquireLock)}: Failed to write signature"); - } - } - else - { - int errno = Marshal.GetLastWin32Error(); - EventMetadata metadata = this.CreateEventMetadata(errno); - this.Tracer.RelatedWarning( - metadata, - $"{nameof(MacFileBasedLock)}.{nameof(this.TryAcquireLock)}: Failed to truncate lock file"); - } - return true; } public override void Dispose() { - if (this.lockFileDescriptor != InvalidFileDescriptor) + if (this.lockFileDescriptor != NativeMethods.InvalidFileDescriptor) { if (NativeMethods.Close(this.lockFileDescriptor) != 0) { + // Failures of close() are logged for diagnostic purposes only. + // It's possible that errors from a previous operation (e.g. write(2)) + // are only reported in close(). We should *not* retry the close() if + // it fails since it may cause a re-used file descriptor from another + // thrad to be closed. + int errno = Marshal.GetLastWin32Error(); EventMetadata metadata = this.CreateEventMetadata(errno); this.Tracer.RelatedWarning( @@ -120,7 +79,7 @@ public override void Dispose() $"{nameof(MacFileBasedLock)}.{nameof(this.Dispose)}: Error when closing lock fd"); } - this.lockFileDescriptor = InvalidFileDescriptor; + this.lockFileDescriptor = NativeMethods.InvalidFileDescriptor; } } @@ -129,7 +88,6 @@ private EventMetadata CreateEventMetadata(int errno = 0) EventMetadata metadata = new EventMetadata(); metadata.Add("Area", "MacFileBasedLock"); metadata.Add(nameof(this.LockPath), this.LockPath); - metadata.Add(nameof(this.Signature), this.Signature); if (errno != 0) { metadata.Add(nameof(errno), errno); @@ -139,16 +97,32 @@ private EventMetadata CreateEventMetadata(int errno = 0) } private static class NativeMethods - { + { + // #define O_WRONLY 0x0001 /* open for writing only */ + public const int OpenWriteOnly = 0x0001; + + // #define O_CREAT 0x0200 /* create if nonexistant */ + public const int OpenCreate = 0x0200; + + // #define EINTR 4 /* Interrupted system call */ + public const int EIntr = 4; + + // #define EAGAIN 35 /* Resource temporarily unavailable */ + // #define EWOULDBLOCK EAGAIN /* Operation would block */ + public const int EWouldBlock = 35; + + public const int LockSh = 1; // #define LOCK_SH 1 /* shared lock */ + public const int LockEx = 2; // #define LOCK_EX 2 /* exclusive lock */ + public const int LockNb = 4; // #define LOCK_NB 4 /* don't block when locking */ + public const int LockUn = 8; // #define LOCK_UN 8 /* unlock */ + + public const int InvalidFileDescriptor = -1; + + public static readonly ushort FileMode644 = Convert.ToUInt16("644", 8); + [DllImport("libc", EntryPoint = "open", SetLastError = true)] public static extern int Open(string pathname, int flags, ushort mode); - [DllImport("libc", EntryPoint = "ftruncate", SetLastError = true)] - public static extern long FTruncate(int fd, long length); - - [DllImport("libc", EntryPoint = "write", SetLastError = true)] - public static extern long Write(int fd, byte[] buf, ulong count); - [DllImport("libc", EntryPoint = "close", SetLastError = true)] public static extern int Close(int fd); diff --git a/GVFS/GVFS.Platform.Mac/MacPlatform.cs b/GVFS/GVFS.Platform.Mac/MacPlatform.cs index e2c641484..c5683de6b 100644 --- a/GVFS/GVFS.Platform.Mac/MacPlatform.cs +++ b/GVFS/GVFS.Platform.Mac/MacPlatform.cs @@ -131,10 +131,9 @@ public override bool IsGitStatusCacheSupported() public override FileBasedLock CreateFileBasedLock( PhysicalFileSystem fileSystem, ITracer tracer, - string lockPath, - string signature) + string lockPath) { - return new MacFileBasedLock(fileSystem, tracer, lockPath, signature); + return new MacFileBasedLock(fileSystem, tracer, lockPath); } } } diff --git a/GVFS/GVFS.Platform.Windows/WindowsFileBasedLock.cs b/GVFS/GVFS.Platform.Windows/WindowsFileBasedLock.cs index 436f28b2b..be66554a5 100644 --- a/GVFS/GVFS.Platform.Windows/WindowsFileBasedLock.cs +++ b/GVFS/GVFS.Platform.Windows/WindowsFileBasedLock.cs @@ -31,9 +31,8 @@ public class WindowsFileBasedLock : FileBasedLock public WindowsFileBasedLock( PhysicalFileSystem fileSystem, ITracer tracer, - string lockPath, - string signature) - : base(fileSystem, tracer, lockPath, signature) + string lockPath) + : base(fileSystem, tracer, lockPath) { } @@ -43,9 +42,9 @@ public override bool TryAcquireLock() { lock (this.deleteOnCloseStreamLock) { - if (this.IsOpen()) + if (this.deleteOnCloseStream != null) { - return true; + throw new InvalidOperationException("Lock has already been acquired"); } this.FileSystem.CreateDirectory(Path.GetDirectoryName(this.LockPath)); @@ -58,16 +57,6 @@ public override bool TryAcquireLock() FileOptions.DeleteOnClose, callFlushFileBuffers: false); - // Pass in true for leaveOpen to ensure that lockStream stays open - using (StreamWriter writer = new StreamWriter( - this.deleteOnCloseStream, - UTF8NoBOM, - DefaultStreamWriterBufferSize, - leaveOpen: true)) - { - this.WriteSignature(writer); - } - return true; } } @@ -115,22 +104,11 @@ public override void Dispose() this.DisposeStream(); } - private bool IsOpen() - { - return this.deleteOnCloseStream != null; - } - - private void WriteSignature(StreamWriter writer) - { - writer.WriteLine(this.Signature); - } - private EventMetadata CreateLockMetadata(Exception exception = null) { EventMetadata metadata = new EventMetadata(); metadata.Add("Area", EtwArea); metadata.Add(nameof(this.LockPath), this.LockPath); - metadata.Add(nameof(this.Signature), this.Signature); if (exception != null) { metadata.Add("Exception", exception.ToString()); diff --git a/GVFS/GVFS.Platform.Windows/WindowsPlatform.cs b/GVFS/GVFS.Platform.Windows/WindowsPlatform.cs index 11c8708c7..80dde743b 100644 --- a/GVFS/GVFS.Platform.Windows/WindowsPlatform.cs +++ b/GVFS/GVFS.Platform.Windows/WindowsPlatform.cs @@ -256,10 +256,9 @@ public override bool IsGitStatusCacheSupported() public override FileBasedLock CreateFileBasedLock( PhysicalFileSystem fileSystem, ITracer tracer, - string lockPath, - string signature) + string lockPath) { - return new WindowsFileBasedLock(fileSystem, tracer, lockPath, signature); + return new WindowsFileBasedLock(fileSystem, tracer, lockPath); } public override bool TryGetGVFSEnlistmentRoot(string directory, out string enlistmentRoot, out string errorMessage) diff --git a/GVFS/GVFS.UnitTests.Windows/Windows/WindowsFileBasedLockTests.cs b/GVFS/GVFS.UnitTests.Windows/Windows/WindowsFileBasedLockTests.cs index 82ecdfdf4..dcd904628 100644 --- a/GVFS/GVFS.UnitTests.Windows/Windows/WindowsFileBasedLockTests.cs +++ b/GVFS/GVFS.UnitTests.Windows/Windows/WindowsFileBasedLockTests.cs @@ -1,9 +1,11 @@ using GVFS.Common; using GVFS.Platform.Windows; using GVFS.Tests.Should; +using GVFS.UnitTests.Category; using GVFS.UnitTests.Mock.Common; using GVFS.UnitTests.Mock.FileSystem; using NUnit.Framework; +using System; using System.IO; namespace GVFS.UnitTests.Windows @@ -18,13 +20,27 @@ public void CreateLockWhenDirectoryMissing() string lockPath = Path.Combine(parentPath, "lock"); MockTracer tracer = new MockTracer(); FileBasedLockFileSystem fs = new FileBasedLockFileSystem(); - FileBasedLock fileBasedLock = new WindowsFileBasedLock(fs, tracer, lockPath, "signature"); + FileBasedLock fileBasedLock = new WindowsFileBasedLock(fs, tracer, lockPath); fileBasedLock.TryAcquireLock().ShouldBeTrue(); fs.CreateDirectoryPath.ShouldNotBeNull(); fs.CreateDirectoryPath.ShouldEqual(parentPath); } + [TestCase] + [Category(CategoryConstants.ExceptionExpected)] + public void AttemptToAcquireLockWhenAlreadyLocked() + { + string parentPath = Path.Combine("mock:", "path", "to"); + string lockPath = Path.Combine(parentPath, "lock"); + MockTracer tracer = new MockTracer(); + FileBasedLockFileSystem fs = new FileBasedLockFileSystem(); + FileBasedLock fileBasedLock = new WindowsFileBasedLock(fs, tracer, lockPath); + + fileBasedLock.TryAcquireLock().ShouldBeTrue(); + Assert.Throws(() => fileBasedLock.TryAcquireLock()); + } + private class FileBasedLockFileSystem : ConfigurableFileSystem { public string CreateDirectoryPath { get; set; } diff --git a/GVFS/GVFS.UnitTests/Mock/Common/MockFileBasedLock.cs b/GVFS/GVFS.UnitTests/Mock/Common/MockFileBasedLock.cs index 9c59d9c02..c18c707b4 100644 --- a/GVFS/GVFS.UnitTests/Mock/Common/MockFileBasedLock.cs +++ b/GVFS/GVFS.UnitTests/Mock/Common/MockFileBasedLock.cs @@ -9,9 +9,8 @@ public class MockFileBasedLock : FileBasedLock public MockFileBasedLock( PhysicalFileSystem fileSystem, ITracer tracer, - string lockPath, - string signature) - : base(fileSystem, tracer, lockPath, signature) + string lockPath) + : base(fileSystem, tracer, lockPath) { } diff --git a/GVFS/GVFS.UnitTests/Mock/Common/MockPlatform.cs b/GVFS/GVFS.UnitTests/Mock/Common/MockPlatform.cs index 2aff728b5..0062d7e08 100644 --- a/GVFS/GVFS.UnitTests/Mock/Common/MockPlatform.cs +++ b/GVFS/GVFS.UnitTests/Mock/Common/MockPlatform.cs @@ -105,9 +105,9 @@ public override bool IsGitStatusCacheSupported() return true; } - public override FileBasedLock CreateFileBasedLock(PhysicalFileSystem fileSystem, ITracer tracer, string lockPath, string signature) + public override FileBasedLock CreateFileBasedLock(PhysicalFileSystem fileSystem, ITracer tracer, string lockPath) { - return new MockFileBasedLock(fileSystem, tracer, lockPath, signature); + return new MockFileBasedLock(fileSystem, tracer, lockPath); } } } diff --git a/GVFS/GVFS.Virtualization/FileSystemCallbacks.cs b/GVFS/GVFS.Virtualization/FileSystemCallbacks.cs index e50ad7bdc..28dd595f6 100644 --- a/GVFS/GVFS.Virtualization/FileSystemCallbacks.cs +++ b/GVFS/GVFS.Virtualization/FileSystemCallbacks.cs @@ -527,8 +527,7 @@ private void PostFetchJob(List packIndexes) using (FileBasedLock postFetchFileLock = GVFSPlatform.Instance.CreateFileBasedLock( this.context.FileSystem, this.context.Tracer, - Path.Combine(this.context.Enlistment.GitObjectsRoot, PostFetchLock), - this.context.Enlistment.EnlistmentRoot)) + Path.Combine(this.context.Enlistment.GitObjectsRoot, PostFetchLock))) { if (!postFetchFileLock.TryAcquireLock()) { diff --git a/ProjFS.Mac/PrjFSLib/PrjFSLib.cpp b/ProjFS.Mac/PrjFSLib/PrjFSLib.cpp index 524e2fe82..d0b4b77bc 100644 --- a/ProjFS.Mac/PrjFSLib/PrjFSLib.cpp +++ b/ProjFS.Mac/PrjFSLib/PrjFSLib.cpp @@ -331,7 +331,7 @@ PrjFS_Result PrjFS_WritePlaceholderFile( goto CleanupAndFail; } - // TODO(Mac): Only call chmod is fileMode is different than the default file mode + // TODO(Mac): Only call chmod if fileMode is different than the default file mode if (chmod(fullPath, fileMode)) { goto CleanupAndFail; From 9eecaf5dff053b1bc1f29a3066cb706894079b24 Mon Sep 17 00:00:00 2001 From: Derrick Stolee Date: Tue, 28 Aug 2018 09:51:32 -0400 Subject: [PATCH 075/272] RepoRegistry: Properly log error from TryGetNormalizedPath While investigating a user issue with auto mount registration, I noticed that their logs contained a failure from TryGetNormalizedPath, but did not log the error message. Add that message to the logs so we can properly diagnose the issue. Further, the 'normalizedEnlistmentRootPath' parameter becomes null, so do not add it to the dictionary in that case. Instead, it should be logged by the RelatedWarning. Signed-off-by: Derrick Stolee --- GVFS/GVFS.Service/RepoRegistry.cs | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/GVFS/GVFS.Service/RepoRegistry.cs b/GVFS/GVFS.Service/RepoRegistry.cs index a4a7345e6..646691440 100644 --- a/GVFS/GVFS.Service/RepoRegistry.cs +++ b/GVFS/GVFS.Service/RepoRegistry.cs @@ -248,10 +248,15 @@ public Dictionary ReadRegistry() { EventMetadata metadata = new EventMetadata(); metadata.Add("registration.EnlistmentRoot", registration.EnlistmentRoot); + metadata.Add("NormalizedEnlistmentRootPath", normalizedEnlistmentRootPath); + metadata.Add("ErrorMessage", errorMessage); this.tracer.RelatedWarning(metadata, $"{nameof(ReadRegistry)}: Failed to get normalized path name for registed enlistment root"); } - allRepos[normalizedEnlistmentRootPath] = registration; + if (normalizedEnlistmentRootPath != null) + { + allRepos[normalizedEnlistmentRootPath] = registration; + } } catch (Exception e) { From 295532d73e9f073c0e8cdea2db358c6c8fc1f3e4 Mon Sep 17 00:00:00 2001 From: William Baker Date: Tue, 28 Aug 2018 14:13:33 -0700 Subject: [PATCH 076/272] Mac: Support basic projection changes and re-categorize functional tests --- .../Windows/Tests/DiskLayoutUpgradeTests.cs | 2 +- .../Windows/Tests/JunctionAndSubstTests.cs | 2 +- .../Windows/Tests/ServiceTests.cs | 2 +- .../Windows/Tests/SharedCacheUpgradeTests.cs | 2 +- .../Windows/Tests/WindowsFileSystemTests.cs | 2 +- GVFS/GVFS.FunctionalTests/Categories.cs | 8 +- GVFS/GVFS.FunctionalTests/Program.cs | 19 +- .../BasicFileSystemTests.cs | 25 +-- .../EnlistmentPerFixture/CacheServerTests.cs | 1 - .../Tests/EnlistmentPerFixture/CloneTests.cs | 1 - .../EnlistmentPerFixture/DehydrateTests.cs | 1 + .../EnlistmentPerFixture/DiagnoseTests.cs | 1 + .../EnlistmentPerFixture/GVFSLockTests.cs | 21 ++- .../EnlistmentPerFixture/GitFilesTests.cs | 16 +- .../GitMoveRenameTests.cs | 3 - .../GitReadAndGitLockTests.cs | 8 +- .../Tests/EnlistmentPerFixture/MountTests.cs | 4 +- .../MoveRenameFileTests.cs | 1 - .../MoveRenameFileTests_2.cs | 1 - .../MoveRenameFolderTests.cs | 47 +++-- .../MultithreadedReadWriteTests.cs | 3 +- .../EnlistmentPerFixture/PrefetchVerbTests.cs | 17 +- .../PrefetchVerbWithoutSharedCacheTests.cs | 1 + .../EnlistmentPerFixture/UnmountTests.cs | 2 + .../UpdatePlaceholderTests.cs | 1 + .../WorkingDirectoryTests.cs | 9 +- .../CaseOnlyFolderRenameTests.cs | 25 ++- .../PersistedModifiedPathsTests.cs | 2 +- .../PersistedWorkingDirectoryTests.cs | 1 - .../EnlistmentPerTestCase/RepairTests.cs | 1 + .../Tests/FastFetchTests.cs | 1 + .../Tests/GVFSVerbTests.cs | 1 - .../Tests/GitCommands/AddStageTests.cs | 18 +- .../Tests/GitCommands/CheckoutTests.cs | 160 +++++++++------- .../GitCommands/CherryPickConflictTests.cs | 1 + .../GitCommands/DeleteEmptyFolderTests.cs | 3 +- .../Tests/GitCommands/EnumerationMergeTest.cs | 1 + .../Tests/GitCommands/GitCommandsTests.cs | 175 +++++++++--------- .../Tests/GitCommands/GitRepoTests.cs | 153 ++++++++------- .../Tests/GitCommands/HashObjectTests.cs | 3 +- .../Tests/GitCommands/MergeConflictTests.cs | 1 + .../Tests/GitCommands/RebaseConflictTests.cs | 1 + .../Tests/GitCommands/RebaseTests.cs | 2 + .../Tests/GitCommands/ResetHardTests.cs | 2 + .../Tests/GitCommands/ResetMixedTests.cs | 1 + .../Tests/GitCommands/ResetSoftTests.cs | 1 + .../Tests/GitCommands/RmTests.cs | 3 + .../Tests/GitCommands/StatusTests.cs | 10 +- .../Tests/GitCommands/UpdateIndexTests.cs | 1 + .../Tests/GitCommands/UpdateRefTests.cs | 2 + .../MultiEnlistmentTests/ServiceVerbTests.cs | 1 + .../MultiEnlistmentTests/SharedCacheTests.cs | 1 + .../MacFileSystemVirtualizer.cs | 38 +++- .../Mock/Mac/MockVirtualizationInstance.cs | 1 + .../MacFileSystemVirtualizerTests.cs | 14 ++ .../PrjFSLib.Mac.Managed/Interop/PrjFSLib.cs | 18 ++ .../VirtualizationInstance.cs | 29 ++- ProjFS.Mac/PrjFSLib/PrjFSLib.cpp | 71 ++++++- ProjFS.Mac/PrjFSLib/PrjFSLib.h | 6 +- Scripts/Mac/RunFunctionalTests.sh | 2 +- 60 files changed, 603 insertions(+), 347 deletions(-) diff --git a/GVFS/GVFS.FunctionalTests.Windows/Windows/Tests/DiskLayoutUpgradeTests.cs b/GVFS/GVFS.FunctionalTests.Windows/Windows/Tests/DiskLayoutUpgradeTests.cs index ae4f30587..6495a9cd6 100644 --- a/GVFS/GVFS.FunctionalTests.Windows/Windows/Tests/DiskLayoutUpgradeTests.cs +++ b/GVFS/GVFS.FunctionalTests.Windows/Windows/Tests/DiskLayoutUpgradeTests.cs @@ -14,7 +14,7 @@ namespace GVFS.FunctionalTests.Windows.Tests { [TestFixture] [Category(Categories.FullSuiteOnly)] - [Category(Categories.Windows)] + [Category(Categories.WindowsOnly)] public class DiskLayoutUpgradeTests : TestsWithEnlistmentPerTestCase { public const int CurrentDiskLayoutMajorVersion = 16; diff --git a/GVFS/GVFS.FunctionalTests.Windows/Windows/Tests/JunctionAndSubstTests.cs b/GVFS/GVFS.FunctionalTests.Windows/Windows/Tests/JunctionAndSubstTests.cs index ad588e870..a5dcdc143 100644 --- a/GVFS/GVFS.FunctionalTests.Windows/Windows/Tests/JunctionAndSubstTests.cs +++ b/GVFS/GVFS.FunctionalTests.Windows/Windows/Tests/JunctionAndSubstTests.cs @@ -13,7 +13,7 @@ namespace GVFS.FunctionalTests.Windows.Tests { [TestFixture] [Category(Categories.FullSuiteOnly)] - [Category(Categories.Windows)] + [Category(Categories.WindowsOnly)] public class JunctionAndSubstTests : TestsWithEnlistmentPerFixture { private const string SubstDrive = "Q:"; diff --git a/GVFS/GVFS.FunctionalTests.Windows/Windows/Tests/ServiceTests.cs b/GVFS/GVFS.FunctionalTests.Windows/Windows/Tests/ServiceTests.cs index 8a57c27c2..3bb713a79 100644 --- a/GVFS/GVFS.FunctionalTests.Windows/Windows/Tests/ServiceTests.cs +++ b/GVFS/GVFS.FunctionalTests.Windows/Windows/Tests/ServiceTests.cs @@ -14,7 +14,7 @@ namespace GVFS.FunctionalTests.Windows.Tests [TestFixture] [NonParallelizable] [Category(Categories.FullSuiteOnly)] - [Category(Categories.Windows)] + [Category(Categories.WindowsOnly)] public class ServiceTests : TestsWithEnlistmentPerFixture { private const string NativeLibPath = @"C:\Program Files\GVFS\ProjectedFSLib.dll"; diff --git a/GVFS/GVFS.FunctionalTests.Windows/Windows/Tests/SharedCacheUpgradeTests.cs b/GVFS/GVFS.FunctionalTests.Windows/Windows/Tests/SharedCacheUpgradeTests.cs index 0ae56247b..f113b0012 100644 --- a/GVFS/GVFS.FunctionalTests.Windows/Windows/Tests/SharedCacheUpgradeTests.cs +++ b/GVFS/GVFS.FunctionalTests.Windows/Windows/Tests/SharedCacheUpgradeTests.cs @@ -14,7 +14,7 @@ namespace GVFS.FunctionalTests.Windows.Windows.Tests { [TestFixture] [Category(Categories.FullSuiteOnly)] - [Category(Categories.Windows)] + [Category(Categories.WindowsOnly)] public class SharedCacheUpgradeTests : TestsWithMultiEnlistment { private string localCachePath; diff --git a/GVFS/GVFS.FunctionalTests.Windows/Windows/Tests/WindowsFileSystemTests.cs b/GVFS/GVFS.FunctionalTests.Windows/Windows/Tests/WindowsFileSystemTests.cs index 1e834a700..1cd0c45a6 100644 --- a/GVFS/GVFS.FunctionalTests.Windows/Windows/Tests/WindowsFileSystemTests.cs +++ b/GVFS/GVFS.FunctionalTests.Windows/Windows/Tests/WindowsFileSystemTests.cs @@ -16,7 +16,7 @@ namespace GVFS.FunctionalTests.Windows.Windows.Tests { [TestFixture] - [Category(Categories.Windows)] + [Category(Categories.WindowsOnly)] public class WindowsFileSystemTests : TestsWithEnlistmentPerFixture { private enum CreationDisposition diff --git a/GVFS/GVFS.FunctionalTests/Categories.cs b/GVFS/GVFS.FunctionalTests/Categories.cs index 14fa221da..328145b02 100644 --- a/GVFS/GVFS.FunctionalTests/Categories.cs +++ b/GVFS/GVFS.FunctionalTests/Categories.cs @@ -6,13 +6,13 @@ public static class Categories public const string FastFetch = "FastFetch"; public const string GitCommands = "GitCommands"; - public const string Windows = "Windows"; + public const string WindowsOnly = "WindowsOnly"; + public const string MacOnly = "MacOnly"; - public static class Mac + public static class MacTODO { - public const string M1 = "M1_CloneAndMount"; + public const string NeedsLockHolder = "NeedsDotCoreLockHolder"; public const string M2 = "M2_StaticViewGitCommands"; - public const string M2TODO = "M2_StaticViewGitCommandsStillTODO"; public const string M3 = "M3_AllGitCommands"; public const string M4 = "M4_All"; } diff --git a/GVFS/GVFS.FunctionalTests/Program.cs b/GVFS/GVFS.FunctionalTests/Program.cs index 62db991d8..add16f3b3 100644 --- a/GVFS/GVFS.FunctionalTests/Program.cs +++ b/GVFS/GVFS.FunctionalTests/Program.cs @@ -60,17 +60,20 @@ public static void Main(string[] args) if (runner.HasCustomArg("--windows-only")) { - includeCategories.Add(Categories.Windows); + includeCategories.Add(Categories.WindowsOnly); } - if (runner.HasCustomArg("--mac-only")) + if (RuntimeInformation.IsOSPlatform(OSPlatform.OSX)) { - includeCategories.Add(Categories.Mac.M1); - includeCategories.Add(Categories.Mac.M2); - excludeCategories.Add(Categories.Mac.M2TODO); - excludeCategories.Add(Categories.Mac.M3); - excludeCategories.Add(Categories.Mac.M4); - excludeCategories.Add(Categories.Windows); + excludeCategories.Add(Categories.MacTODO.NeedsLockHolder); + excludeCategories.Add(Categories.MacTODO.M2); + excludeCategories.Add(Categories.MacTODO.M3); + excludeCategories.Add(Categories.MacTODO.M4); + excludeCategories.Add(Categories.WindowsOnly); + } + else + { + excludeCategories.Add(Categories.MacOnly); } GVFSTestConfig.RepoToClone = diff --git a/GVFS/GVFS.FunctionalTests/Tests/EnlistmentPerFixture/BasicFileSystemTests.cs b/GVFS/GVFS.FunctionalTests/Tests/EnlistmentPerFixture/BasicFileSystemTests.cs index ac87acb32..214f12697 100644 --- a/GVFS/GVFS.FunctionalTests/Tests/EnlistmentPerFixture/BasicFileSystemTests.cs +++ b/GVFS/GVFS.FunctionalTests/Tests/EnlistmentPerFixture/BasicFileSystemTests.cs @@ -12,7 +12,6 @@ namespace GVFS.FunctionalTests.Tests.LongRunningEnlistment { [TestFixture] - [Category(Categories.Mac.M1)] public class BasicFileSystemTests : TestsWithEnlistmentPerFixture { [TestCaseSource(typeof(FileRunnersAndFolders), FileRunnersAndFolders.TestRunners)] @@ -75,7 +74,7 @@ public void FilesAreBufferedAndCanBeFlushed(FileSystemRunner fileSystem, string } [TestCaseSource(typeof(FileRunnersAndFolders), FileRunnersAndFolders.TestFolders)] - [Category(Categories.Windows)] + [Category(Categories.WindowsOnly)] public void NewFileAttributesAreUpdated(string parentFolder) { string filename = Path.Combine(parentFolder, "FileAttributesAreUpdated"); @@ -103,7 +102,7 @@ public void NewFileAttributesAreUpdated(string parentFolder) } [TestCaseSource(typeof(FileRunnersAndFolders), FileRunnersAndFolders.TestFolders)] - [Category(Categories.Windows)] + [Category(Categories.WindowsOnly)] public void NewFolderAttributesAreUpdated(string parentFolder) { string folderName = Path.Combine(parentFolder, "FolderAttributesAreUpdated"); @@ -130,7 +129,7 @@ public void NewFolderAttributesAreUpdated(string parentFolder) } [TestCase] - [Category(Categories.Windows)] + [Category(Categories.WindowsOnly)] public void ExpandedFileAttributesAreUpdated() { FileSystemRunner fileSystem = FileSystemRunner.DefaultRunner; @@ -157,7 +156,7 @@ public void ExpandedFileAttributesAreUpdated() } [TestCase] - [Category(Categories.Windows)] + [Category(Categories.WindowsOnly)] public void UnhydratedFolderAttributesAreUpdated() { FileSystemRunner fileSystem = FileSystemRunner.DefaultRunner; @@ -423,8 +422,9 @@ public void DeletedFilesCanBeImmediatelyRecreated(FileSystemRunner fileSystem, s fileSystem.DeleteFile(filePath); } + // WindowsOnly due to differences between POSIX and Windows delete [TestCaseSource(typeof(FileRunnersAndFolders), FileRunnersAndFolders.TestCanDeleteFilesWhileTheyAreOpenRunners)] - [Category(Categories.Windows)] + [Category(Categories.WindowsOnly)] public void CanDeleteFilesWhileTheyAreOpen(FileSystemRunner fileSystem, string parentFolder) { string filename = Path.Combine(parentFolder, "CanDeleteFilesWhileTheyAreOpen"); @@ -455,8 +455,9 @@ public void CanDeleteFilesWhileTheyAreOpen(FileSystemRunner fileSystem, string p filePath.ShouldNotExistOnDisk(fileSystem); } + // WindowsOnly due to differences between POSIX and Windows delete [TestCase] - [Category(Categories.Windows)] + [Category(Categories.WindowsOnly)] public void CanDeleteHydratedFilesWhileTheyAreOpenForWrite() { FileSystemRunner fileSystem = FileSystemRunner.DefaultRunner; @@ -490,7 +491,7 @@ public void CanDeleteHydratedFilesWhileTheyAreOpenForWrite() } [TestCase] - [Category(Categories.Windows)] + [Category(Categories.WindowsOnly)] public void ProjectedBlobFileTimesMatchHead() { // TODO: 467539 - Update all runners to support getting create/modify/access times @@ -514,7 +515,7 @@ public void ProjectedBlobFileTimesMatchHead() } [TestCase] - [Category(Categories.Windows)] + [Category(Categories.WindowsOnly)] public void ProjectedBlobFolderTimesMatchHead() { // TODO: 467539 - Update all runners to support getting create/modify/access times @@ -784,7 +785,7 @@ public void MoveDotGitFullFolderTreeToDotGitFullFolder(FileSystemRunner fileSyst } [TestCaseSource(typeof(FileSystemRunner), FileSystemRunner.TestRunners)] - [Category(Categories.Mac.M4)] + [Category(Categories.MacTODO.M4)] public void DeleteIndexFileFails(FileSystemRunner fileSystem) { string indexFilePath = this.Enlistment.GetVirtualPathTo(Path.Combine(".git", "index")); @@ -842,7 +843,7 @@ public void MoveVirtualNTFSFolderIntoInvalidFolder(FileSystemRunner fileSystem, } [TestCaseSource(typeof(FileRunnersAndFolders), FileRunnersAndFolders.TestFolders)] - [Category(Categories.Windows)] + [Category(Categories.WindowsOnly)] public void CreateFileInheritsParentDirectoryAttributes(string parentFolder) { string parentDirectoryPath = this.Enlistment.GetVirtualPathTo(Path.Combine(parentFolder, "CreateFileInheritsParentDirectoryAttributes")); @@ -859,7 +860,7 @@ public void CreateFileInheritsParentDirectoryAttributes(string parentFolder) } [TestCaseSource(typeof(FileRunnersAndFolders), FileRunnersAndFolders.TestFolders)] - [Category(Categories.Windows)] + [Category(Categories.WindowsOnly)] public void CreateDirectoryInheritsParentDirectoryAttributes(string parentFolder) { string parentDirectoryPath = this.Enlistment.GetVirtualPathTo(Path.Combine(parentFolder, "CreateDirectoryInheritsParentDirectoryAttributes")); diff --git a/GVFS/GVFS.FunctionalTests/Tests/EnlistmentPerFixture/CacheServerTests.cs b/GVFS/GVFS.FunctionalTests/Tests/EnlistmentPerFixture/CacheServerTests.cs index f3642542d..85b10188b 100644 --- a/GVFS/GVFS.FunctionalTests/Tests/EnlistmentPerFixture/CacheServerTests.cs +++ b/GVFS/GVFS.FunctionalTests/Tests/EnlistmentPerFixture/CacheServerTests.cs @@ -6,7 +6,6 @@ namespace GVFS.FunctionalTests.Tests.EnlistmentPerFixture { [TestFixture] [Category(Categories.FullSuiteOnly)] - [Category(Categories.Mac.M2)] public class CacheServerTests : TestsWithEnlistmentPerFixture { private const string CustomUrl = "https://myCache"; diff --git a/GVFS/GVFS.FunctionalTests/Tests/EnlistmentPerFixture/CloneTests.cs b/GVFS/GVFS.FunctionalTests/Tests/EnlistmentPerFixture/CloneTests.cs index e26fd7efa..4d696d541 100644 --- a/GVFS/GVFS.FunctionalTests/Tests/EnlistmentPerFixture/CloneTests.cs +++ b/GVFS/GVFS.FunctionalTests/Tests/EnlistmentPerFixture/CloneTests.cs @@ -7,7 +7,6 @@ namespace GVFS.FunctionalTests.Tests.EnlistmentPerFixture { [TestFixture] - [Category(Categories.Mac.M1)] public class CloneTests : TestsWithEnlistmentPerFixture { private const int GVFSGenericError = 3; diff --git a/GVFS/GVFS.FunctionalTests/Tests/EnlistmentPerFixture/DehydrateTests.cs b/GVFS/GVFS.FunctionalTests/Tests/EnlistmentPerFixture/DehydrateTests.cs index f14e9d5a6..1fc03382b 100644 --- a/GVFS/GVFS.FunctionalTests/Tests/EnlistmentPerFixture/DehydrateTests.cs +++ b/GVFS/GVFS.FunctionalTests/Tests/EnlistmentPerFixture/DehydrateTests.cs @@ -11,6 +11,7 @@ namespace GVFS.FunctionalTests.Tests.EnlistmentPerFixture { [TestFixture] [Category(Categories.FullSuiteOnly)] + [Category(Categories.MacTODO.M4)] public class DehydrateTests : TestsWithEnlistmentPerFixture { private const int GVFSGenericError = 3; diff --git a/GVFS/GVFS.FunctionalTests/Tests/EnlistmentPerFixture/DiagnoseTests.cs b/GVFS/GVFS.FunctionalTests/Tests/EnlistmentPerFixture/DiagnoseTests.cs index 4e83cd3dd..2741e247c 100644 --- a/GVFS/GVFS.FunctionalTests/Tests/EnlistmentPerFixture/DiagnoseTests.cs +++ b/GVFS/GVFS.FunctionalTests/Tests/EnlistmentPerFixture/DiagnoseTests.cs @@ -9,6 +9,7 @@ namespace GVFS.FunctionalTests.Tests.EnlistmentPerFixture { [TestFixture] [Category(Categories.FullSuiteOnly)] + [Category(Categories.MacTODO.M4)] public class DiagnoseTests : TestsWithEnlistmentPerFixture { private FileSystemRunner fileSystem; diff --git a/GVFS/GVFS.FunctionalTests/Tests/EnlistmentPerFixture/GVFSLockTests.cs b/GVFS/GVFS.FunctionalTests/Tests/EnlistmentPerFixture/GVFSLockTests.cs index 5f9d48000..c7047c266 100644 --- a/GVFS/GVFS.FunctionalTests/Tests/EnlistmentPerFixture/GVFSLockTests.cs +++ b/GVFS/GVFS.FunctionalTests/Tests/EnlistmentPerFixture/GVFSLockTests.cs @@ -1,4 +1,5 @@ using GVFS.FunctionalTests.FileSystemRunners; +using GVFS.FunctionalTests.Properties; using GVFS.FunctionalTests.Tools; using GVFS.Tests.Should; using NUnit.Framework; @@ -31,19 +32,20 @@ private enum MoveFileFlags : uint } [TestCase] + [Category(Categories.MacTODO.M2)] public void GitCheckoutFailsOutsideLock() { const string BackupPrefix = "BACKUP_"; - const string PreCommand = "pre-command.exe"; - const string PostCommand = "post-command.exe"; + string preCommand = "pre-command" + Settings.Default.BinaryFileNameExtension; + string postCommand = "post-command" + Settings.Default.BinaryFileNameExtension; string hooksBase = Path.Combine(this.Enlistment.RepoRoot, ".git", "hooks"); try { // Get hooks out of the way to simulate lock not being acquired as expected - this.fileSystem.MoveFile(Path.Combine(hooksBase, PreCommand), Path.Combine(hooksBase, BackupPrefix + PreCommand)); - this.fileSystem.MoveFile(Path.Combine(hooksBase, PostCommand), Path.Combine(hooksBase, BackupPrefix + PostCommand)); + this.fileSystem.MoveFile(Path.Combine(hooksBase, preCommand), Path.Combine(hooksBase, BackupPrefix + preCommand)); + this.fileSystem.MoveFile(Path.Combine(hooksBase, postCommand), Path.Combine(hooksBase, BackupPrefix + postCommand)); ProcessResult result = GitHelpers.InvokeGitAgainstGVFSRepo(this.Enlistment.RepoRoot, "checkout FunctionalTests/20170510_minor"); result.Errors.ShouldContain("fatal: unable to write new index file"); @@ -57,27 +59,30 @@ public void GitCheckoutFailsOutsideLock() finally { // Reset hooks for cleanup. - this.fileSystem.MoveFile(Path.Combine(hooksBase, BackupPrefix + PreCommand), Path.Combine(hooksBase, PreCommand)); - this.fileSystem.MoveFile(Path.Combine(hooksBase, BackupPrefix + PostCommand), Path.Combine(hooksBase, PostCommand)); + this.fileSystem.MoveFile(Path.Combine(hooksBase, BackupPrefix + preCommand), Path.Combine(hooksBase, preCommand)); + this.fileSystem.MoveFile(Path.Combine(hooksBase, BackupPrefix + postCommand), Path.Combine(hooksBase, postCommand)); } } [TestCase] + [Category(Categories.MacTODO.M4)] public void LockPreventsRenameFromOutsideRootOnTopOfIndex() { this.OverwritingIndexShouldFail(Path.Combine(this.Enlistment.EnlistmentRoot, "LockPreventsRenameFromOutsideRootOnTopOfIndex.txt")); } [TestCase] + [Category(Categories.MacTODO.M4)] public void LockPreventsRenameFromInsideWorkingTreeOnTopOfIndex() { this.OverwritingIndexShouldFail(this.Enlistment.GetVirtualPathTo("LockPreventsRenameFromInsideWorkingTreeOnTopOfIndex.txt")); } [TestCase] + [Category(Categories.MacTODO.M4)] public void LockPreventsRenameOfIndexLockOnTopOfIndex() { - this.OverwritingIndexShouldFail(this.Enlistment.GetVirtualPathTo(".git\\index.lock")); + this.OverwritingIndexShouldFail(this.Enlistment.GetVirtualPathTo(".git", "index.lock")); } [DllImport("kernel32.dll", SetLastError = true, CharSet = CharSet.Unicode)] @@ -88,7 +93,7 @@ private static extern bool MoveFileEx( private void OverwritingIndexShouldFail(string testFilePath) { - string indexPath = this.Enlistment.GetVirtualPathTo(".git\\index"); + string indexPath = this.Enlistment.GetVirtualPathTo(".git", "index"); this.Enlistment.WaitForBackgroundOperations().ShouldEqual(true, "Background operations failed to complete."); byte[] indexContents = File.ReadAllBytes(indexPath); diff --git a/GVFS/GVFS.FunctionalTests/Tests/EnlistmentPerFixture/GitFilesTests.cs b/GVFS/GVFS.FunctionalTests/Tests/EnlistmentPerFixture/GitFilesTests.cs index 5ec2ee3b3..d1b56fc48 100644 --- a/GVFS/GVFS.FunctionalTests/Tests/EnlistmentPerFixture/GitFilesTests.cs +++ b/GVFS/GVFS.FunctionalTests/Tests/EnlistmentPerFixture/GitFilesTests.cs @@ -12,7 +12,6 @@ namespace GVFS.FunctionalTests.Tests.EnlistmentPerFixture { [TestFixtureSource(typeof(FileSystemRunner), FileSystemRunner.TestRunners)] - [Category(Categories.Mac.M2)] public class GitFilesTests : TestsWithEnlistmentPerFixture { private FileSystemRunner fileSystem; @@ -66,12 +65,12 @@ public void CreateHardLinkTest() } [TestCase, Order(3)] - [Category(Categories.Mac.M2TODO)] + [Category(Categories.MacTODO.M2)] public void CreateFileInFolderTest() { string folderName = "folder2"; string fileName = "file2.txt"; - string filePath = folderName + "\\" + fileName; + string filePath = Path.Combine(folderName, fileName); this.Enlistment.GetVirtualPathTo(filePath).ShouldNotExistOnDisk(this.fileSystem); GVFSHelpers.ModifiedPathsShouldNotContain(this.fileSystem, this.Enlistment.DotGVFSRoot, filePath); @@ -87,7 +86,7 @@ public void CreateFileInFolderTest() } [TestCase, Order(4)] - [Category(Categories.Mac.M2TODO)] + [Category(Categories.MacTODO.M3)] public void RenameEmptyFolderTest() { string folderName = "folder3a"; @@ -108,7 +107,7 @@ public void RenameEmptyFolderTest() } [TestCase, Order(5)] - [Category(Categories.Mac.M2TODO)] + [Category(Categories.MacTODO.M2)] public void RenameFolderTest() { string folderName = "folder4a"; @@ -141,7 +140,7 @@ public void RenameFolderTest() } [TestCase, Order(6)] - [Category(Categories.Mac.M2TODO)] + [Category(Categories.MacTODO.M2)] public void CaseOnlyRenameOfNewFolderKeepsExcludeEntries() { string[] expectedModifiedPathsEntries = @@ -189,9 +188,8 @@ public void ReadingFileDoesNotUpdateIndexOrSparseCheckout() GVFSHelpers.ModifiedPathsShouldNotContain(this.fileSystem, this.Enlistment.DotGVFSRoot, gitFileToCheck); } - // TODO(Mac): Enable this test once the LockHolder is converted to .NET Core [TestCase, Order(8)] - [Category(Categories.Mac.M2TODO)] + [Category(Categories.MacTODO.NeedsLockHolder)] public void ModifiedFileWillGetAddedToModifiedPathsFile() { string gitFileToTest = "GVFS/GVFS.Common/RetryWrapper.cs"; @@ -337,7 +335,7 @@ public void OverwrittenFileAddedToSparseCheckoutAndSkipWorktreeBitCleared() } [TestCase, Order(15)] - [Category(Categories.Mac.M2TODO)] + [Category(Categories.MacTODO.M2)] public void SupersededFileAddedToSparseCheckoutAndSkipWorktreeBitCleared() { string fileToSupersedeEntry = "GVFlt_FileOperationTest/WriteAndVerify.txt"; diff --git a/GVFS/GVFS.FunctionalTests/Tests/EnlistmentPerFixture/GitMoveRenameTests.cs b/GVFS/GVFS.FunctionalTests/Tests/EnlistmentPerFixture/GitMoveRenameTests.cs index fa0fbe74f..b26a3e3b7 100644 --- a/GVFS/GVFS.FunctionalTests/Tests/EnlistmentPerFixture/GitMoveRenameTests.cs +++ b/GVFS/GVFS.FunctionalTests/Tests/EnlistmentPerFixture/GitMoveRenameTests.cs @@ -12,7 +12,6 @@ namespace GVFS.FunctionalTests.Tests.EnlistmentPerFixture { [TestFixtureSource(typeof(FileSystemRunner), FileSystemRunner.TestRunners)] [Category(Categories.GitCommands)] - [Category(Categories.Mac.M2)] public class GitMoveRenameTests : TestsWithEnlistmentPerFixture { private string testFileContents = "0123456789"; @@ -92,7 +91,6 @@ public void GitStatusAfterFileRename() } [TestCase, Order(5)] - [Category(Categories.Mac.M3)] public void GitStatusAndObjectAfterGitAdd() { string existingFilename = "test.cs"; @@ -128,7 +126,6 @@ public void GitStatusAndObjectAfterGitAdd() } [TestCase, Order(6)] - [Category(Categories.Mac.M3)] public void GitStatusAfterUnstage() { string existingFilename = "test.cs"; diff --git a/GVFS/GVFS.FunctionalTests/Tests/EnlistmentPerFixture/GitReadAndGitLockTests.cs b/GVFS/GVFS.FunctionalTests/Tests/EnlistmentPerFixture/GitReadAndGitLockTests.cs index 41ddad0f4..4a247ab05 100644 --- a/GVFS/GVFS.FunctionalTests/Tests/EnlistmentPerFixture/GitReadAndGitLockTests.cs +++ b/GVFS/GVFS.FunctionalTests/Tests/EnlistmentPerFixture/GitReadAndGitLockTests.cs @@ -22,7 +22,6 @@ public GitReadAndGitLockTests() } [TestCase, Order(1)] - [Category(Categories.Mac.M2)] public void GitStatus() { GitHelpers.CheckGitCommandAgainstGVFSRepo( @@ -33,14 +32,12 @@ public void GitStatus() } [TestCase, Order(2)] - [Category(Categories.Mac.M2)] public void GitLog() { GitHelpers.CheckGitCommandAgainstGVFSRepo(this.Enlistment.RepoRoot, "log -n1", "commit", "Author:", "Date:"); } [TestCase, Order(3)] - [Category(Categories.Mac.M2)] public void GitBranch() { GitHelpers.CheckGitCommandAgainstGVFSRepo( @@ -51,6 +48,7 @@ public void GitBranch() } [TestCase, Order(4)] + [Category(Categories.MacTODO.NeedsLockHolder)] public void GitCommandWaitsWhileAnotherIsRunning() { int pid; @@ -61,6 +59,7 @@ public void GitCommandWaitsWhileAnotherIsRunning() } [TestCase, Order(5)] + [Category(Categories.MacTODO.NeedsLockHolder)] public void GitAliasNamedAfterKnownCommandAcquiresLock() { string alias = nameof(this.GitAliasNamedAfterKnownCommandAcquiresLock); @@ -73,6 +72,7 @@ public void GitAliasNamedAfterKnownCommandAcquiresLock() } [TestCase, Order(6)] + [Category(Categories.MacTODO.NeedsLockHolder)] public void GitAliasInSubfolderNamedAfterKnownCommandAcquiresLock() { string alias = nameof(this.GitAliasInSubfolderNamedAfterKnownCommandAcquiresLock); @@ -89,6 +89,7 @@ public void GitAliasInSubfolderNamedAfterKnownCommandAcquiresLock() } [TestCase, Order(7)] + [Category(Categories.MacTODO.NeedsLockHolder)] public void ExternalLockHolderReportedWhenBackgroundTasksArePending() { int pid; @@ -106,6 +107,7 @@ public void ExternalLockHolderReportedWhenBackgroundTasksArePending() } [TestCase, Order(8)] + [Category(Categories.MacTODO.NeedsLockHolder)] public void OrphanedGVFSLockIsCleanedUp() { int pid; diff --git a/GVFS/GVFS.FunctionalTests/Tests/EnlistmentPerFixture/MountTests.cs b/GVFS/GVFS.FunctionalTests/Tests/EnlistmentPerFixture/MountTests.cs index 0c19a2f81..d1d53ff5c 100644 --- a/GVFS/GVFS.FunctionalTests/Tests/EnlistmentPerFixture/MountTests.cs +++ b/GVFS/GVFS.FunctionalTests/Tests/EnlistmentPerFixture/MountTests.cs @@ -14,7 +14,6 @@ namespace GVFS.FunctionalTests.Tests.EnlistmentPerFixture { [TestFixture] [Category(Categories.FullSuiteOnly)] - [Category(Categories.Mac.M1)] public class MountTests : TestsWithEnlistmentPerFixture { private const int GVFSGenericError = 3; @@ -41,7 +40,6 @@ public void MountFailsOutsideEnlistment() } [TestCase] - [Category(Categories.Mac.M2)] public void MountCopiesMissingReadObjectHook() { this.Enlistment.UnmountGVFS(); @@ -262,7 +260,7 @@ public void MountFailsUpgradingFromInvalidUpgradePath(string mountSubfolder) // Ported from ProjFS's BugRegressionTest [TestCase] - [Category(Categories.Windows)] + [Category(Categories.WindowsOnly)] public void ProjFS_CMDHangNoneActiveInstance() { this.Enlistment.UnmountGVFS(); diff --git a/GVFS/GVFS.FunctionalTests/Tests/EnlistmentPerFixture/MoveRenameFileTests.cs b/GVFS/GVFS.FunctionalTests/Tests/EnlistmentPerFixture/MoveRenameFileTests.cs index 43af93dc6..e83547cd4 100644 --- a/GVFS/GVFS.FunctionalTests/Tests/EnlistmentPerFixture/MoveRenameFileTests.cs +++ b/GVFS/GVFS.FunctionalTests/Tests/EnlistmentPerFixture/MoveRenameFileTests.cs @@ -10,7 +10,6 @@ namespace GVFS.FunctionalTests.Tests.EnlistmentPerFixture // TODO 452590 - Combine all of the MoveRenameTests into a single fixture, and have each use different // well known files [TestFixtureSource(typeof(FileSystemRunner), FileSystemRunner.TestRunners)] - [Category(Categories.Mac.M2)] public class MoveRenameFileTests : TestsWithEnlistmentPerFixture { public const string TestFileContents = diff --git a/GVFS/GVFS.FunctionalTests/Tests/EnlistmentPerFixture/MoveRenameFileTests_2.cs b/GVFS/GVFS.FunctionalTests/Tests/EnlistmentPerFixture/MoveRenameFileTests_2.cs index 746f22561..b38d460ac 100644 --- a/GVFS/GVFS.FunctionalTests/Tests/EnlistmentPerFixture/MoveRenameFileTests_2.cs +++ b/GVFS/GVFS.FunctionalTests/Tests/EnlistmentPerFixture/MoveRenameFileTests_2.cs @@ -8,7 +8,6 @@ namespace GVFS.FunctionalTests.Tests.EnlistmentPerFixture // TODO 452590 - Combine all of the MoveRenameTests into a single fixture, and have each use different // well known files [TestFixtureSource(typeof(FileSystemRunner), FileSystemRunner.TestRunners)] - [Category(Categories.Mac.M2)] public class MoveRenameFileTests_2 : TestsWithEnlistmentPerFixture { private const string TestFileFolder = "Test_EPF_MoveRenameFileTests_2"; diff --git a/GVFS/GVFS.FunctionalTests/Tests/EnlistmentPerFixture/MoveRenameFolderTests.cs b/GVFS/GVFS.FunctionalTests/Tests/EnlistmentPerFixture/MoveRenameFolderTests.cs index abd24be71..7e5c70ba5 100644 --- a/GVFS/GVFS.FunctionalTests/Tests/EnlistmentPerFixture/MoveRenameFolderTests.cs +++ b/GVFS/GVFS.FunctionalTests/Tests/EnlistmentPerFixture/MoveRenameFolderTests.cs @@ -6,7 +6,6 @@ namespace GVFS.FunctionalTests.Tests.EnlistmentPerFixture { [TestFixtureSource(typeof(FileSystemRunner), FileSystemRunner.TestRunners)] - [Category(Categories.Mac.M2TODO)] public class MoveRenameFolderTests : TestsWithEnlistmentPerFixture { private const string TestFileContents = @@ -40,12 +39,14 @@ public MoveRenameFolderTests(FileSystemRunner fileSystem) this.fileSystem = fileSystem; } + // WindowsOnly because renames of partial folders are blocked only on Windows [TestCase] + [Category(Categories.WindowsOnly)] public void RenameFolderShouldFail() { string testFileName = "RenameFolderShouldFail.cpp"; - string oldFolderName = "Test_EPF_MoveRenameFolderTests\\RenameFolderShouldFail\\source"; - string newFolderName = "Test_EPF_MoveRenameFolderTests\\RenameFolderShouldFail\\sourcerenamed"; + string oldFolderName = Path.Combine("Test_EPF_MoveRenameFolderTests", "RenameFolderShouldFail", "source"); + string newFolderName = Path.Combine("Test_EPF_MoveRenameFolderTests", "RenameFolderShouldFail", "sourcerenamed"); this.Enlistment.GetVirtualPathTo(newFolderName).ShouldNotExistOnDisk(this.fileSystem); this.fileSystem.MoveDirectory_RequestShouldNotBeSupported(this.Enlistment.GetVirtualPathTo(oldFolderName), this.Enlistment.GetVirtualPathTo(newFolderName)); @@ -57,13 +58,16 @@ public void RenameFolderShouldFail() this.Enlistment.GetVirtualPathTo(Path.Combine(oldFolderName, testFileName)).ShouldBeAFile(this.fileSystem).WithContents(TestFileContents); } + // This test requires expansion on PreDelete to be implemented + // MacOnly because renames of partial folders are blocked on Windows [TestCase] - [Ignore("Disabled until moving partial folders is supported")] + [Category(Categories.MacTODO.M2)] + [Category(Categories.MacOnly)] public void ChangeUnhydratedFolderName() { string testFileName = "ChangeUnhydratedFolderName.cpp"; - string oldFolderName = "Test_EPF_MoveRenameFolderTests\\ChangeUnhydratedFolderName\\source"; - string newFolderName = "Test_EPF_MoveRenameFolderTests\\ChangeUnhydratedFolderName\\source_renamed"; + string oldFolderName = Path.Combine("Test_EPF_MoveRenameFolderTests", "ChangeUnhydratedFolderName", "source"); + string newFolderName = Path.Combine("Test_EPF_MoveRenameFolderTests", "ChangeUnhydratedFolderName", "source_renamed"); this.Enlistment.GetVirtualPathTo(newFolderName).ShouldNotExistOnDisk(this.fileSystem); this.fileSystem.MoveDirectory(this.Enlistment.GetVirtualPathTo(oldFolderName), this.Enlistment.GetVirtualPathTo(newFolderName)); @@ -75,12 +79,15 @@ public void ChangeUnhydratedFolderName() this.Enlistment.GetVirtualPathTo(Path.Combine(newFolderName, testFileName)).ShouldBeAFile(this.fileSystem).WithContents(TestFileContents); } + // This test requires expansion on PreDelete to be implemented + // MacOnly because renames of partial folders are blocked on Windows [TestCase] - [Ignore("Disabled until moving partial folders is supported")] + [Category(Categories.MacTODO.M2)] + [Category(Categories.MacOnly)] public void MoveUnhydratedFolderToNewFolder() { string testFileName = "MoveUnhydratedFolderToVirtualNTFSFolder.cpp"; - string oldFolderName = "Test_EPF_MoveRenameFolderTests\\MoveUnhydratedFolderToVirtualNTFSFolder"; + string oldFolderName = Path.Combine("Test_EPF_MoveRenameFolderTests", "MoveUnhydratedFolderToVirtualNTFSFolder"); string newFolderName = "NewPerFixtureParent"; this.Enlistment.GetVirtualPathTo(newFolderName).ShouldNotExistOnDisk(this.fileSystem); @@ -97,14 +104,17 @@ public void MoveUnhydratedFolderToNewFolder() this.Enlistment.GetVirtualPathTo(Path.Combine(movedFolderPath, testFileName)).ShouldBeAFile(this.fileSystem).WithContents(TestFileContents); } + // This test requires expansion on PreDelete to be implemented + // MacOnly because renames of partial folders are blocked on Windows [TestCase] - [Ignore("Disabled until moving partial folders is supported")] + [Category(Categories.MacTODO.M2)] + [Category(Categories.MacOnly)] public void MoveUnhydratedFolderToFullFolderInDotGitFolder() { string testFileName = "MoveUnhydratedFolderToFullFolderInDotGitFolder.cpp"; - string oldFolderName = "Test_EPF_MoveRenameFolderTests\\MoveUnhydratedFolderToFullFolderInDotGitFolder"; + string oldFolderName = Path.Combine("Test_EPF_MoveRenameFolderTests", "MoveUnhydratedFolderToFullFolderInDotGitFolder"); - string newFolderName = ".git\\NewPerFixtureParent"; + string newFolderName = Path.Combine(".git", "NewPerFixtureParent"); this.Enlistment.GetVirtualPathTo(newFolderName).ShouldNotExistOnDisk(this.fileSystem); this.fileSystem.CreateDirectory(this.Enlistment.GetVirtualPathTo(newFolderName)); this.Enlistment.GetVirtualPathTo(newFolderName).ShouldBeADirectory(this.fileSystem); @@ -119,7 +129,6 @@ public void MoveUnhydratedFolderToFullFolderInDotGitFolder() } [TestCase] - [Category(Categories.Mac.M2)] public void MoveFullFolderToFullFolderInDotGitFolder() { string fileContents = "Test contents for MoveFullFolderToFullFolderInDotGitFolder"; @@ -147,12 +156,15 @@ public void MoveFullFolderToFullFolderInDotGitFolder() Path.Combine(movedFolderPath, testFileName).ShouldBeAFile(this.fileSystem).WithContents(fileContents); } + // This test requires expansion on PreDelete to be implemented + // MacOnly because renames of partial folders are blocked on Windows [TestCase] - [Ignore("Disabled until moving partial folders is supported")] + [Category(Categories.MacTODO.M2)] + [Category(Categories.MacOnly)] public void MoveAndRenameUnhydratedFolderToNewFolder() { string testFileName = "MoveAndRenameUnhydratedFolderToNewFolder.cpp"; - string oldFolderName = "Test_EPF_MoveRenameFolderTests\\MoveAndRenameUnhydratedFolderToNewFolder"; + string oldFolderName = Path.Combine("Test_EPF_MoveRenameFolderTests", "MoveAndRenameUnhydratedFolderToNewFolder"); string newFolderName = "NewPerTestCaseParent"; this.Enlistment.GetVirtualPathTo(newFolderName).ShouldNotExistOnDisk(this.fileSystem); @@ -169,12 +181,15 @@ public void MoveAndRenameUnhydratedFolderToNewFolder() this.Enlistment.GetVirtualPathTo(Path.Combine(movedFolderPath, testFileName)).ShouldBeAFile(this.fileSystem).WithContents(TestFileContents); } + // This test requires expansion on PreDelete to be implemented + // MacOnly because renames of partial folders are blocked on Windows [TestCase] - [Ignore("Disabled until moving partial folders is supported")] + [Category(Categories.MacTODO.M2)] + [Category(Categories.MacOnly)] public void MoveFolderWithUnhydratedAndFullContents() { string testFileName = "MoveFolderWithUnhydratedAndFullContents.cs"; - string oldFolderName = "Test_EPF_MoveRenameFolderTests\\MoveFolderWithUnhydratedAndFullContents"; + string oldFolderName = Path.Combine("Test_EPF_MoveRenameFolderTests", "MoveFolderWithUnhydratedAndFullContents"); string newFile = "TestFile.txt"; string newFileContents = "Contents of TestFile.txt"; diff --git a/GVFS/GVFS.FunctionalTests/Tests/EnlistmentPerFixture/MultithreadedReadWriteTests.cs b/GVFS/GVFS.FunctionalTests/Tests/EnlistmentPerFixture/MultithreadedReadWriteTests.cs index 9c3a21e0c..21924e706 100644 --- a/GVFS/GVFS.FunctionalTests/Tests/EnlistmentPerFixture/MultithreadedReadWriteTests.cs +++ b/GVFS/GVFS.FunctionalTests/Tests/EnlistmentPerFixture/MultithreadedReadWriteTests.cs @@ -11,11 +11,10 @@ namespace GVFS.FunctionalTests.Tests.EnlistmentPerFixture { // TODO 469238: Elaborate on these tests? [TestFixture] - [Category(Categories.Mac.M1)] public class MultithreadedReadWriteTests : TestsWithEnlistmentPerFixture { [TestCase, Order(1)] - [Category(Categories.Windows)] + [Category(Categories.WindowsOnly)] public void CanReadVirtualFileInParallel() { // Note: This test MUST go first, or else it needs to ensure that it is reading a unique path compared to the diff --git a/GVFS/GVFS.FunctionalTests/Tests/EnlistmentPerFixture/PrefetchVerbTests.cs b/GVFS/GVFS.FunctionalTests/Tests/EnlistmentPerFixture/PrefetchVerbTests.cs index 28117a919..3500e0977 100644 --- a/GVFS/GVFS.FunctionalTests/Tests/EnlistmentPerFixture/PrefetchVerbTests.cs +++ b/GVFS/GVFS.FunctionalTests/Tests/EnlistmentPerFixture/PrefetchVerbTests.cs @@ -29,8 +29,8 @@ public void PrefetchAllMustBeExplicit() [TestCase, Order(2)] public void PrefetchSpecificFiles() { - this.ExpectBlobCount(this.Enlistment.Prefetch(@"--files GVFS\GVFS\Program.cs"), 1); - this.ExpectBlobCount(this.Enlistment.Prefetch(@"--files GVFS\GVFS\Program.cs;GVFS\GVFS.FunctionalTests\GVFS.FunctionalTests.csproj"), 2); + this.ExpectBlobCount(this.Enlistment.Prefetch($"--files {Path.Combine("GVFS", "GVFS", "Program.cs")}"), 1); + this.ExpectBlobCount(this.Enlistment.Prefetch($"--files {Path.Combine("GVFS", "GVFS", "Program.cs")};{Path.Combine("GVFS", "GVFS.FunctionalTests", "GVFS.FunctionalTests.csproj")}"), 2); } [TestCase, Order(3)] @@ -41,6 +41,7 @@ public void PrefetchByFileExtension() } [TestCase, Order(4)] + [Category(Categories.MacTODO.M4)] public void PrefetchByFileExtensionWithHydrate() { int expectedCount = 3; @@ -50,10 +51,12 @@ public void PrefetchByFileExtensionWithHydrate() } [TestCase, Order(5)] + [Category(Categories.MacTODO.M4)] public void PrefetchByFilesWithHydrateWhoseObjectsAreAlreadyDownloaded() { int expectedCount = 2; - string output = this.Enlistment.Prefetch(@"--files GVFS\GVFS\Program.cs;GVFS\GVFS.FunctionalTests\GVFS.FunctionalTests.csproj --hydrate"); + string output = this.Enlistment.Prefetch( + $"--files {Path.Combine("GVFS", "GVFS", "Program.cs")};{Path.Combine("GVFS", "GVFS.FunctionalTests", "GVFS.FunctionalTests.csproj")} --hydrate"); this.ExpectBlobCount(output, expectedCount); output.ShouldContain("Hydrated files: " + expectedCount); output.ShouldContain("Downloaded: 0"); @@ -62,8 +65,8 @@ public void PrefetchByFilesWithHydrateWhoseObjectsAreAlreadyDownloaded() [TestCase, Order(6)] public void PrefetchFolders() { - this.ExpectBlobCount(this.Enlistment.Prefetch(@"--folders GVFS\GVFS"), 17); - this.ExpectBlobCount(this.Enlistment.Prefetch(@"--folders GVFS\GVFS;GVFS\GVFS.FunctionalTests"), 65); + this.ExpectBlobCount(this.Enlistment.Prefetch($"--folders {Path.Combine("GVFS", "GVFS")}"), 17); + this.ExpectBlobCount(this.Enlistment.Prefetch($"--folders {Path.Combine("GVFS", "GVFS")};{Path.Combine("GVFS", "GVFS.FunctionalTests")}"), 65); } [TestCase, Order(7)] @@ -97,10 +100,12 @@ public void PrefetchAll() { this.ExpectBlobCount(this.Enlistment.Prefetch("--files *"), 494); this.ExpectBlobCount(this.Enlistment.Prefetch("--folders /"), 494); - this.ExpectBlobCount(this.Enlistment.Prefetch("--folders \\"), 494); + this.ExpectBlobCount(this.Enlistment.Prefetch($"--folders {Path.DirectorySeparatorChar}"), 494); } + // TODO(Mac): Handle that lock files are not deleted on Mac, they are simply unlocked [TestCase, Order(10)] + [Category(Categories.MacTODO.M4)] public void PrefetchCleansUpStalePrefetchLock() { this.Enlistment.Prefetch("--commits"); diff --git a/GVFS/GVFS.FunctionalTests/Tests/EnlistmentPerFixture/PrefetchVerbWithoutSharedCacheTests.cs b/GVFS/GVFS.FunctionalTests/Tests/EnlistmentPerFixture/PrefetchVerbWithoutSharedCacheTests.cs index d0016c61d..a64bae950 100644 --- a/GVFS/GVFS.FunctionalTests/Tests/EnlistmentPerFixture/PrefetchVerbWithoutSharedCacheTests.cs +++ b/GVFS/GVFS.FunctionalTests/Tests/EnlistmentPerFixture/PrefetchVerbWithoutSharedCacheTests.cs @@ -11,6 +11,7 @@ namespace GVFS.FunctionalTests.Tests.EnlistmentPerFixture { [TestFixture] [Category(Categories.FullSuiteOnly)] + [Category(Categories.MacTODO.M4)] public class PrefetchVerbWithoutSharedCacheTests : TestsWithEnlistmentPerFixture { private const string PrefetchPackPrefix = "prefetch"; diff --git a/GVFS/GVFS.FunctionalTests/Tests/EnlistmentPerFixture/UnmountTests.cs b/GVFS/GVFS.FunctionalTests/Tests/EnlistmentPerFixture/UnmountTests.cs index 1e8f56a23..d0b87f9a9 100644 --- a/GVFS/GVFS.FunctionalTests/Tests/EnlistmentPerFixture/UnmountTests.cs +++ b/GVFS/GVFS.FunctionalTests/Tests/EnlistmentPerFixture/UnmountTests.cs @@ -34,6 +34,7 @@ public void SetupTest() } [TestCase] + [Category(Categories.MacTODO.NeedsLockHolder)] public void UnmountWaitsForLock() { ManualResetEventSlim lockHolder = GitHelpers.AcquireGVFSLock(this.Enlistment, out _); @@ -50,6 +51,7 @@ public void UnmountWaitsForLock() } [TestCase] + [Category(Categories.MacTODO.NeedsLockHolder)] public void UnmountSkipLock() { ManualResetEventSlim lockHolder = GitHelpers.AcquireGVFSLock(this.Enlistment, out _, Timeout.Infinite, true); diff --git a/GVFS/GVFS.FunctionalTests/Tests/EnlistmentPerFixture/UpdatePlaceholderTests.cs b/GVFS/GVFS.FunctionalTests/Tests/EnlistmentPerFixture/UpdatePlaceholderTests.cs index 22146b1ae..c36877f58 100644 --- a/GVFS/GVFS.FunctionalTests/Tests/EnlistmentPerFixture/UpdatePlaceholderTests.cs +++ b/GVFS/GVFS.FunctionalTests/Tests/EnlistmentPerFixture/UpdatePlaceholderTests.cs @@ -13,6 +13,7 @@ namespace GVFS.FunctionalTests.Tests.EnlistmentPerFixture { [TestFixture] [Category(Categories.GitCommands)] + [Category(Categories.MacTODO.M3)] public class UpdatePlaceholderTests : TestsWithEnlistmentPerFixture { private const string TestParentFolderName = "Test_EPF_UpdatePlaceholderTests"; diff --git a/GVFS/GVFS.FunctionalTests/Tests/EnlistmentPerFixture/WorkingDirectoryTests.cs b/GVFS/GVFS.FunctionalTests/Tests/EnlistmentPerFixture/WorkingDirectoryTests.cs index 01ac6c8dc..2e32f9dd2 100644 --- a/GVFS/GVFS.FunctionalTests/Tests/EnlistmentPerFixture/WorkingDirectoryTests.cs +++ b/GVFS/GVFS.FunctionalTests/Tests/EnlistmentPerFixture/WorkingDirectoryTests.cs @@ -14,7 +14,6 @@ namespace GVFS.FunctionalTests.Tests.EnlistmentPerFixture { [TestFixtureSource(typeof(FileSystemRunner), FileSystemRunner.TestRunners)] - [Category(Categories.Mac.M2)] public class WorkingDirectoryTests : TestsWithEnlistmentPerFixture { private const int CurrentPlaceholderVersion = 1; @@ -393,7 +392,7 @@ public void FolderPlaceHolderHasVersionInfo() [TestCase, Order(13)] [Category(Categories.GitCommands)] - [Category(Categories.Mac.M3)] + [Category(Categories.MacTODO.M3)] public void FolderContentsProjectedAfterFolderCreateAndCheckout() { string folderName = "GVFlt_MultiThreadTest"; @@ -419,7 +418,7 @@ public void FolderContentsProjectedAfterFolderCreateAndCheckout() [TestCase, Order(14)] [Category(Categories.GitCommands)] - [Category(Categories.Mac.M3)] + [Category(Categories.MacTODO.M3)] public void FolderContentsCorrectAfterCreateNewFolderRenameAndCheckoutCommitWithSameFolder() { // 3a55d3b760c87642424e834228a3408796501e7c is the commit prior to adding Test_EPF_MoveRenameFileTests @@ -450,7 +449,7 @@ public void FolderContentsCorrectAfterCreateNewFolderRenameAndCheckoutCommitWith // TODO(Mac) This *should* be working already, we need further investigation of why this test fails on build agents, but not on dev machines. [TestCase, Order(15)] - [Category(Categories.Mac.M2TODO)] + [Category(Categories.MacTODO.M2)] public void FilterNonUTF8FileName() { string encodingFilename = "ريلٌأكتوبرûمارسأغسطسºٰٰۂْٗ۵ريلٌأك.txt"; @@ -499,7 +498,7 @@ public void AllNullObjectRedownloaded() // TODO(Mac): Figure out why git for Mac is not requesting a redownload of the truncated object [TestCase, Order(17)] - [Category(Categories.Mac.M3)] + [Category(Categories.MacTODO.M3)] public void TruncatedObjectRedownloaded() { GitProcess.InvokeProcess(this.Enlistment.RepoRoot, "checkout " + this.Enlistment.Commitish); diff --git a/GVFS/GVFS.FunctionalTests/Tests/EnlistmentPerTestCase/CaseOnlyFolderRenameTests.cs b/GVFS/GVFS.FunctionalTests/Tests/EnlistmentPerTestCase/CaseOnlyFolderRenameTests.cs index fa0e29bba..d5fb1e49a 100644 --- a/GVFS/GVFS.FunctionalTests/Tests/EnlistmentPerTestCase/CaseOnlyFolderRenameTests.cs +++ b/GVFS/GVFS.FunctionalTests/Tests/EnlistmentPerTestCase/CaseOnlyFolderRenameTests.cs @@ -10,9 +10,18 @@ namespace GVFS.FunctionalTests.Tests.EnlistmentPerTestCase [TestFixture] public class CaseOnlyFolderRenameTests : TestsWithEnlistmentPerTestCase { - [TestCaseSource(typeof(FileSystemRunner), FileSystemRunner.TestRunners)] - [Ignore("Disabled until moving partial folders is supported")] - public void CaseRenameFoldersAndRemountAndReanmeAgain(FileSystemRunner fileSystem) + private FileSystemRunner fileSystem; + + public CaseOnlyFolderRenameTests() + : base() + { + this.fileSystem = new BashRunner(); + } + + // MacOnly because renames of partial folders are blocked on Windows + [TestCase] + [Category(Categories.MacOnly)] + public void CaseRenameFoldersAndRemountAndRenameAgain() { // Projected folder without a physical folder string parentFolderName = "GVFS"; @@ -23,8 +32,7 @@ public void CaseRenameFoldersAndRemountAndReanmeAgain(FileSystemRunner fileSyste this.Enlistment.GetVirtualPathTo(oldGVFSSubFolderPath).ShouldBeADirectory(fileSystem).WithCaseMatchingName(oldGVFSSubFolderName); - // Use NativeMethods rather than the runner as it supports case-only rename - NativeMethods.MoveFile(this.Enlistment.GetVirtualPathTo(oldGVFSSubFolderPath), this.Enlistment.GetVirtualPathTo(newGVFSSubFolderPath)); + this.fileSystem.MoveFile(this.Enlistment.GetVirtualPathTo(oldGVFSSubFolderPath), this.Enlistment.GetVirtualPathTo(newGVFSSubFolderPath)); this.Enlistment.GetVirtualPathTo(newGVFSSubFolderPath).ShouldBeADirectory(fileSystem).WithCaseMatchingName(newGVFSSubFolderName); @@ -41,8 +49,7 @@ public void CaseRenameFoldersAndRemountAndReanmeAgain(FileSystemRunner fileSyste this.Enlistment.GetVirtualPathTo(oldTestsSubFolderPath).ShouldBeADirectory(fileSystem).WithCaseMatchingName(oldTestsSubFolderName); - // Use NativeMethods rather than the runner as it supports case-only rename - NativeMethods.MoveFile(this.Enlistment.GetVirtualPathTo(oldTestsSubFolderPath), this.Enlistment.GetVirtualPathTo(newTestsSubFolderPath)); + this.fileSystem.MoveFile(this.Enlistment.GetVirtualPathTo(oldTestsSubFolderPath), this.Enlistment.GetVirtualPathTo(newTestsSubFolderPath)); this.Enlistment.GetVirtualPathTo(newTestsSubFolderPath).ShouldBeADirectory(fileSystem).WithCaseMatchingName(newTestsSubFolderName); @@ -57,12 +64,12 @@ public void CaseRenameFoldersAndRemountAndReanmeAgain(FileSystemRunner fileSyste // Rename each folder again string finalGVFSSubFolderName = "gvFS"; string finalGVFSSubFolderPath = Path.Combine(parentFolderName, finalGVFSSubFolderName); - NativeMethods.MoveFile(this.Enlistment.GetVirtualPathTo(newGVFSSubFolderPath), this.Enlistment.GetVirtualPathTo(finalGVFSSubFolderPath)); + this.fileSystem.MoveFile(this.Enlistment.GetVirtualPathTo(newGVFSSubFolderPath), this.Enlistment.GetVirtualPathTo(finalGVFSSubFolderPath)); this.Enlistment.GetVirtualPathTo(finalGVFSSubFolderPath).ShouldBeADirectory(fileSystem).WithCaseMatchingName(finalGVFSSubFolderName); string finalTestsSubFolderName = "gvfs.FunctionalTESTS"; string finalTestsSubFolderPath = Path.Combine(parentFolderName, finalTestsSubFolderName); - NativeMethods.MoveFile(this.Enlistment.GetVirtualPathTo(newTestsSubFolderPath), this.Enlistment.GetVirtualPathTo(finalTestsSubFolderPath)); + this.fileSystem.MoveFile(this.Enlistment.GetVirtualPathTo(newTestsSubFolderPath), this.Enlistment.GetVirtualPathTo(finalTestsSubFolderPath)); this.Enlistment.GetVirtualPathTo(finalTestsSubFolderPath).ShouldBeADirectory(fileSystem).WithCaseMatchingName(finalTestsSubFolderName); this.Enlistment.GetVirtualPathTo(Path.Combine(finalTestsSubFolderPath, fileToAdd)).ShouldBeAFile(fileSystem).WithContents().ShouldEqual(fileToAddContent); } diff --git a/GVFS/GVFS.FunctionalTests/Tests/EnlistmentPerTestCase/PersistedModifiedPathsTests.cs b/GVFS/GVFS.FunctionalTests/Tests/EnlistmentPerTestCase/PersistedModifiedPathsTests.cs index dfc1055c7..ac5ade5e3 100644 --- a/GVFS/GVFS.FunctionalTests/Tests/EnlistmentPerTestCase/PersistedModifiedPathsTests.cs +++ b/GVFS/GVFS.FunctionalTests/Tests/EnlistmentPerTestCase/PersistedModifiedPathsTests.cs @@ -10,7 +10,7 @@ namespace GVFS.FunctionalTests.Tests.EnlistmentPerTestCase { [TestFixture] - [Category(Categories.Mac.M2TODO)] + [Category(Categories.MacTODO.M2)] public class PersistedModifiedPathsTests : TestsWithEnlistmentPerTestCase { private static readonly string FileToAdd = Path.Combine("GVFS", "TestAddFile.txt"); diff --git a/GVFS/GVFS.FunctionalTests/Tests/EnlistmentPerTestCase/PersistedWorkingDirectoryTests.cs b/GVFS/GVFS.FunctionalTests/Tests/EnlistmentPerTestCase/PersistedWorkingDirectoryTests.cs index 1bb1ec167..02b44fced 100644 --- a/GVFS/GVFS.FunctionalTests/Tests/EnlistmentPerTestCase/PersistedWorkingDirectoryTests.cs +++ b/GVFS/GVFS.FunctionalTests/Tests/EnlistmentPerTestCase/PersistedWorkingDirectoryTests.cs @@ -9,7 +9,6 @@ namespace GVFS.FunctionalTests.Tests.EnlistmentPerTestCase { [TestFixture] [Category(Categories.FullSuiteOnly)] - [Category(Categories.Mac.M1)] public class PersistedWorkingDirectoryTests : TestsWithEnlistmentPerTestCase { [TestCaseSource(typeof(FileSystemRunner), FileSystemRunner.TestRunners)] diff --git a/GVFS/GVFS.FunctionalTests/Tests/EnlistmentPerTestCase/RepairTests.cs b/GVFS/GVFS.FunctionalTests/Tests/EnlistmentPerTestCase/RepairTests.cs index 3302029a2..34da63c00 100644 --- a/GVFS/GVFS.FunctionalTests/Tests/EnlistmentPerTestCase/RepairTests.cs +++ b/GVFS/GVFS.FunctionalTests/Tests/EnlistmentPerTestCase/RepairTests.cs @@ -10,6 +10,7 @@ namespace GVFS.FunctionalTests.Tests.EnlistmentPerTestCase { [TestFixture] [Category(Categories.FullSuiteOnly)] + [Category(Categories.MacTODO.M4)] public class RepairTests : TestsWithEnlistmentPerTestCase { [TestCase] diff --git a/GVFS/GVFS.FunctionalTests/Tests/FastFetchTests.cs b/GVFS/GVFS.FunctionalTests/Tests/FastFetchTests.cs index 54db8a59a..de9db40fd 100644 --- a/GVFS/GVFS.FunctionalTests/Tests/FastFetchTests.cs +++ b/GVFS/GVFS.FunctionalTests/Tests/FastFetchTests.cs @@ -16,6 +16,7 @@ namespace GVFS.FunctionalTests.Tests [TestFixture] [Category(Categories.FastFetch)] [Category(Categories.FullSuiteOnly)] + [Category(Categories.MacTODO.M4)] public class FastFetchTests { private readonly string fastFetchRepoRoot = Settings.Default.FastFetchRoot; diff --git a/GVFS/GVFS.FunctionalTests/Tests/GVFSVerbTests.cs b/GVFS/GVFS.FunctionalTests/Tests/GVFSVerbTests.cs index b76db7433..7d1f07337 100644 --- a/GVFS/GVFS.FunctionalTests/Tests/GVFSVerbTests.cs +++ b/GVFS/GVFS.FunctionalTests/Tests/GVFSVerbTests.cs @@ -6,7 +6,6 @@ namespace GVFS.FunctionalTests.Tests { [TestFixture] - [Category(Categories.Mac.M1)] public class GVFSVerbTests { public GVFSVerbTests() diff --git a/GVFS/GVFS.FunctionalTests/Tests/GitCommands/AddStageTests.cs b/GVFS/GVFS.FunctionalTests/Tests/GitCommands/AddStageTests.cs index ac38b5e19..e8a36a23c 100644 --- a/GVFS/GVFS.FunctionalTests/Tests/GitCommands/AddStageTests.cs +++ b/GVFS/GVFS.FunctionalTests/Tests/GitCommands/AddStageTests.cs @@ -1,12 +1,12 @@ using GVFS.FunctionalTests.Tools; using NUnit.Framework; +using System.IO; using System.Threading; namespace GVFS.FunctionalTests.Tests.GitCommands { [TestFixture] [Category(Categories.GitCommands)] - [Category(Categories.Mac.M3)] public class AddStageTests : GitRepoTests { public AddStageTests() : base(enlistmentPerTest: false) @@ -14,25 +14,22 @@ public AddStageTests() : base(enlistmentPerTest: false) } [TestCase, Order(1)] - [Category(Categories.Mac.M2)] public void AddBasicTest() { - this.EditFile("Readme.md", "Some new content."); + this.EditFile("Some new content.", "Readme.md"); this.ValidateGitCommand("add Readme.md"); this.RunGitCommand("commit -m \"Changing the Readme.md\""); } [TestCase, Order(2)] - [Category(Categories.Mac.M2)] public void StageBasicTest() { - this.EditFile("AuthoringTests.md", "Some new content."); + this.EditFile("Some new content.", "AuthoringTests.md"); this.ValidateGitCommand("stage AuthoringTests.md"); this.RunGitCommand("commit -m \"Changing the AuthoringTests.md\""); } [TestCase, Order(3)] - [Category(Categories.Mac.M2)] public void AddAndStageHardLinksTest() { if (!this.FileSystem.SupportsHardlinkCreation) @@ -52,18 +49,19 @@ public void AddAndStageHardLinksTest() [TestCase, Order(4)] public void AddAllowsPlaceholderCreation() { - this.CommandAllowsPlaceholderCreation("add", @"GVFS\GVFS\Program.cs"); + this.CommandAllowsPlaceholderCreation("add", "GVFS", "GVFS", "Program.cs"); } [TestCase, Order(5)] public void StageAllowsPlaceholderCreation() { - this.CommandAllowsPlaceholderCreation("stage", @"GVFS\GVFS\App.config"); + this.CommandAllowsPlaceholderCreation("stage", "GVFS", "GVFS", "App.config"); } - private void CommandAllowsPlaceholderCreation(string command, string fileToRead) + private void CommandAllowsPlaceholderCreation(string command, params string[] fileToReadPathParts) { - this.EditFile("Readme.md", $"Some new content for {command}."); + string fileToRead = Path.Combine(fileToReadPathParts); + this.EditFile($"Some new content for {command}.", "Readme.md"); ManualResetEventSlim resetEvent = GitHelpers.RunGitCommandWithWaitAndStdIn(this.Enlistment, resetTimeout: 3000, command: $"{command} -p", stdinToQuit: "q", processId: out _); this.FileContentsShouldMatch(fileToRead); this.ValidateGitCommand("--no-optional-locks status"); diff --git a/GVFS/GVFS.FunctionalTests/Tests/GitCommands/CheckoutTests.cs b/GVFS/GVFS.FunctionalTests/Tests/GitCommands/CheckoutTests.cs index e290e021e..ed4066332 100644 --- a/GVFS/GVFS.FunctionalTests/Tests/GitCommands/CheckoutTests.cs +++ b/GVFS/GVFS.FunctionalTests/Tests/GitCommands/CheckoutTests.cs @@ -88,38 +88,41 @@ private enum NativeFileAccess : uint } [TestCase] + [Category(Categories.MacTODO.M3)] public void CheckoutNewBranchFromStartingPointTest() { // In commit 8df701986dea0a5e78b742d2eaf9348825b14d35 the CheckoutNewBranchFromStartingPointTest files were not present this.ValidateGitCommand("checkout 8df701986dea0a5e78b742d2eaf9348825b14d35"); - this.ShouldNotExistOnDisk("GitCommandsTests\\CheckoutNewBranchFromStartingPointTest\\test1.txt"); - this.ShouldNotExistOnDisk("GitCommandsTests\\CheckoutNewBranchFromStartingPointTest\\test2.txt"); + this.ShouldNotExistOnDisk("GitCommandsTests", "CheckoutNewBranchFromStartingPointTest", "test1.txt"); + this.ShouldNotExistOnDisk("GitCommandsTests", "CheckoutNewBranchFromStartingPointTest", "test2.txt"); // In commit cd5c55fea4d58252bb38058dd3818da75aff6685 the CheckoutNewBranchFromStartingPointTest files were present this.ValidateGitCommand("checkout -b tests/functional/CheckoutNewBranchFromStartingPointTest cd5c55fea4d58252bb38058dd3818da75aff6685"); - this.FileShouldHaveContents("GitCommandsTests\\CheckoutNewBranchFromStartingPointTest\\test1.txt", "TestFile1 \r\n"); - this.FileShouldHaveContents("GitCommandsTests\\CheckoutNewBranchFromStartingPointTest\\test2.txt", "TestFile2 \r\n"); + this.FileShouldHaveContents("TestFile1 \r\n", "GitCommandsTests", "CheckoutNewBranchFromStartingPointTest", "test1.txt"); + this.FileShouldHaveContents("TestFile2 \r\n", "GitCommandsTests", "CheckoutNewBranchFromStartingPointTest", "test2.txt"); this.ValidateGitCommand("status"); } [TestCase] + [Category(Categories.MacTODO.M3)] public void CheckoutOrhpanBranchFromStartingPointTest() { // In commit 8df701986dea0a5e78b742d2eaf9348825b14d35 the CheckoutOrhpanBranchFromStartingPointTest files were not present this.ValidateGitCommand("checkout 8df701986dea0a5e78b742d2eaf9348825b14d35"); - this.ShouldNotExistOnDisk("GitCommandsTests\\CheckoutOrhpanBranchFromStartingPointTest\\test1.txt"); - this.ShouldNotExistOnDisk("GitCommandsTests\\CheckoutOrhpanBranchFromStartingPointTest\\test2.txt"); + this.ShouldNotExistOnDisk("GitCommandsTests", "CheckoutOrhpanBranchFromStartingPointTest", "test1.txt"); + this.ShouldNotExistOnDisk("GitCommandsTests", "CheckoutOrhpanBranchFromStartingPointTest", "test2.txt"); // In commit 15a9676c9192448820bd243807f6dab1bac66680 the CheckoutOrhpanBranchFromStartingPointTest files were present this.ValidateGitCommand("checkout --orphan tests/functional/CheckoutOrhpanBranchFromStartingPointTest 15a9676c9192448820bd243807f6dab1bac66680"); - this.FileShouldHaveContents("GitCommandsTests\\CheckoutOrhpanBranchFromStartingPointTest\\test1.txt", "TestFile1 \r\n"); - this.FileShouldHaveContents("GitCommandsTests\\CheckoutOrhpanBranchFromStartingPointTest\\test2.txt", "TestFile2 \r\n"); + this.FileShouldHaveContents("TestFile1 \r\n", "GitCommandsTests", "CheckoutOrhpanBranchFromStartingPointTest", "test1.txt"); + this.FileShouldHaveContents("TestFile2 \r\n", "GitCommandsTests", "CheckoutOrhpanBranchFromStartingPointTest", "test2.txt"); this.ValidateGitCommand("status"); } [TestCase] + [Category(Categories.MacTODO.M3)] public void MoveFileFromDotGitFolderToWorkingDirectoryAndAddAndCheckout() { string testFileContents = "Test file contents for MoveFileFromDotGitFolderToWorkingDirectoryAndAddAndCheckout"; @@ -135,8 +138,8 @@ public void MoveFileFromDotGitFolderToWorkingDirectoryAndAddAndCheckout() this.ValidateGitCommand("checkout -b " + newBranchName); this.ShouldNotExistOnDisk(targetPath); - this.CreateFile(dotGitFilePath, testFileContents); - this.FileShouldHaveContents(dotGitFilePath, testFileContents); + this.CreateFile(testFileContents, dotGitFilePath); + this.FileShouldHaveContents(testFileContents, dotGitFilePath); // Move file to working directory this.MoveFile(dotGitFilePath, targetPath); @@ -152,6 +155,7 @@ public void MoveFileFromDotGitFolderToWorkingDirectoryAndAddAndCheckout() } [TestCase] + [Category(Categories.MacTODO.M3)] public void CheckoutBranchNoCrashOnStatus() { this.ControlGitRepo.Fetch("FunctionalTests/20170331_git_crash"); @@ -160,6 +164,7 @@ public void CheckoutBranchNoCrashOnStatus() } [TestCase] + [Category(Categories.MacTODO.M3)] public void CheckoutCommitWhereFileContentsChangeAfterRead() { this.ControlGitRepo.Fetch(GitRepoTests.ConflictSourceBranch); @@ -168,23 +173,24 @@ public void CheckoutCommitWhereFileContentsChangeAfterRead() // In commit db95d631e379d366d26d899523f8136a77441914 the initial files for the FunctionalTests/20170206_Conflict_Source branch were created this.ValidateGitCommand("checkout db95d631e379d366d26d899523f8136a77441914"); - this.FileContentsShouldMatch(@"Test_ConflictTests\ModifiedFiles\" + fileName); + this.FileContentsShouldMatch("Test_ConflictTests", "ModifiedFiles", fileName); // A read should not add the file to the modified paths GVFSHelpers.ModifiedPathsShouldNotContain(this.FileSystem, this.Enlistment.DotGVFSRoot, fileName); this.ValidateGitCommand("checkout FunctionalTests/20170206_Conflict_Source"); - this.FileContentsShouldMatch(@"Test_ConflictTests\ModifiedFiles\" + fileName); + this.FileContentsShouldMatch("Test_ConflictTests", "ModifiedFiles", fileName); GVFSHelpers.ModifiedPathsShouldNotContain(this.FileSystem, this.Enlistment.DotGVFSRoot, fileName); } [TestCase] + [Category(Categories.MacTODO.M3)] public void CheckoutCommitWhereFileDeletedAfterRead() { this.ControlGitRepo.Fetch(GitRepoTests.ConflictSourceBranch); string fileName = "DeleteInSource.txt"; - string filePath = @"Test_ConflictTests\DeletedFiles\" + fileName; + string filePath = Path.Combine("Test_ConflictTests", "DeletedFiles", fileName); // In commit db95d631e379d366d26d899523f8136a77441914 the initial files for the FunctionalTests/20170206_Conflict_Source branch were created this.ValidateGitCommand("checkout db95d631e379d366d26d899523f8136a77441914"); @@ -199,6 +205,7 @@ public void CheckoutCommitWhereFileDeletedAfterRead() } [TestCase] + [Category(Categories.MacTODO.M3)] public void CheckoutBranchAfterReadingFileAndVerifyContentsCorrect() { this.ControlGitRepo.Fetch(GitRepoTests.ConflictSourceBranch); @@ -215,6 +222,7 @@ public void CheckoutBranchAfterReadingFileAndVerifyContentsCorrect() } [TestCase] + [Category(Categories.MacTODO.M3)] public void CheckoutBranchAfterReadingAllFilesAndVerifyContentsCorrect() { this.ControlGitRepo.Fetch(GitRepoTests.ConflictSourceBranch); @@ -233,15 +241,16 @@ public void CheckoutBranchAfterReadingAllFilesAndVerifyContentsCorrect() } [TestCase] + [Category(Categories.MacTODO.M3)] public void CheckoutBranchThatHasFolderShouldGetDeleted() { // this.ControlGitRepo.Commitish should not have the folder Test_ConflictTests\AddedFiles - string testFolder = @"Test_ConflictTests\AddedFiles"; + string testFolder = Path.Combine("Test_ConflictTests", "AddedFiles"); this.ShouldNotExistOnDisk(testFolder); this.ControlGitRepo.Fetch(GitRepoTests.ConflictSourceBranch); this.ValidateGitCommand("checkout " + GitRepoTests.ConflictSourceBranch); - string testFile = testFolder + @"\AddedByBothDifferentContent.txt"; + string testFile = Path.Combine(testFolder, "AddedByBothDifferentContent.txt"); this.FileContentsShouldMatch(testFile); // Move back to this.ControlGitRepo.Commitish where testFolder and testFile are not in the repo @@ -259,16 +268,17 @@ public void CheckoutBranchThatHasFolderShouldGetDeleted() } [TestCase] + [Category(Categories.MacTODO.M3)] public void CheckoutBranchThatDoesNotHaveFolderShouldNotHaveFolder() { // this.ControlGitRepo.Commitish should not have the folder Test_ConflictTests\AddedFiles - string testFolder = @"Test_ConflictTests\AddedFiles"; + string testFolder = Path.Combine("Test_ConflictTests", "AddedFiles"); this.ShouldNotExistOnDisk(testFolder); this.ControlGitRepo.Fetch(GitRepoTests.ConflictSourceBranch); this.ValidateGitCommand("checkout " + GitRepoTests.ConflictSourceBranch); - string testFile = testFolder + @"\AddedByBothDifferentContent.txt"; + string testFile = Path.Combine(testFolder, "AddedByBothDifferentContent.txt"); this.FileContentsShouldMatch(testFile); this.ValidateGitCommand("checkout " + this.ControlGitRepo.Commitish); this.ShouldNotExistOnDisk(testFile); @@ -284,19 +294,20 @@ public void CheckoutBranchThatDoesNotHaveFolderShouldNotHaveFolder() } [TestCase] + [Category(Categories.MacTODO.M3)] public void EditFileReadFileAndCheckoutConflict() { // editFilePath was changed on ConflictTargetBranch - string editFilePath = @"Test_ConflictTests\ModifiedFiles\ChangeInTarget.txt"; + string editFilePath = Path.Combine("Test_ConflictTests", "ModifiedFiles", "ChangeInTarget.txt"); // readFilePath has different contents on ConflictSourceBranch and ConflictTargetBranch - string readFilePath = @"Test_ConflictTests\ModifiedFiles\ChangeInSource.txt"; + string readFilePath = Path.Combine("Test_ConflictTests", "ModifiedFiles", "ChangeInSource.txt"); this.ControlGitRepo.Fetch(GitRepoTests.ConflictSourceBranch); this.ControlGitRepo.Fetch(GitRepoTests.ConflictTargetBranch); this.ValidateGitCommand("checkout " + GitRepoTests.ConflictSourceBranch); - this.EditFile(editFilePath, "New content"); + this.EditFile("New content", editFilePath); this.FileContentsShouldMatch(readFilePath); string originalReadFileContents = this.Enlistment.GetVirtualPathTo(readFilePath).ShouldBeAFile(this.FileSystem).WithContents(); @@ -318,9 +329,10 @@ public void EditFileReadFileAndCheckoutConflict() } [TestCase] + [Category(Categories.MacTODO.M3)] public void MarkFileAsReadOnlyAndCheckoutCommitWhereFileIsDifferent() { - string filePath = @"Test_ConflictTests\ModifiedFiles\ConflictingChange.txt"; + string filePath = Path.Combine("Test_ConflictTests", "ModifiedFiles", "ConflictingChange.txt"); this.ControlGitRepo.Fetch(GitRepoTests.ConflictSourceBranch); this.ControlGitRepo.Fetch(GitRepoTests.ConflictTargetBranch); @@ -333,9 +345,10 @@ public void MarkFileAsReadOnlyAndCheckoutCommitWhereFileIsDifferent() } [TestCase] + [Category(Categories.MacTODO.M3)] public void MarkFileAsReadOnlyAndCheckoutCommitWhereFileIsDeleted() { - string filePath = @"Test_ConflictTests\AddedFiles\AddedBySource.txt"; + string filePath = Path.Combine("Test_ConflictTests", "AddedFiles", "AddedBySource.txt"); this.ControlGitRepo.Fetch(GitRepoTests.ConflictSourceBranch); this.ControlGitRepo.Fetch(GitRepoTests.ConflictTargetBranch); @@ -348,6 +361,7 @@ public void MarkFileAsReadOnlyAndCheckoutCommitWhereFileIsDeleted() } [TestCase] + [Category(Categories.MacTODO.M3)] public void ModifyAndCheckoutFirstOfSeveralFilesWhoseNamesAppearBeforeDot() { // Commit cb2d05febf64e3b0df50bd8d3fe8f05c0e2caa47 has the files (a).txt and (z).txt @@ -356,15 +370,16 @@ public void ModifyAndCheckoutFirstOfSeveralFilesWhoseNamesAppearBeforeDot() string newContent = "content to append"; this.ValidateGitCommand("checkout cb2d05febf64e3b0df50bd8d3fe8f05c0e2caa47"); - this.EditFile("DeleteFileWithNameAheadOfDotAndSwitchCommits\\(a).txt", newContent); - this.FileShouldHaveContents("DeleteFileWithNameAheadOfDotAndSwitchCommits\\(a).txt", originalContent + newContent); + this.EditFile(newContent, "DeleteFileWithNameAheadOfDotAndSwitchCommits", "(a).txt"); + this.FileShouldHaveContents(originalContent + newContent, "DeleteFileWithNameAheadOfDotAndSwitchCommits", "(a).txt"); this.ValidateGitCommand("status"); this.ValidateGitCommand("checkout -- DeleteFileWithNameAheadOfDotAndSwitchCommits/(a).txt"); this.ValidateGitCommand("status"); - this.FileShouldHaveContents("DeleteFileWithNameAheadOfDotAndSwitchCommits\\(a).txt", originalContent); + this.FileShouldHaveContents(originalContent, "DeleteFileWithNameAheadOfDotAndSwitchCommits", "(a).txt"); } [TestCase] + [Category(Categories.MacTODO.M3)] public void ResetMixedToCommitWithNewFileThenCheckoutNewBranchAndCheckoutCommitWithNewFile() { this.ControlGitRepo.Fetch(GitRepoTests.ConflictSourceBranch); @@ -386,6 +401,7 @@ public void ResetMixedToCommitWithNewFileThenCheckoutNewBranchAndCheckoutCommitW // ReadFileAfterTryingToReadFileAtCommitWhereFileDoesNotExist is meant to exercise the NegativePathCache and its // behavior when projections change [TestCase] + [Category(Categories.MacTODO.M3)] public void ReadFileAfterTryingToReadFileAtCommitWhereFileDoesNotExist() { this.ControlGitRepo.Fetch(GitRepoTests.ConflictSourceBranch); @@ -398,38 +414,39 @@ public void ReadFileAfterTryingToReadFileAtCommitWhereFileDoesNotExist() this.ValidateGitCommand("checkout db95d631e379d366d26d899523f8136a77441914"); // Files should not exist - this.ShouldNotExistOnDisk(@"Test_ConflictTests\AddedFiles\AddedByBothDifferentContent.txt"); - this.ShouldNotExistOnDisk(@"Test_ConflictTests\AddedFiles\AddedByBothSameContent.txt"); - this.ShouldNotExistOnDisk(@"Test_ConflictTests\AddedFiles\AddedBySource.txt"); + this.ShouldNotExistOnDisk("Test_ConflictTests", "AddedFiles", "AddedByBothDifferentContent.txt"); + this.ShouldNotExistOnDisk("Test_ConflictTests", "AddedFiles", "AddedByBothSameContent.txt"); + this.ShouldNotExistOnDisk("Test_ConflictTests", "AddedFiles", "AddedBySource.txt"); // Check a second time to exercise the ProjFS negative cache - this.ShouldNotExistOnDisk(@"Test_ConflictTests\AddedFiles\AddedByBothDifferentContent.txt"); - this.ShouldNotExistOnDisk(@"Test_ConflictTests\AddedFiles\AddedByBothSameContent.txt"); - this.ShouldNotExistOnDisk(@"Test_ConflictTests\AddedFiles\AddedBySource.txt"); + this.ShouldNotExistOnDisk("Test_ConflictTests", "AddedFiles", "AddedByBothDifferentContent.txt"); + this.ShouldNotExistOnDisk("Test_ConflictTests", "AddedFiles", "AddedByBothSameContent.txt"); + this.ShouldNotExistOnDisk("Test_ConflictTests", "AddedFiles", "AddedBySource.txt"); // Switch to commit where files should exist this.ValidateGitCommand("checkout 51d15f7584e81d59d44c1511ce17d7c493903390"); // Confirm files exist - this.FileContentsShouldMatch(@"Test_ConflictTests\AddedFiles\AddedByBothDifferentContent.txt"); - this.FileContentsShouldMatch(@"Test_ConflictTests\AddedFiles\AddedByBothSameContent.txt"); - this.FileContentsShouldMatch(@"Test_ConflictTests\AddedFiles\AddedBySource.txt"); + this.FileContentsShouldMatch("Test_ConflictTests", "AddedFiles", "AddedByBothDifferentContent.txt"); + this.FileContentsShouldMatch("Test_ConflictTests", "AddedFiles", "AddedByBothSameContent.txt"); + this.FileContentsShouldMatch("Test_ConflictTests", "AddedFiles", "AddedBySource.txt"); // Switch to commit where files should not exist this.ValidateGitCommand("checkout db95d631e379d366d26d899523f8136a77441914"); // Verify files do not not exist - this.ShouldNotExistOnDisk(@"Test_ConflictTests\AddedFiles\AddedByBothDifferentContent.txt"); - this.ShouldNotExistOnDisk(@"Test_ConflictTests\AddedFiles\AddedByBothSameContent.txt"); - this.ShouldNotExistOnDisk(@"Test_ConflictTests\AddedFiles\AddedBySource.txt"); + this.ShouldNotExistOnDisk("Test_ConflictTests", "AddedFiles", "AddedByBothDifferentContent.txt"); + this.ShouldNotExistOnDisk("Test_ConflictTests", "AddedFiles", "AddedByBothSameContent.txt"); + this.ShouldNotExistOnDisk("Test_ConflictTests", "AddedFiles", "AddedBySource.txt"); // Check a second time to exercise the ProjFS negative cache - this.ShouldNotExistOnDisk(@"Test_ConflictTests\AddedFiles\AddedByBothDifferentContent.txt"); - this.ShouldNotExistOnDisk(@"Test_ConflictTests\AddedFiles\AddedByBothSameContent.txt"); - this.ShouldNotExistOnDisk(@"Test_ConflictTests\AddedFiles\AddedBySource.txt"); + this.ShouldNotExistOnDisk("Test_ConflictTests", "AddedFiles", "AddedByBothDifferentContent.txt"); + this.ShouldNotExistOnDisk("Test_ConflictTests", "AddedFiles", "AddedByBothSameContent.txt"); + this.ShouldNotExistOnDisk("Test_ConflictTests", "AddedFiles", "AddedBySource.txt"); } [TestCase] + [Category(Categories.MacTODO.M3)] public void CheckoutBranchWithOpenHandleBlockingRepoMetdataUpdate() { this.ControlGitRepo.Fetch(GitRepoTests.ConflictSourceBranch); @@ -479,6 +496,7 @@ public void CheckoutBranchWithOpenHandleBlockingRepoMetdataUpdate() } [TestCase] + [Category(Categories.MacTODO.M3)] public void CheckoutBranchWithOpenHandleBlockingProjectionDeleteAndRepoMetdataUpdate() { this.ControlGitRepo.Fetch(GitRepoTests.ConflictSourceBranch); @@ -534,6 +552,7 @@ public void CheckoutBranchWithOpenHandleBlockingProjectionDeleteAndRepoMetdataUp } [TestCase] + [Category(Categories.MacTODO.M3)] public void CheckoutBranchWithStaleRepoMetadataTmpFileOnDisk() { this.ControlGitRepo.Fetch(GitRepoTests.ConflictSourceBranch); @@ -544,6 +563,7 @@ public void CheckoutBranchWithStaleRepoMetadataTmpFileOnDisk() } [TestCase] + [Category(Categories.MacTODO.M3)] public void CheckoutBranchWhileOutsideToolDoesNotAllowDeleteOfOpenRepoMetadata() { this.ControlGitRepo.Fetch(GitRepoTests.ConflictSourceBranch); @@ -593,6 +613,7 @@ public void CheckoutBranchWhileOutsideToolDoesNotAllowDeleteOfOpenRepoMetadata() } [TestCase] + [Category(Categories.MacTODO.M4)] public void CheckoutBranchWhileOutsideToolHasExclusiveReadHandleOnDatabasesFolder() { this.ControlGitRepo.Fetch(GitRepoTests.ConflictSourceBranch); @@ -650,6 +671,7 @@ public void CheckoutBranchWhileOutsideToolHasExclusiveReadHandleOnDatabasesFolde } [TestCase] + [Category(Categories.MacTODO.M3)] public void ResetMixedTwiceThenCheckoutWithChanges() { this.ControlGitRepo.Fetch("FunctionalTests/20171219_MultipleFileEdits"); @@ -669,6 +691,7 @@ public void ResetMixedTwiceThenCheckoutWithChanges() } [TestCase] + [Category(Categories.MacTODO.M3)] public void ResetMixedTwiceThenCheckoutWithRemovedFiles() { this.ControlGitRepo.Fetch("FunctionalTests/20180102_MultipleFileDeletes"); @@ -688,6 +711,7 @@ public void ResetMixedTwiceThenCheckoutWithRemovedFiles() } [TestCase] + [Category(Categories.MacTODO.M3)] public void DeleteFolderAndChangeBranchToFolderWithDifferentCase() { // 692765 - Recursive modified paths entries for folders should be case insensitive when @@ -708,6 +732,7 @@ public void DeleteFolderAndChangeBranchToFolderWithDifferentCase() } [TestCase] + [Category(Categories.MacTODO.M3)] public void SuccessfullyChecksOutDirectoryToFileToDirectory() { // This test switches between two branches and verifies specific transitions occured @@ -726,16 +751,16 @@ public void SuccessfullyChecksOutDirectoryToFileToDirectory() // where a\a contains "file contents one" // and b contains "file contents two" // This tests two types of renames crossing into each other - this.FileShouldHaveContents("a\\a", "file contents one"); - this.FileShouldHaveContents("b", "file contents two"); + this.FileShouldHaveContents("file contents one", "a", "a"); + this.FileShouldHaveContents("file contents two", "b"); // Delta of interest - Check initial state // renamed: c\c <-> d\c && d\d <-> c\d // where c\c contains "file contents c" // and d\d contains "file contents d" // This tests two types of renames crossing into each other - this.FileShouldHaveContents("c\\c", "file contents c"); - this.FileShouldHaveContents("d\\d", "file contents d"); + this.FileShouldHaveContents("file contents c", "c", "c"); + this.FileShouldHaveContents("file contents d", "d", "d"); // Now switch to second branch, part2 and verify transitions this.ValidateGitCommand("checkout FunctionalTests/20171103_DirectoryFileTransitionsPart2"); @@ -746,15 +771,15 @@ public void SuccessfullyChecksOutDirectoryToFileToDirectory() // Delta of interest - Verify change // renamed: a\a <-> b && b <-> a - this.FileShouldHaveContents("a", "file contents two"); - this.FileShouldHaveContents("b", "file contents one"); + this.FileShouldHaveContents("file contents two", "a"); + this.FileShouldHaveContents("file contents one", "b"); // Delta of interest - Verify change // renamed: c\c <-> d\c && d\d <-> c\d - this.FileShouldHaveContents("c\\d", "file contents d"); - this.FileShouldHaveContents("d\\c", "file contents c"); - this.ShouldNotExistOnDisk("c\\c"); - this.ShouldNotExistOnDisk("d\\d"); + this.FileShouldHaveContents("file contents d", "c", "d"); + this.FileShouldHaveContents("file contents c", "d", "c"); + this.ShouldNotExistOnDisk("c", "c"); + this.ShouldNotExistOnDisk("d", "d"); // And back again this.ValidateGitCommand("checkout FunctionalTests/20171103_DirectoryFileTransitionsPart1"); @@ -765,37 +790,39 @@ public void SuccessfullyChecksOutDirectoryToFileToDirectory() // Delta of interest - Final validation // renamed: a\a <-> b && b <-> a - this.FileShouldHaveContents("a\\a", "file contents one"); - this.FileShouldHaveContents("b", "file contents two"); + this.FileShouldHaveContents("file contents one", "a", "a"); + this.FileShouldHaveContents("file contents two", "b"); // Delta of interest - Final validation // renamed: c\c <-> d\c && d\d <-> c\d - this.FileShouldHaveContents("c\\c", "file contents c"); - this.FileShouldHaveContents("d\\d", "file contents d"); - this.ShouldNotExistOnDisk("c\\d"); - this.ShouldNotExistOnDisk("d\\c"); + this.FileShouldHaveContents("file contents c", "c", "c"); + this.FileShouldHaveContents("file contents d", "d", "d"); + this.ShouldNotExistOnDisk("c", "d"); + this.ShouldNotExistOnDisk("d", "c"); } [TestCase] + [Category(Categories.MacTODO.M3)] public void DeleteFileThenCheckout() { - this.FolderShouldExistAndHaveFile("GitCommandsTests\\DeleteFileTests\\1", "#test"); - this.DeleteFile("GitCommandsTests\\DeleteFileTests\\1\\#test"); - this.FolderShouldExistAndBeEmpty("GitCommandsTests\\DeleteFileTests\\1"); + this.FolderShouldExistAndHaveFile("GitCommandsTests", "DeleteFileTests", "1", "#test"); + this.DeleteFile("GitCommandsTests", "DeleteFileTests", "1", "#test"); + this.FolderShouldExistAndBeEmpty("GitCommandsTests", "DeleteFileTests", "1"); // Commit cb2d05febf64e3b0df50bd8d3fe8f05c0e2caa47 is before // the files in GitCommandsTests\DeleteFileTests were added this.ValidateGitCommand("checkout cb2d05febf64e3b0df50bd8d3fe8f05c0e2caa47"); - this.ShouldNotExistOnDisk("GitCommandsTests\\DeleteFileTests\\1"); - this.ShouldNotExistOnDisk("GitCommandsTests\\DeleteFileTests"); + this.ShouldNotExistOnDisk("GitCommandsTests", "DeleteFileTests", "1"); + this.ShouldNotExistOnDisk("GitCommandsTests", "DeleteFileTests"); } [TestCase] + [Category(Categories.MacTODO.M3)] public void CheckoutEditCheckoutWithoutFolderThenCheckoutWithMultipleFiles() { // Edit the file to get the entry in the sparse-checkout file - this.EditFile("DeleteFileWithNameAheadOfDotAndSwitchCommits\\1", "Changing the content of one file"); + this.EditFile("Changing the content of one file", "DeleteFileWithNameAheadOfDotAndSwitchCommits", "1"); this.RunGitCommand("reset --hard -q HEAD"); // This commit should remove the DeleteFileWithNameAheadOfDotAndSwitchCommits folder @@ -805,6 +832,7 @@ public void CheckoutEditCheckoutWithoutFolderThenCheckoutWithMultipleFiles() } [TestCase] + [Category(Categories.MacTODO.M3)] public void CreateAFolderThenCheckoutBranchWithFolder() { this.FolderShouldExistAndHaveFile("DeleteFileWithNameAheadOfDotAndSwitchCommits", "1"); @@ -818,6 +846,7 @@ public void CreateAFolderThenCheckoutBranchWithFolder() } [TestCase] + [Category(Categories.MacTODO.M3)] public void CheckoutBranchWithDirectoryNameSameAsFile() { this.SetupForFileDirectoryTest(); @@ -825,24 +854,28 @@ public void CheckoutBranchWithDirectoryNameSameAsFile() } [TestCase] + [Category(Categories.MacTODO.M3)] public void CheckoutBranchWithDirectoryNameSameAsFileEnumerate() { this.RunFileDirectoryEnumerateTest("checkout"); } [TestCase] + [Category(Categories.MacTODO.M3)] public void CheckoutBranchWithDirectoryNameSameAsFileWithRead() { this.RunFileDirectoryReadTest("checkout"); } [TestCase] + [Category(Categories.MacTODO.M3)] public void CheckoutBranchWithDirectoryNameSameAsFileWithWrite() { this.RunFileDirectoryWriteTest("checkout"); } [TestCase] + [Category(Categories.MacTODO.M3)] public void CheckoutBranchDirectoryWithOneFile() { this.SetupForFileDirectoryTest(commandBranch: GitRepoTests.DirectoryWithDifferentFileAfterBranch); @@ -850,18 +883,21 @@ public void CheckoutBranchDirectoryWithOneFile() } [TestCase] + [Category(Categories.MacTODO.M3)] public void CheckoutBranchDirectoryWithOneFileEnumerate() { this.RunFileDirectoryEnumerateTest("checkout", commandBranch: GitRepoTests.DirectoryWithDifferentFileAfterBranch); } [TestCase] + [Category(Categories.MacTODO.M3)] public void CheckoutBranchDirectoryWithOneFileRead() { this.RunFileDirectoryReadTest("checkout", commandBranch: GitRepoTests.DirectoryWithDifferentFileAfterBranch); } [TestCase] + [Category(Categories.MacTODO.M3)] public void CheckoutBranchDirectoryWithOneFileWrite() { this.RunFileDirectoryWriteTest("checkout", commandBranch: GitRepoTests.DirectoryWithDifferentFileAfterBranch); diff --git a/GVFS/GVFS.FunctionalTests/Tests/GitCommands/CherryPickConflictTests.cs b/GVFS/GVFS.FunctionalTests/Tests/GitCommands/CherryPickConflictTests.cs index 9009c4519..5c3ceb735 100644 --- a/GVFS/GVFS.FunctionalTests/Tests/GitCommands/CherryPickConflictTests.cs +++ b/GVFS/GVFS.FunctionalTests/Tests/GitCommands/CherryPickConflictTests.cs @@ -4,6 +4,7 @@ namespace GVFS.FunctionalTests.Tests.GitCommands { [TestFixture] [Category(Categories.GitCommands)] + [Category(Categories.MacTODO.M3)] public class CherryPickConflictTests : GitRepoTests { public CherryPickConflictTests() : base(enlistmentPerTest: true) diff --git a/GVFS/GVFS.FunctionalTests/Tests/GitCommands/DeleteEmptyFolderTests.cs b/GVFS/GVFS.FunctionalTests/Tests/GitCommands/DeleteEmptyFolderTests.cs index 66d1da28e..7bfb99cde 100644 --- a/GVFS/GVFS.FunctionalTests/Tests/GitCommands/DeleteEmptyFolderTests.cs +++ b/GVFS/GVFS.FunctionalTests/Tests/GitCommands/DeleteEmptyFolderTests.cs @@ -6,6 +6,7 @@ namespace GVFS.FunctionalTests.Tests.GitCommands { [TestFixture] [Category(Categories.GitCommands)] + [Category(Categories.MacTODO.M4)] public class DeleteEmptyFolderTests : GitRepoTests { public DeleteEmptyFolderTests() : base(enlistmentPerTest: true) @@ -36,7 +37,7 @@ private void SetupFolderDeleteTest() { ControlGitRepo.Fetch("FunctionalTests/20170202_RenameTestMergeTarget"); this.ValidateGitCommand("checkout FunctionalTests/20170202_RenameTestMergeTarget"); - this.DeleteFile("Test_EPF_GitCommandsTestOnlyFileFolder\\file.txt"); + this.DeleteFile("Test_EPF_GitCommandsTestOnlyFileFolder", "file.txt"); this.ValidateGitCommand("add ."); this.RunGitCommand("commit -m\"Delete only file.\""); } diff --git a/GVFS/GVFS.FunctionalTests/Tests/GitCommands/EnumerationMergeTest.cs b/GVFS/GVFS.FunctionalTests/Tests/GitCommands/EnumerationMergeTest.cs index 4306529ab..6b77a2fdf 100644 --- a/GVFS/GVFS.FunctionalTests/Tests/GitCommands/EnumerationMergeTest.cs +++ b/GVFS/GVFS.FunctionalTests/Tests/GitCommands/EnumerationMergeTest.cs @@ -4,6 +4,7 @@ namespace GVFS.FunctionalTests.Tests.GitCommands { [TestFixture] [Category(Categories.GitCommands)] + [Category(Categories.MacTODO.M3)] public class EnumerationMergeTest : GitRepoTests { // Commit that found GvFlt Bug 12258777: Entries are sometimes skipped during diff --git a/GVFS/GVFS.FunctionalTests/Tests/GitCommands/GitCommandsTests.cs b/GVFS/GVFS.FunctionalTests/Tests/GitCommands/GitCommandsTests.cs index 81383b9e5..3a550d801 100644 --- a/GVFS/GVFS.FunctionalTests/Tests/GitCommands/GitCommandsTests.cs +++ b/GVFS/GVFS.FunctionalTests/Tests/GitCommands/GitCommandsTests.cs @@ -29,7 +29,6 @@ public GitCommandsTests() : base(enlistmentPerTest: false) } [TestCase] - [Category(Categories.Mac.M2)] public void VerifyTestFilesExist() { // Sanity checks to ensure that the test files we expect to be in our test repo are present @@ -41,28 +40,24 @@ public void VerifyTestFilesExist() } [TestCase] - [Category(Categories.Mac.M2)] public void StatusTest() { this.ValidateGitCommand("status"); } [TestCase] - [Category(Categories.Mac.M2)] public void StatusShortTest() { this.ValidateGitCommand("status -s"); } [TestCase] - [Category(Categories.Mac.M2)] public void BranchTest() { this.ValidateGitCommand("branch"); } [TestCase] - [Category(Categories.Mac.M2)] public void NewBranchTest() { this.ValidateGitCommand("branch tests/functional/NewBranchTest"); @@ -70,7 +65,6 @@ public void NewBranchTest() } [TestCase] - [Category(Categories.Mac.M2)] public void DeleteBranchTest() { this.ValidateGitCommand("branch tests/functional/DeleteBranchTest"); @@ -88,56 +82,48 @@ public void RenameCurrentBranchTest() } [TestCase] - [Category(Categories.Mac.M2)] public void UntrackedFileTest() { this.BasicCommit(this.CreateFile, addCommand: "add ."); } [TestCase] - [Category(Categories.Mac.M2)] public void UntrackedEmptyFileTest() { this.BasicCommit(this.CreateEmptyFile, addCommand: "add ."); } [TestCase] - [Category(Categories.Mac.M2)] public void UntrackedFileAddAllTest() { this.BasicCommit(this.CreateFile, addCommand: "add --all"); } [TestCase] - [Category(Categories.Mac.M2)] public void UntrackedEmptyFileAddAllTest() { this.BasicCommit(this.CreateEmptyFile, addCommand: "add --all"); } [TestCase] - [Category(Categories.Mac.M2)] public void StageUntrackedFileTest() { this.BasicCommit(this.CreateFile, addCommand: "stage ."); } [TestCase] - [Category(Categories.Mac.M2)] public void StageUntrackedEmptyFileTest() { this.BasicCommit(this.CreateEmptyFile, addCommand: "stage ."); } [TestCase] - [Category(Categories.Mac.M2)] public void StageUntrackedFileAddAllTest() { this.BasicCommit(this.CreateFile, addCommand: "stage --all"); } [TestCase] - [Category(Categories.Mac.M2)] public void StageUntrackedEmptyFileAddAllTest() { this.BasicCommit(this.CreateEmptyFile, addCommand: "stage --all"); @@ -210,8 +196,8 @@ public void DeleteFileCommitChangesSwitchBranchSwitchBackDeleteFolderTest() { // 663045 - Confirm that folder can be deleted after deleting file then changing // branches - string deleteFolderPath = @"GVFlt_DeleteFolderTest\GVFlt_DeletePlaceholderNonEmptyFolder_DeleteOnClose\NonEmptyFolder"; - string deleteFilePath = deleteFolderPath + @"\bar.txt"; + string deleteFolderPath = Path.Combine("GVFlt_DeleteFolderTest", "GVFlt_DeletePlaceholderNonEmptyFolder_DeleteOnClose", "NonEmptyFolder"); + string deleteFilePath = Path.Combine(deleteFolderPath, "bar.txt"); this.CommitChangesSwitchBranchSwitchBack(fileSystemAction: () => this.DeleteFile(deleteFilePath)); this.DeleteFolder(deleteFolderPath); @@ -220,54 +206,53 @@ public void DeleteFileCommitChangesSwitchBranchSwitchBackDeleteFolderTest() [TestCase] public void DeleteFolderSwitchBranchTest() { - this.SwitchBranch(fileSystemAction: () => this.DeleteFolder(@"GVFlt_DeleteFolderTest\GVFlt_DeleteLocalEmptyFolder_DeleteOnClose")); + this.SwitchBranch(fileSystemAction: () => this.DeleteFolder("GVFlt_DeleteFolderTest", "GVFlt_DeleteLocalEmptyFolder_DeleteOnClose")); } [TestCase] public void DeleteFolderStageChangesSwitchBranchTest() { - this.StageChangesSwitchBranch(fileSystemAction: () => this.DeleteFolder(@"GVFlt_DeleteFolderTest\GVFlt_DeleteLocalEmptyFolder_SetDisposition")); + this.StageChangesSwitchBranch(fileSystemAction: () => this.DeleteFolder("GVFlt_DeleteFolderTest", "GVFlt_DeleteLocalEmptyFolder_SetDisposition")); } [TestCase] public void DeleteFolderCommitChangesSwitchBranchTest() { - this.CommitChangesSwitchBranch(fileSystemAction: () => this.DeleteFolder(@"GVFlt_DeleteFolderTest\GVFlt_DeleteNonRootVirtualFolder_DeleteOnClose")); + this.CommitChangesSwitchBranch(fileSystemAction: () => this.DeleteFolder("GVFlt_DeleteFolderTest", "GVFlt_DeleteNonRootVirtualFolder_DeleteOnClose")); } [TestCase] public void DeleteFolderCommitChangesSwitchBranchSwitchBackTest() { - this.CommitChangesSwitchBranchSwitchBack(fileSystemAction: () => this.DeleteFolder(@"GVFlt_DeleteFolderTest\GVFlt_DeleteNonRootVirtualFolder_SetDisposition")); + this.CommitChangesSwitchBranchSwitchBack(fileSystemAction: () => this.DeleteFolder("GVFlt_DeleteFolderTest", "GVFlt_DeleteNonRootVirtualFolder_SetDisposition")); } [TestCase] - [Category(Categories.Mac.M2)] public void DeleteFilesWithNameAheadOfDot() { string folder = Path.Combine("GitCommandsTests", "DeleteFileTests", "1"); this.FolderShouldExistAndHaveFile(folder, "#test"); - this.DeleteFile(Path.Combine(folder, "#test")); + this.DeleteFile(folder, "#test"); this.FolderShouldExistAndBeEmpty(folder); folder = Path.Combine("GitCommandsTests", "DeleteFileTests", "2"); this.FolderShouldExistAndHaveFile(folder, "$test"); - this.DeleteFile(Path.Combine(folder, "$test")); + this.DeleteFile(folder, "$test"); this.FolderShouldExistAndBeEmpty(folder); folder = Path.Combine("GitCommandsTests", "DeleteFileTests", "3"); this.FolderShouldExistAndHaveFile(folder, ")"); - this.DeleteFile(Path.Combine(folder, ")")); + this.DeleteFile(folder, ")"); this.FolderShouldExistAndBeEmpty(folder); folder = Path.Combine("GitCommandsTests", "DeleteFileTests", "4"); this.FolderShouldExistAndHaveFile(folder, "+.test"); - this.DeleteFile(Path.Combine(folder, "+.test")); + this.DeleteFile(folder, "+.test"); this.FolderShouldExistAndBeEmpty(folder); folder = Path.Combine("GitCommandsTests", "DeleteFileTests", "5"); this.FolderShouldExistAndHaveFile(folder, "-.test"); - this.DeleteFile(Path.Combine(folder, "-.test")); + this.DeleteFile(folder, "-.test"); this.FolderShouldExistAndBeEmpty(folder); this.ValidateGitCommand("status"); @@ -276,20 +261,30 @@ public void DeleteFilesWithNameAheadOfDot() [TestCase] public void RenameFilesWithNameAheadOfDot() { - this.FolderShouldExistAndHaveFile("GitCommandsTests\\RenameFileTests\\1", "#test"); - this.MoveFile("GitCommandsTests\\RenameFileTests\\1\\#test", "GitCommandsTests\\RenameFileTests\\1\\#testRenamed"); + this.FolderShouldExistAndHaveFile("GitCommandsTests", "RenameFileTests", "1", "#test"); + this.MoveFile( + Path.Combine("GitCommandsTests", "RenameFileTests", "1", "#test"), + Path.Combine("GitCommandsTests", "RenameFileTests", "1", "#testRenamed")); - this.FolderShouldExistAndHaveFile("GitCommandsTests\\RenameFileTests\\2", "$test"); - this.MoveFile("GitCommandsTests\\RenameFileTests\\2\\$test", "GitCommandsTests\\RenameFileTests\\2\\$testRenamed"); + this.FolderShouldExistAndHaveFile("GitCommandsTests", "RenameFileTests", "2", "$test"); + this.MoveFile( + Path.Combine("GitCommandsTests", "RenameFileTests", "2", "$test"), + Path.Combine("GitCommandsTests", "RenameFileTests", "2", "$testRenamed")); - this.FolderShouldExistAndHaveFile("GitCommandsTests\\RenameFileTests\\3", ")"); - this.MoveFile("GitCommandsTests\\RenameFileTests\\3\\)", "GitCommandsTests\\RenameFileTests\\3\\)Renamed"); + this.FolderShouldExistAndHaveFile("GitCommandsTests", "RenameFileTests", "3", ")"); + this.MoveFile( + Path.Combine("GitCommandsTests", "RenameFileTests", "3", ")"), + Path.Combine("GitCommandsTests", "RenameFileTests", "3", ")Renamed")); - this.FolderShouldExistAndHaveFile("GitCommandsTests\\RenameFileTests\\4", "+.test"); - this.MoveFile("GitCommandsTests\\RenameFileTests\\4\\+.test", "GitCommandsTests\\RenameFileTests\\4\\+.testRenamed"); + this.FolderShouldExistAndHaveFile("GitCommandsTests", "RenameFileTests", "4", "+.test"); + this.MoveFile( + Path.Combine("GitCommandsTests", "RenameFileTests", "4", "+.test"), + Path.Combine("GitCommandsTests", "RenameFileTests", "4", "+.testRenamed")); - this.FolderShouldExistAndHaveFile("GitCommandsTests\\RenameFileTests\\5", "-.test"); - this.MoveFile("GitCommandsTests\\RenameFileTests\\5\\-.test", "GitCommandsTests\\RenameFileTests\\5\\-.testRenamed"); + this.FolderShouldExistAndHaveFile("GitCommandsTests", "RenameFileTests", "5", "-.test"); + this.MoveFile( + Path.Combine("GitCommandsTests", "RenameFileTests", "5", "-.test"), + Path.Combine("GitCommandsTests", "RenameFileTests", "5", "-.testRenamed")); this.ValidateGitCommand("status"); } @@ -297,10 +292,11 @@ public void RenameFilesWithNameAheadOfDot() [TestCase] public void DeleteFileWithNameAheadOfDotAndSwitchCommits() { - this.DeleteFile("DeleteFileWithNameAheadOfDotAndSwitchCommits\\(1).txt"); + string fileRelativePath = Path.Combine("DeleteFileWithNameAheadOfDotAndSwitchCommits", "(1).txt"); + this.DeleteFile(fileRelativePath); this.ValidateGitCommand("status"); this.ValidateGitCommand("checkout -- DeleteFileWithNameAheadOfDotAndSwitchCommits/(1).txt"); - this.DeleteFile("DeleteFileWithNameAheadOfDotAndSwitchCommits\\(1).txt"); + this.DeleteFile(fileRelativePath); this.ValidateGitCommand("status"); // 14cf226119766146b1fa5c5aa4cd0896d05f6b63 is the commit prior to creating (1).txt, it has two different files with @@ -308,21 +304,22 @@ public void DeleteFileWithNameAheadOfDotAndSwitchCommits() // (a).txt // (z).txt this.ValidateGitCommand("checkout 14cf226119766146b1fa5c5aa4cd0896d05f6b63"); - this.DeleteFile("DeleteFileWithNameAheadOfDotAndSwitchCommits\\(a).txt"); + this.DeleteFile("DeleteFileWithNameAheadOfDotAndSwitchCommits", "(a).txt"); this.ValidateGitCommand("checkout -- DeleteFileWithNameAheadOfDotAndSwitchCommits/(a).txt"); this.ValidateGitCommand("status"); } [TestCase] + [Category(Categories.MacTODO.M3)] public void AddFileAndCommitOnNewBranchSwitchDeleteFolderAndSwitchBack() { // 663045 - Confirm that folder can be deleted after adding a file then changing branches - string newFileParentFolderPath = @"GVFS\GVFS\CommandLine"; - string newFilePath = newFileParentFolderPath + @"\testfile.txt"; + string newFileParentFolderPath = Path.Combine("GVFS", "GVFS", "CommandLine"); + string newFilePath = Path.Combine(newFileParentFolderPath + "testfile.txt"); string newFileContents = "test contents"; this.CommitChangesSwitchBranch( - fileSystemAction: () => this.CreateFile(newFilePath, newFileContents), + fileSystemAction: () => this.CreateFile(newFileContents, newFilePath), test: "AddFileAndCommitOnNewBranchSwitchDeleteFolderAndSwitchBack"); this.ValidateGitCommand("checkout " + this.ControlGitRepo.Commitish); @@ -332,20 +329,20 @@ public void AddFileAndCommitOnNewBranchSwitchDeleteFolderAndSwitchBack() this.ValidateGitCommand("checkout tests/functional/AddFileAndCommitOnNewBranchSwitchDeleteFolderAndSwitchBack"); this.FolderShouldExist(newFileParentFolderPath); - this.FileShouldHaveContents(newFilePath, newFileContents); + this.FileShouldHaveContents(newFileContents, newFilePath); } [TestCase] public void OverwriteFileInSubfolderAndCommitOnNewBranchSwitchDeleteFolderAndSwitchBack() { - string overwrittenFileParentFolderPath = @"GVFlt_DeleteFolderTest\GVFlt_DeletePlaceholderNonEmptyFolder_SetDisposition"; + string overwrittenFileParentFolderPath = Path.Combine("GVFlt_DeleteFolderTest", "GVFlt_DeletePlaceholderNonEmptyFolder_SetDisposition"); // GVFlt_DeleteFolderTest\GVFlt_DeletePlaceholderNonEmptyFolder_SetDispositiontestfile.txt already exists in the repo as TestFile.txt - string fileToOverwritePath = overwrittenFileParentFolderPath + @"\testfile.txt"; + string fileToOverwritePath = Path.Combine(overwrittenFileParentFolderPath, "testfile.txt"); string newFileContents = "test contents"; this.CommitChangesSwitchBranch( - fileSystemAction: () => this.CreateFile(fileToOverwritePath, newFileContents), + fileSystemAction: () => this.CreateFile(newFileContents, fileToOverwritePath), test: "OverwriteFileInSubfolderAndCommitOnNewBranchSwitchDeleteFolderAndSwitchBack"); this.ValidateGitCommand("checkout " + this.ControlGitRepo.Commitish); @@ -354,10 +351,10 @@ public void OverwriteFileInSubfolderAndCommitOnNewBranchSwitchDeleteFolderAndSwi this.ValidateGitCommand("checkout " + this.ControlGitRepo.Commitish); this.ValidateGitCommand("checkout tests/functional/OverwriteFileInSubfolderAndCommitOnNewBranchSwitchDeleteFolderAndSwitchBack"); - string subFolderPath = @"GVFlt_DeleteFolderTest\GVFlt_DeletePlaceholderNonEmptyFolder_SetDisposition\NonEmptyFolder"; + string subFolderPath = Path.Combine("GVFlt_DeleteFolderTest", "GVFlt_DeletePlaceholderNonEmptyFolder_SetDisposition", "NonEmptyFolder"); this.ShouldNotExistOnDisk(subFolderPath); this.FolderShouldExist(overwrittenFileParentFolderPath); - this.FileShouldHaveContents(fileToOverwritePath, newFileContents); + this.FileShouldHaveContents(newFileContents, fileToOverwritePath); } [TestCase] @@ -365,13 +362,13 @@ public void AddFileInSubfolderAndCommitOnNewBranchSwitchDeleteFolderAndSwitchBac { // 663045 - Confirm that grandparent folder can be deleted after adding a (granchild) file // then changing branches - string newFileParentFolderPath = @"GVFlt_DeleteFolderTest\GVFlt_DeleteVirtualNonEmptyFolder_DeleteOnClose\NonEmptyFolder"; - string newFileGrandParentFolderPath = @"GVFlt_DeleteFolderTest\GVFlt_DeleteVirtualNonEmptyFolder_DeleteOnClose"; - string newFilePath = newFileParentFolderPath + @"\testfile.txt"; + string newFileParentFolderPath = Path.Combine("GVFlt_DeleteFolderTest", "GVFlt_DeleteVirtualNonEmptyFolder_DeleteOnClose", "NonEmptyFolder"); + string newFileGrandParentFolderPath = Path.Combine("GVFlt_DeleteFolderTest", "GVFlt_DeleteVirtualNonEmptyFolder_DeleteOnClose"); + string newFilePath = Path.Combine(newFileParentFolderPath, "testfile.txt"); string newFileContents = "test contents"; this.CommitChangesSwitchBranch( - fileSystemAction: () => this.CreateFile(newFilePath, newFileContents), + fileSystemAction: () => this.CreateFile(newFileContents, newFilePath), test: "AddFileInSubfolderAndCommitOnNewBranchSwitchDeleteFolderAndSwitchBack"); this.ValidateGitCommand("checkout " + this.ControlGitRepo.Commitish); @@ -382,7 +379,7 @@ public void AddFileInSubfolderAndCommitOnNewBranchSwitchDeleteFolderAndSwitchBac this.FolderShouldExist(newFileParentFolderPath); this.FolderShouldExist(newFileGrandParentFolderPath); - this.FileShouldHaveContents(newFilePath, newFileContents); + this.FileShouldHaveContents(newFileContents, newFilePath); } [TestCase] @@ -541,8 +538,10 @@ public void RenameFileCommitChangesSwitchBranchSwitchBackTest() this.CommitChangesSwitchBranchSwitchBack(fileSystemAction: this.RenameFile); } + // MacOnly because renames of partial folders are blocked on Windows [TestCase] - [Ignore("Disabled until moving partial folders is supported")] + [Category(Categories.MacOnly)] + [Category(Categories.MacTODO.M3)] public void MoveFolderCommitChangesSwitchBranchSwitchBackTest() { this.CommitChangesSwitchBranchSwitchBack(fileSystemAction: this.MoveFolder); @@ -553,8 +552,8 @@ public void AddFileCommitThenDeleteAndCommit() { this.ValidateGitCommand("checkout -b tests/functional/AddFileCommitThenDeleteAndCommit_before"); this.ValidateGitCommand("checkout -b tests/functional/AddFileCommitThenDeleteAndCommit_after"); - string filePath = @"GVFS\testfile.txt"; - this.CreateFile(filePath, "Some new content for the file"); + string filePath = Path.Combine("GVFS", "testfile.txt"); + this.CreateFile("Some new content for the file", filePath); this.ValidateGitCommand("status"); this.ValidateGitCommand("add ."); this.RunGitCommand("commit -m \"Change for AddFileCommitThenDeleteAndCommit\""); @@ -572,8 +571,8 @@ public void AddFileCommitThenDeleteAndCommit() public void AddFileCommitThenDeleteAndResetSoft() { this.ValidateGitCommand("checkout -b tests/functional/AddFileCommitThenDeleteAndResetSoft"); - string filePath = @"GVFS\testfile.txt"; - this.CreateFile(filePath, "Some new content for the file"); + string filePath = Path.Combine("GVFS", "testfile.txt"); + this.CreateFile("Some new content for the file", filePath); this.ValidateGitCommand("status"); this.ValidateGitCommand("add ."); this.RunGitCommand("commit -m \"Change for AddFileCommitThenDeleteAndCommit\""); @@ -586,8 +585,8 @@ public void AddFileCommitThenDeleteAndResetSoft() public void AddFileCommitThenDeleteAndResetMixed() { this.ValidateGitCommand("checkout -b tests/functional/AddFileCommitThenDeleteAndResetSoft"); - string filePath = @"GVFS\testfile.txt"; - this.CreateFile(filePath, "Some new content for the file"); + string filePath = Path.Combine("GVFS", "testfile.txt"); + this.CreateFile("Some new content for the file", filePath); this.ValidateGitCommand("status"); this.ValidateGitCommand("add ."); this.RunGitCommand("commit -m \"Change for AddFileCommitThenDeleteAndCommit\""); @@ -602,8 +601,8 @@ public void AddFolderAndFileCommitThenDeleteAndResetSoft() this.ValidateGitCommand("checkout -b tests/functional/AddFileCommitThenDeleteAndResetSoft"); string folderPath = "test_folder"; this.CreateFolder(folderPath); - string filePath = folderPath + @"\testfile.txt"; - this.CreateFile(filePath, "Some new content for the file"); + string filePath = Path.Combine(folderPath, "testfile.txt"); + this.CreateFile("Some new content for the file", filePath); this.ValidateGitCommand("status"); this.ValidateGitCommand("add ."); this.RunGitCommand("commit -m \"Change for AddFileCommitThenDeleteAndCommit\""); @@ -619,8 +618,8 @@ public void AddFolderAndFileCommitThenDeleteAndResetMixed() this.ValidateGitCommand("checkout -b tests/functional/AddFileCommitThenDeleteAndResetSoft"); string folderPath = "test_folder"; this.CreateFolder(folderPath); - string filePath = folderPath + @"\testfile.txt"; - this.CreateFile(filePath, "Some new content for the file"); + string filePath = Path.Combine(folderPath, "testfile.txt"); + this.CreateFile("Some new content for the file", filePath); this.ValidateGitCommand("status"); this.ValidateGitCommand("add ."); this.RunGitCommand("commit -m \"Change for AddFileCommitThenDeleteAndCommit\""); @@ -636,8 +635,8 @@ public void AddFolderAndFileCommitThenResetSoftAndResetHard() this.ValidateGitCommand("checkout -b tests/functional/AddFileCommitThenDeleteAndResetSoft"); string folderPath = "test_folder"; this.CreateFolder(folderPath); - string filePath = folderPath + @"\testfile.txt"; - this.CreateFile(filePath, "Some new content for the file"); + string filePath = Path.Combine(folderPath, "testfile.txt"); + this.CreateFile("Some new content for the file", filePath); this.ValidateGitCommand("status"); this.ValidateGitCommand("add ."); this.RunGitCommand("commit -m \"Change for AddFileCommitThenDeleteAndCommit\""); @@ -652,8 +651,8 @@ public void AddFolderAndFileCommitThenResetSoftAndResetMixed() this.ValidateGitCommand("checkout -b tests/functional/AddFileCommitThenDeleteAndResetSoft"); string folderPath = "test_folder"; this.CreateFolder(folderPath); - string filePath = folderPath + @"\testfile.txt"; - this.CreateFile(filePath, "Some new content for the file"); + string filePath = Path.Combine(folderPath + "testfile.txt"); + this.CreateFile("Some new content for the file", filePath); this.ValidateGitCommand("status"); this.ValidateGitCommand("add ."); this.RunGitCommand("commit -m \"Change for AddFileCommitThenDeleteAndCommit\""); @@ -669,28 +668,28 @@ public void AddFoldersAndFilesAndRenameFolder() string topMostNewFolder = "AddFoldersAndFilesAndRenameFolder_Test"; this.CreateFolder(topMostNewFolder); - this.CreateFile(topMostNewFolder + @"\top_level_test_file.txt", "test contents"); + this.CreateFile("test contents", topMostNewFolder, "top_level_test_file.txt"); - string testFolderLevel1 = topMostNewFolder + @"\TestFolderLevel1"; + string testFolderLevel1 = Path.Combine(topMostNewFolder, "TestFolderLevel1"); this.CreateFolder(testFolderLevel1); - this.CreateFile(testFolderLevel1 + @"\level_1_test_file.txt", "test contents"); + this.CreateFile("test contents", testFolderLevel1, "level_1_test_file.txt"); - string testFolderLevel2 = testFolderLevel1 + @"\TestFolderLevel2"; + string testFolderLevel2 = Path.Combine(testFolderLevel1, "TestFolderLevel2"); this.CreateFolder(testFolderLevel2); - this.CreateFile(testFolderLevel2 + @"\level_2_test_file.txt", "test contents"); + this.CreateFile("test contents", testFolderLevel2, "level_2_test_file.txt"); - string testFolderLevel3 = testFolderLevel2 + @"\TestFolderLevel3"; + string testFolderLevel3 = Path.Combine(testFolderLevel2, "TestFolderLevel3"); this.CreateFolder(testFolderLevel3); - this.CreateFile(testFolderLevel3 + @"\level_3_test_file.txt", "test contents"); + this.CreateFile("test contents", testFolderLevel3, "level_3_test_file.txt"); this.ValidateGitCommand("status"); - this.MoveFolder(testFolderLevel3, testFolderLevel2 + @"\TestFolderLevel3Renamed"); + this.MoveFolder(testFolderLevel3, Path.Combine(testFolderLevel2, "TestFolderLevel3Renamed")); this.ValidateGitCommand("status"); - this.MoveFolder(testFolderLevel2, testFolderLevel1 + @"\TestFolderLevel2Renamed"); + this.MoveFolder(testFolderLevel2, Path.Combine(testFolderLevel1, "TestFolderLevel2Renamed")); this.ValidateGitCommand("status"); - this.MoveFolder(testFolderLevel1, topMostNewFolder + @"\TestFolderLevel1Renamed"); + this.MoveFolder(testFolderLevel1, Path.Combine(topMostNewFolder, "TestFolderLevel1Renamed")); this.ValidateGitCommand("status"); this.MoveFolder(topMostNewFolder, "AddFoldersAndFilesAndRenameFolder_TestRenamed"); @@ -706,11 +705,12 @@ public void AddFileAfterFolderRename() string renamedFolder = "AddFileAfterFolderRename_TestRenamed"; this.CreateFolder(folder); this.MoveFolder(folder, renamedFolder); - this.CreateFile(renamedFolder + @"\test_file.txt", "test contents"); + this.CreateFile("test contents", renamedFolder, "test_file.txt"); this.ValidateGitCommand("status"); } [TestCase] + [Category(Categories.MacTODO.M3)] public void ResetSoft() { this.ValidateGitCommand("checkout -b tests/functional/ResetSoft"); @@ -718,6 +718,7 @@ public void ResetSoft() } [TestCase] + [Category(Categories.MacTODO.M3)] public void ResetMixed() { this.ValidateGitCommand("checkout -b tests/functional/ResetMixed"); @@ -725,6 +726,7 @@ public void ResetMixed() } [TestCase] + [Category(Categories.MacTODO.M3)] public void ResetMixed2() { this.ValidateGitCommand("checkout -b tests/functional/ResetMixed2"); @@ -735,11 +737,12 @@ public void ResetMixed2() public void ManuallyModifyHead() { this.ValidateGitCommand("status"); - this.ReplaceText(TestConstants.DotGit.Head, "f1bce402a7a980a8320f3f235cf8c8fdade4b17a"); + this.ReplaceText("f1bce402a7a980a8320f3f235cf8c8fdade4b17a", TestConstants.DotGit.Head); this.ValidateGitCommand("status"); } [TestCase] + [Category(Categories.MacTODO.M3)] public void ResetSoftTwice() { this.ValidateGitCommand("checkout -b tests/functional/ResetSoftTwice"); @@ -751,6 +754,7 @@ public void ResetSoftTwice() } [TestCase] + [Category(Categories.MacTODO.M3)] public void ResetMixedTwice() { this.ValidateGitCommand("checkout -b tests/functional/ResetMixedTwice"); @@ -980,8 +984,8 @@ public void EditFileNeedingUtf8Encoding() GVFSHelpers.ModifiedPathsShouldNotContain(this.FileSystem, this.Enlistment.DotGVFSRoot, EncodingFilename); this.ValidateGitCommand("status"); - this.AppendAllText(virtualFile, ContentWhenEditingFile); - this.AppendAllText(controlFile, ContentWhenEditingFile); + this.AppendAllText(ContentWhenEditingFile, virtualFile); + this.AppendAllText(ContentWhenEditingFile, controlFile); this.ValidateGitCommand("status"); @@ -997,13 +1001,14 @@ public void UseAlias() } [TestCase] + [Category(Categories.MacTODO.M3)] public void RenameOnlyFileInFolder() { ControlGitRepo.Fetch("FunctionalTests/20170202_RenameTestMergeTarget"); ControlGitRepo.Fetch("FunctionalTests/20170202_RenameTestMergeSource"); this.ValidateGitCommand("checkout FunctionalTests/20170202_RenameTestMergeTarget"); - this.FileSystem.ReadAllText(this.Enlistment.GetVirtualPathTo("Test_EPF_GitCommandsTestOnlyFileFolder\\file.txt")); + this.FileSystem.ReadAllText(this.Enlistment.GetVirtualPathTo("Test_EPF_GitCommandsTestOnlyFileFolder", "file.txt")); this.ValidateGitCommand("merge origin/FunctionalTests/20170202_RenameTestMergeSource"); } @@ -1073,12 +1078,12 @@ private void CommitChangesSwitchBranchSwitchBack(Action fileSystemAction, [Calle private void CreateFile() { - this.CreateFile(Path.GetRandomFileName() + "tempFile.txt", "Some content here"); + this.CreateFile("Some content here", Path.GetRandomFileName() + "tempFile.txt"); } private void EditFile() { - this.AppendAllText(GitCommandsTests.EditFilePath, ContentWhenEditingFile); + this.AppendAllText(ContentWhenEditingFile, GitCommandsTests.EditFilePath); } private void DeleteFile() diff --git a/GVFS/GVFS.FunctionalTests/Tests/GitCommands/GitRepoTests.cs b/GVFS/GVFS.FunctionalTests/Tests/GitCommands/GitRepoTests.cs index 89fe43289..39301b73e 100644 --- a/GVFS/GVFS.FunctionalTests/Tests/GitCommands/GitRepoTests.cs +++ b/GVFS/GVFS.FunctionalTests/Tests/GitCommands/GitRepoTests.cs @@ -191,8 +191,9 @@ protected void CreateEmptyFile() this.FileSystem.CreateEmptyFile(controlFile); } - protected void CreateFile(string filePath, string content) + protected void CreateFile(string content, params string[] filePathPaths) { + string filePath = Path.Combine(filePathPaths); string virtualFile = Path.Combine(this.Enlistment.RepoRoot, filePath); string controlFile = Path.Combine(this.ControlGitRepo.RootPath, filePath); this.FileSystem.WriteAllText(virtualFile, content); @@ -207,8 +208,9 @@ protected void CreateFolder(string folderPath) this.FileSystem.CreateDirectory(controlFolder); } - protected void EditFile(string filePath, string content) + protected void EditFile(string content, params string[] filePathParts) { + string filePath = Path.Combine(filePathParts); string virtualFile = Path.Combine(this.Enlistment.RepoRoot, filePath); string controlFile = Path.Combine(this.ControlGitRepo.RootPath, filePath); this.FileSystem.AppendAllText(virtualFile, content); @@ -255,8 +257,9 @@ protected void MoveFile(string pathFrom, string pathTo) controlFileTo.ShouldBeAFile(this.FileSystem); } - protected void DeleteFile(string filePath) + protected void DeleteFile(params string[] filePathParts) { + string filePath = Path.Combine(filePathParts); string virtualFile = Path.Combine(this.Enlistment.RepoRoot, filePath); string controlFile = Path.Combine(this.ControlGitRepo.RootPath, filePath); this.FileSystem.DeleteFile(virtualFile); @@ -265,8 +268,9 @@ protected void DeleteFile(string filePath) controlFile.ShouldNotExistOnDisk(this.FileSystem); } - protected void DeleteFolder(string folderPath) + protected void DeleteFolder(params string[] folderPathParts) { + string folderPath = Path.Combine(folderPathParts); string virtualFolder = Path.Combine(this.Enlistment.RepoRoot, folderPath); string controlFolder = Path.Combine(this.ControlGitRepo.RootPath, folderPath); this.FileSystem.DeleteDirectory(virtualFolder); @@ -287,48 +291,57 @@ protected void MoveFolder(string pathFrom, string pathTo) controlFileFrom.ShouldNotExistOnDisk(this.FileSystem); } - protected void FolderShouldExist(string folderPath) + protected void FolderShouldExist(params string[] folderPathParts) { + string folderPath = Path.Combine(folderPathParts); string virtualFolder = Path.Combine(this.Enlistment.RepoRoot, folderPath); string controlFolder = Path.Combine(this.ControlGitRepo.RootPath, folderPath); virtualFolder.ShouldBeADirectory(this.FileSystem); controlFolder.ShouldBeADirectory(this.FileSystem); } - protected void FolderShouldExistAndHaveFile(string folderPath, string fileName) + protected void FolderShouldExistAndHaveFile(params string[] filePathParts) { + string filePath = Path.Combine(filePathParts); + string folderPath = Path.GetDirectoryName(filePath); + string fileName = Path.GetFileName(filePath); + string virtualFolder = Path.Combine(this.Enlistment.RepoRoot, folderPath); string controlFolder = Path.Combine(this.ControlGitRepo.RootPath, folderPath); virtualFolder.ShouldBeADirectory(this.FileSystem).WithItems(fileName).Count().ShouldEqual(1); controlFolder.ShouldBeADirectory(this.FileSystem).WithItems(fileName).Count().ShouldEqual(1); } - protected void FolderShouldExistAndBeEmpty(string folderPath) + protected void FolderShouldExistAndBeEmpty(params string[] folderPathParts) { + string folderPath = Path.Combine(folderPathParts); string virtualFolder = Path.Combine(this.Enlistment.RepoRoot, folderPath); string controlFolder = Path.Combine(this.ControlGitRepo.RootPath, folderPath); virtualFolder.ShouldBeADirectory(this.FileSystem).WithNoItems(); controlFolder.ShouldBeADirectory(this.FileSystem).WithNoItems(); } - protected void ShouldNotExistOnDisk(string path) + protected void ShouldNotExistOnDisk(params string[] pathParts) { + string path = Path.Combine(pathParts); string virtualPath = Path.Combine(this.Enlistment.RepoRoot, path); string controlPath = Path.Combine(this.ControlGitRepo.RootPath, path); virtualPath.ShouldNotExistOnDisk(this.FileSystem); controlPath.ShouldNotExistOnDisk(this.FileSystem); } - protected void FileShouldHaveContents(string filePath, string contents) + protected void FileShouldHaveContents(string contents, params string[] filePathParts) { + string filePath = Path.Combine(filePathParts); string virtualFilePath = Path.Combine(this.Enlistment.RepoRoot, filePath); string controlFilePath = Path.Combine(this.ControlGitRepo.RootPath, filePath); virtualFilePath.ShouldBeAFile(this.FileSystem).WithContents(contents); controlFilePath.ShouldBeAFile(this.FileSystem).WithContents(contents); } - protected void FileContentsShouldMatch(string filePath) + protected void FileContentsShouldMatch(params string[] filePathPaths) { + string filePath = Path.Combine(filePathPaths); string virtualFilePath = Path.Combine(this.Enlistment.RepoRoot, filePath); string controlFilePath = Path.Combine(this.ControlGitRepo.RootPath, filePath); virtualFilePath.ShouldBeAFile(this.FileSystem).WithContents(controlFilePath.ShouldBeAFile(this.FileSystem).WithContents()); @@ -350,16 +363,18 @@ protected void FolderShouldHaveCaseMatchingName(string folderPath, string caseSe controlFolderPath.ShouldBeADirectory(this.FileSystem).WithCaseMatchingName(caseSensitiveName); } - protected void AppendAllText(string filePath, string content) + protected void AppendAllText(string content, params string[] filePathParts) { + string filePath = Path.Combine(filePathParts); string virtualFile = Path.Combine(this.Enlistment.RepoRoot, filePath); string controlFile = Path.Combine(this.ControlGitRepo.RootPath, filePath); this.FileSystem.AppendAllText(virtualFile, content); this.FileSystem.AppendAllText(controlFile, content); } - protected void ReplaceText(string filePath, string newContent) + protected void ReplaceText(string newContent, params string[] filePathParts) { + string filePath = Path.Combine(filePathParts); string virtualFile = Path.Combine(this.Enlistment.RepoRoot, filePath); string controlFile = Path.Combine(this.ControlGitRepo.RootPath, filePath); this.FileSystem.WriteAllText(virtualFile, newContent); @@ -375,7 +390,7 @@ protected void SetupForFileDirectoryTest(string commandBranch = DirectoryWithFil protected void ValidateFileDirectoryTest(string command, string commandBranch = DirectoryWithFileAfterBranch) { - this.EditFile("Readme.md", "Change file"); + this.EditFile("Change file", "Readme.md"); this.ValidateGitCommand("add --all"); this.RunGitCommand("commit -m \"Some change\""); this.ValidateGitCommand($"{command} {commandBranch}"); @@ -401,88 +416,88 @@ protected void RunFileDirectoryReadTest(string command, string commandBranch = D protected void RunFileDirectoryWriteTest(string command, string commandBranch = DirectoryWithFileAfterBranch) { this.SetupForFileDirectoryTest(commandBranch); - this.EditFile("file.txt\\file.txt", "Change file"); + this.EditFile("Change file", "file.txt", "file.txt"); this.ValidateFileDirectoryTest(command, commandBranch); } protected void ReadConflictTargetFiles() { - this.FileContentsShouldMatch(@"Test_ConflictTests\AddedFiles\AddedByBothDifferentContent.txt"); - this.FileContentsShouldMatch(@"Test_ConflictTests\AddedFiles\AddedByBothSameContent.txt"); - this.FileContentsShouldMatch(@"Test_ConflictTests\AddedFiles\AddedByTarget.txt"); - this.FileContentsShouldMatch(@"Test_ConflictTests\ModifiedFiles\ChangeInSource.txt"); - this.FileContentsShouldMatch(@"Test_ConflictTests\ModifiedFiles\ChangeInTarget.txt"); - this.FileContentsShouldMatch(@"Test_ConflictTests\ModifiedFiles\ChangeInTargetDeleteInSource.txt"); - this.FileContentsShouldMatch(@"Test_ConflictTests\ModifiedFiles\ConflictingChange.txt"); - this.FileContentsShouldMatch(@"Test_ConflictTests\ModifiedFiles\SameChange.txt"); - this.FileContentsShouldMatch(@"Test_ConflictTests\ModifiedFiles\SuccessfulMerge.txt"); - this.FileContentsShouldMatch(@"Test_ConflictTests\DeletedFiles\DeleteInSource.txt"); + this.FileContentsShouldMatch("Test_ConflictTests", "AddedFiles", "AddedByBothDifferentContent.txt"); + this.FileContentsShouldMatch("Test_ConflictTests", "AddedFiles", "AddedByBothSameContent.txt"); + this.FileContentsShouldMatch("Test_ConflictTests", "AddedFiles", "AddedByTarget.txt"); + this.FileContentsShouldMatch("Test_ConflictTests", "ModifiedFiles", "ChangeInSource.txt"); + this.FileContentsShouldMatch("Test_ConflictTests", "ModifiedFiles", "ChangeInTarget.txt"); + this.FileContentsShouldMatch("Test_ConflictTests", "ModifiedFiles", "ChangeInTargetDeleteInSource.txt"); + this.FileContentsShouldMatch("Test_ConflictTests", "ModifiedFiles", "ConflictingChange.txt"); + this.FileContentsShouldMatch("Test_ConflictTests", "ModifiedFiles", "SameChange.txt"); + this.FileContentsShouldMatch("Test_ConflictTests", "ModifiedFiles", "SuccessfulMerge.txt"); + this.FileContentsShouldMatch("Test_ConflictTests", "DeletedFiles", "DeleteInSource.txt"); } protected void FilesShouldMatchCheckoutOfTargetBranch() { - this.FileContentsShouldMatch(@"Test_ConflictTests\AddedFiles\AddedByBothDifferentContent.txt"); - this.FileContentsShouldMatch(@"Test_ConflictTests\AddedFiles\AddedByBothSameContent.txt"); - this.FileContentsShouldMatch(@"Test_ConflictTests\AddedFiles\AddedByTarget.txt"); - this.FileContentsShouldMatch(@"Test_ConflictTests\AddedFiles\NoChange.txt"); + this.FileContentsShouldMatch("Test_ConflictTests", "AddedFiles", "AddedByBothDifferentContent.txt"); + this.FileContentsShouldMatch("Test_ConflictTests", "AddedFiles", "AddedByBothSameContent.txt"); + this.FileContentsShouldMatch("Test_ConflictTests", "AddedFiles", "AddedByTarget.txt"); + this.FileContentsShouldMatch("Test_ConflictTests", "AddedFiles", "NoChange.txt"); - this.FileContentsShouldMatch(@"Test_ConflictTests\DeletedFiles\DeleteInSource.txt"); + this.FileContentsShouldMatch("Test_ConflictTests", "DeletedFiles", "DeleteInSource.txt"); - this.FileContentsShouldMatch(@"Test_ConflictTests\ModifiedFiles\ChangeInSource.txt"); - this.FileContentsShouldMatch(@"Test_ConflictTests\ModifiedFiles\ChangeInTarget.txt"); - this.FileContentsShouldMatch(@"Test_ConflictTests\ModifiedFiles\ChangeInTargetDeleteInSource.txt"); - this.FileContentsShouldMatch(@"Test_ConflictTests\ModifiedFiles\ConflictingChange.txt"); - this.FileContentsShouldMatch(@"Test_ConflictTests\ModifiedFiles\SameChange.txt"); - this.FileContentsShouldMatch(@"Test_ConflictTests\ModifiedFiles\SuccessfulMerge.txt"); + this.FileContentsShouldMatch("Test_ConflictTests", "ModifiedFiles", "ChangeInSource.txt"); + this.FileContentsShouldMatch("Test_ConflictTests", "ModifiedFiles", "ChangeInTarget.txt"); + this.FileContentsShouldMatch("Test_ConflictTests", "ModifiedFiles", "ChangeInTargetDeleteInSource.txt"); + this.FileContentsShouldMatch("Test_ConflictTests", "ModifiedFiles", "ConflictingChange.txt"); + this.FileContentsShouldMatch("Test_ConflictTests", "ModifiedFiles", "SameChange.txt"); + this.FileContentsShouldMatch("Test_ConflictTests", "ModifiedFiles", "SuccessfulMerge.txt"); } protected void FilesShouldMatchCheckoutOfSourceBranch() { - this.FileContentsShouldMatch(@"Test_ConflictTests\AddedFiles\AddedByBothDifferentContent.txt"); - this.FileContentsShouldMatch(@"Test_ConflictTests\AddedFiles\AddedByBothSameContent.txt"); - this.FileContentsShouldMatch(@"Test_ConflictTests\AddedFiles\AddedBySource.txt"); - this.FileContentsShouldMatch(@"Test_ConflictTests\AddedFiles\NoChange.txt"); + this.FileContentsShouldMatch("Test_ConflictTests", "AddedFiles", "AddedByBothDifferentContent.txt"); + this.FileContentsShouldMatch("Test_ConflictTests", "AddedFiles", "AddedByBothSameContent.txt"); + this.FileContentsShouldMatch("Test_ConflictTests", "AddedFiles", "AddedBySource.txt"); + this.FileContentsShouldMatch("Test_ConflictTests", "AddedFiles", "NoChange.txt"); - this.FileContentsShouldMatch(@"Test_ConflictTests\DeletedFiles\DeleteInTarget.txt"); + this.FileContentsShouldMatch("Test_ConflictTests", "DeletedFiles", "DeleteInTarget.txt"); - this.FileContentsShouldMatch(@"Test_ConflictTests\ModifiedFiles\ChangeInSource.txt"); - this.FileContentsShouldMatch(@"Test_ConflictTests\ModifiedFiles\ChangeInSourceDeleteInTarget.txt"); - this.FileContentsShouldMatch(@"Test_ConflictTests\ModifiedFiles\ChangeInTarget.txt"); - this.FileContentsShouldMatch(@"Test_ConflictTests\ModifiedFiles\ConflictingChange.txt"); - this.FileContentsShouldMatch(@"Test_ConflictTests\ModifiedFiles\SameChange.txt"); - this.FileContentsShouldMatch(@"Test_ConflictTests\ModifiedFiles\SuccessfulMerge.txt"); + this.FileContentsShouldMatch("Test_ConflictTests", "ModifiedFiles", "ChangeInSource.txt"); + this.FileContentsShouldMatch("Test_ConflictTests", "ModifiedFiles", "ChangeInSourceDeleteInTarget.txt"); + this.FileContentsShouldMatch("Test_ConflictTests", "ModifiedFiles", "ChangeInTarget.txt"); + this.FileContentsShouldMatch("Test_ConflictTests", "ModifiedFiles", "ConflictingChange.txt"); + this.FileContentsShouldMatch("Test_ConflictTests", "ModifiedFiles", "SameChange.txt"); + this.FileContentsShouldMatch("Test_ConflictTests", "ModifiedFiles", "SuccessfulMerge.txt"); } protected void FilesShouldMatchAfterNoConflict() { - this.FileContentsShouldMatch(@"Test_ConflictTests\AddedFiles\AddedByBothDifferentContent.txt"); - this.FileContentsShouldMatch(@"Test_ConflictTests\AddedFiles\AddedByBothSameContent.txt"); - this.FileContentsShouldMatch(@"Test_ConflictTests\AddedFiles\AddedByTarget.txt"); - this.FileContentsShouldMatch(@"Test_ConflictTests\AddedFiles\NoChange.txt"); + this.FileContentsShouldMatch("Test_ConflictTests", "AddedFiles", "AddedByBothDifferentContent.txt"); + this.FileContentsShouldMatch("Test_ConflictTests", "AddedFiles", "AddedByBothSameContent.txt"); + this.FileContentsShouldMatch("Test_ConflictTests", "AddedFiles", "AddedByTarget.txt"); + this.FileContentsShouldMatch("Test_ConflictTests", "AddedFiles", "NoChange.txt"); - this.FileContentsShouldMatch(@"Test_ConflictTests\ModifiedFiles\ChangeInSource.txt"); - this.FileContentsShouldMatch(@"Test_ConflictTests\ModifiedFiles\ChangeInTarget.txt"); - this.FileContentsShouldMatch(@"Test_ConflictTests\ModifiedFiles\ChangeInTargetDeleteInSource.txt"); - this.FileContentsShouldMatch(@"Test_ConflictTests\ModifiedFiles\ConflictingChange.txt"); - this.FileContentsShouldMatch(@"Test_ConflictTests\ModifiedFiles\SameChange.txt"); - this.FileContentsShouldMatch(@"Test_ConflictTests\ModifiedFiles\SuccessfulMerge.txt"); + this.FileContentsShouldMatch("Test_ConflictTests", "ModifiedFiles", "ChangeInSource.txt"); + this.FileContentsShouldMatch("Test_ConflictTests", "ModifiedFiles", "ChangeInTarget.txt"); + this.FileContentsShouldMatch("Test_ConflictTests", "ModifiedFiles", "ChangeInTargetDeleteInSource.txt"); + this.FileContentsShouldMatch("Test_ConflictTests", "ModifiedFiles", "ConflictingChange.txt"); + this.FileContentsShouldMatch("Test_ConflictTests", "ModifiedFiles", "SameChange.txt"); + this.FileContentsShouldMatch("Test_ConflictTests", "ModifiedFiles", "SuccessfulMerge.txt"); } protected void FilesShouldMatchAfterConflict() { - this.FileContentsShouldMatch(@"Test_ConflictTests\AddedFiles\AddedByBothDifferentContent.txt"); - this.FileContentsShouldMatch(@"Test_ConflictTests\AddedFiles\AddedByBothSameContent.txt"); - this.FileContentsShouldMatch(@"Test_ConflictTests\AddedFiles\AddedBySource.txt"); - this.FileContentsShouldMatch(@"Test_ConflictTests\AddedFiles\AddedByTarget.txt"); - this.FileContentsShouldMatch(@"Test_ConflictTests\AddedFiles\NoChange.txt"); - - this.FileContentsShouldMatch(@"Test_ConflictTests\ModifiedFiles\ChangeInSource.txt"); - this.FileContentsShouldMatch(@"Test_ConflictTests\ModifiedFiles\ChangeInSourceDeleteInTarget.txt"); - this.FileContentsShouldMatch(@"Test_ConflictTests\ModifiedFiles\ChangeInTarget.txt"); - this.FileContentsShouldMatch(@"Test_ConflictTests\ModifiedFiles\ChangeInTargetDeleteInSource.txt"); - this.FileContentsShouldMatch(@"Test_ConflictTests\ModifiedFiles\ConflictingChange.txt"); - this.FileContentsShouldMatch(@"Test_ConflictTests\ModifiedFiles\SameChange.txt"); - this.FileContentsShouldMatch(@"Test_ConflictTests\ModifiedFiles\SuccessfulMerge.txt"); + this.FileContentsShouldMatch("Test_ConflictTests", "AddedFiles", "AddedByBothDifferentContent.txt"); + this.FileContentsShouldMatch("Test_ConflictTests", "AddedFiles", "AddedByBothSameContent.txt"); + this.FileContentsShouldMatch("Test_ConflictTests", "AddedFiles", "AddedBySource.txt"); + this.FileContentsShouldMatch("Test_ConflictTests", "AddedFiles", "AddedByTarget.txt"); + this.FileContentsShouldMatch("Test_ConflictTests", "AddedFiles", "NoChange.txt"); + + this.FileContentsShouldMatch("Test_ConflictTests", "ModifiedFiles", "ChangeInSource.txt"); + this.FileContentsShouldMatch("Test_ConflictTests", "ModifiedFiles", "ChangeInSourceDeleteInTarget.txt"); + this.FileContentsShouldMatch("Test_ConflictTests", "ModifiedFiles", "ChangeInTarget.txt"); + this.FileContentsShouldMatch("Test_ConflictTests", "ModifiedFiles", "ChangeInTargetDeleteInSource.txt"); + this.FileContentsShouldMatch("Test_ConflictTests", "ModifiedFiles", "ConflictingChange.txt"); + this.FileContentsShouldMatch("Test_ConflictTests", "ModifiedFiles", "SameChange.txt"); + this.FileContentsShouldMatch("Test_ConflictTests", "ModifiedFiles", "SuccessfulMerge.txt"); } } } diff --git a/GVFS/GVFS.FunctionalTests/Tests/GitCommands/HashObjectTests.cs b/GVFS/GVFS.FunctionalTests/Tests/GitCommands/HashObjectTests.cs index 3c2bc5d86..aca835817 100644 --- a/GVFS/GVFS.FunctionalTests/Tests/GitCommands/HashObjectTests.cs +++ b/GVFS/GVFS.FunctionalTests/Tests/GitCommands/HashObjectTests.cs @@ -5,7 +5,8 @@ namespace GVFS.FunctionalTests.Tests.GitCommands { [TestFixture] - [Category(Categories.Mac.M3)] + [Category(Categories.GitCommands)] + [Category(Categories.MacTODO.M3)] public class HashObjectTests : GitRepoTests { public HashObjectTests() : base(enlistmentPerTest: false) diff --git a/GVFS/GVFS.FunctionalTests/Tests/GitCommands/MergeConflictTests.cs b/GVFS/GVFS.FunctionalTests/Tests/GitCommands/MergeConflictTests.cs index 594d8aeb4..47472e030 100644 --- a/GVFS/GVFS.FunctionalTests/Tests/GitCommands/MergeConflictTests.cs +++ b/GVFS/GVFS.FunctionalTests/Tests/GitCommands/MergeConflictTests.cs @@ -6,6 +6,7 @@ namespace GVFS.FunctionalTests.Tests.GitCommands { [TestFixture] [Category(Categories.GitCommands)] + [Category(Categories.MacTODO.M3)] public class MergeConflictTests : GitRepoTests { public MergeConflictTests() : base(enlistmentPerTest: true) diff --git a/GVFS/GVFS.FunctionalTests/Tests/GitCommands/RebaseConflictTests.cs b/GVFS/GVFS.FunctionalTests/Tests/GitCommands/RebaseConflictTests.cs index 87ccee56f..2744913b1 100644 --- a/GVFS/GVFS.FunctionalTests/Tests/GitCommands/RebaseConflictTests.cs +++ b/GVFS/GVFS.FunctionalTests/Tests/GitCommands/RebaseConflictTests.cs @@ -4,6 +4,7 @@ namespace GVFS.FunctionalTests.Tests.GitCommands { [TestFixture] [Category(Categories.GitCommands)] + [Category(Categories.MacTODO.M3)] public class RebaseConflictTests : GitRepoTests { public RebaseConflictTests() : base(enlistmentPerTest: true) diff --git a/GVFS/GVFS.FunctionalTests/Tests/GitCommands/RebaseTests.cs b/GVFS/GVFS.FunctionalTests/Tests/GitCommands/RebaseTests.cs index e2c600920..72b1726d6 100644 --- a/GVFS/GVFS.FunctionalTests/Tests/GitCommands/RebaseTests.cs +++ b/GVFS/GVFS.FunctionalTests/Tests/GitCommands/RebaseTests.cs @@ -3,6 +3,8 @@ namespace GVFS.FunctionalTests.Tests.GitCommands { [TestFixture] + [Category(Categories.GitCommands)] + [Category(Categories.MacTODO.M3)] public class RebaseTests : GitRepoTests { public RebaseTests() : base(enlistmentPerTest: true) diff --git a/GVFS/GVFS.FunctionalTests/Tests/GitCommands/ResetHardTests.cs b/GVFS/GVFS.FunctionalTests/Tests/GitCommands/ResetHardTests.cs index 55ba679c9..8b3972b16 100644 --- a/GVFS/GVFS.FunctionalTests/Tests/GitCommands/ResetHardTests.cs +++ b/GVFS/GVFS.FunctionalTests/Tests/GitCommands/ResetHardTests.cs @@ -4,6 +4,8 @@ namespace GVFS.FunctionalTests.Tests.GitCommands { [TestFixture] + [Category(Categories.GitCommands)] + [Category(Categories.MacTODO.M3)] public class ResetHardTests : GitRepoTests { private const string ResetHardCommand = "reset --hard"; diff --git a/GVFS/GVFS.FunctionalTests/Tests/GitCommands/ResetMixedTests.cs b/GVFS/GVFS.FunctionalTests/Tests/GitCommands/ResetMixedTests.cs index d0b87606e..ed90a670a 100644 --- a/GVFS/GVFS.FunctionalTests/Tests/GitCommands/ResetMixedTests.cs +++ b/GVFS/GVFS.FunctionalTests/Tests/GitCommands/ResetMixedTests.cs @@ -5,6 +5,7 @@ namespace GVFS.FunctionalTests.Tests.GitCommands { [TestFixture] [Category(Categories.GitCommands)] + [Category(Categories.MacTODO.M3)] public class ResetMixedTests : GitRepoTests { public ResetMixedTests() : base(enlistmentPerTest: true) diff --git a/GVFS/GVFS.FunctionalTests/Tests/GitCommands/ResetSoftTests.cs b/GVFS/GVFS.FunctionalTests/Tests/GitCommands/ResetSoftTests.cs index 8aaed0884..01ce8336b 100644 --- a/GVFS/GVFS.FunctionalTests/Tests/GitCommands/ResetSoftTests.cs +++ b/GVFS/GVFS.FunctionalTests/Tests/GitCommands/ResetSoftTests.cs @@ -4,6 +4,7 @@ namespace GVFS.FunctionalTests.Tests.GitCommands { [TestFixture] [Category(Categories.GitCommands)] + [Category(Categories.MacTODO.M3)] public class ResetSoftTests : GitRepoTests { public ResetSoftTests() : base(enlistmentPerTest: true) diff --git a/GVFS/GVFS.FunctionalTests/Tests/GitCommands/RmTests.cs b/GVFS/GVFS.FunctionalTests/Tests/GitCommands/RmTests.cs index bfd9b026e..301cf4832 100644 --- a/GVFS/GVFS.FunctionalTests/Tests/GitCommands/RmTests.cs +++ b/GVFS/GVFS.FunctionalTests/Tests/GitCommands/RmTests.cs @@ -10,7 +10,10 @@ public RmTests() : base(enlistmentPerTest: false) { } + // Mac(TODO): Something is triggering Readme.md to get created on disk before this + // test validates that it's not present [TestCase] + [Category(Categories.MacTODO.M4)] public void CanReadFileAfterGitRmDryRun() { this.ValidateGitCommand("status"); diff --git a/GVFS/GVFS.FunctionalTests/Tests/GitCommands/StatusTests.cs b/GVFS/GVFS.FunctionalTests/Tests/GitCommands/StatusTests.cs index 9865431c3..116ebd700 100644 --- a/GVFS/GVFS.FunctionalTests/Tests/GitCommands/StatusTests.cs +++ b/GVFS/GVFS.FunctionalTests/Tests/GitCommands/StatusTests.cs @@ -24,6 +24,7 @@ public void MoveFileIntoDotGitDirectory() } [TestCase] + [Category(Categories.MacTODO.M4)] public void ModifyingAndDeletingRepositoryExcludeFileInvalidatesCache() { string repositoryExcludeFile = Path.Combine(".git", "info", "exclude"); @@ -31,7 +32,7 @@ public void ModifyingAndDeletingRepositoryExcludeFileInvalidatesCache() this.RepositoryIgnoreTestSetup(); // Add ignore pattern to existing exclude file - this.EditFile(repositoryExcludeFile, "*.ign"); + this.EditFile("*.ign", repositoryExcludeFile); // The exclude file has been modified, verify this status // excludes the "test.ign" file as expected. @@ -49,6 +50,7 @@ public void ModifyingAndDeletingRepositoryExcludeFileInvalidatesCache() } [TestCase] + [Category(Categories.MacTODO.M4)] public void NewRepositoryExcludeFileInvalidatesCache() { string repositoryExcludeFileRelativePath = Path.Combine(".git", "info", "exclude"); @@ -61,7 +63,7 @@ public void NewRepositoryExcludeFileInvalidatesCache() File.Exists(repositoryExcludeFilePath).ShouldBeFalse("Repository exclude path should not exist"); // Create new exclude file with ignore pattern - this.CreateFile(repositoryExcludeFileRelativePath, "*.ign"); + this.CreateFile("*.ign", repositoryExcludeFileRelativePath); // The exclude file has been modified, verify this status // excludes the "test.ign" file as expected. @@ -69,6 +71,7 @@ public void NewRepositoryExcludeFileInvalidatesCache() } [TestCase] + [Category(Categories.MacTODO.M4)] public void ModifyingHeadSymbolicRefInvalidatesCache() { this.ValidateGitCommand("status"); @@ -84,6 +87,7 @@ public void ModifyingHeadSymbolicRefInvalidatesCache() } [TestCase] + [Category(Categories.MacTODO.M4)] public void ModifyingHeadRefInvalidatesCache() { this.ValidateGitCommand("status"); @@ -104,7 +108,7 @@ private void RepositoryIgnoreTestSetup() File.Delete(statusCachePath); // Create a new file with an extension that will be ignored later in the test. - this.CreateFile("test.ign", "file to be ignored"); + this.CreateFile("file to be ignored", "test.ign"); this.WaitForStatusCacheToBeGenerated(); diff --git a/GVFS/GVFS.FunctionalTests/Tests/GitCommands/UpdateIndexTests.cs b/GVFS/GVFS.FunctionalTests/Tests/GitCommands/UpdateIndexTests.cs index 1ba2f46d5..943e54abb 100644 --- a/GVFS/GVFS.FunctionalTests/Tests/GitCommands/UpdateIndexTests.cs +++ b/GVFS/GVFS.FunctionalTests/Tests/GitCommands/UpdateIndexTests.cs @@ -40,6 +40,7 @@ public void UpdateIndexRemoveFileOnDiskDontCheckStatus() } [TestCase] + [Category(Categories.MacTODO.M4)] public void UpdateIndexRemoveAddFileOpenForWrite() { // TODO 940287: Remove this test and re-enable UpdateIndexRemoveFileOnDisk diff --git a/GVFS/GVFS.FunctionalTests/Tests/GitCommands/UpdateRefTests.cs b/GVFS/GVFS.FunctionalTests/Tests/GitCommands/UpdateRefTests.cs index 1a3dacec6..3ff64c14e 100644 --- a/GVFS/GVFS.FunctionalTests/Tests/GitCommands/UpdateRefTests.cs +++ b/GVFS/GVFS.FunctionalTests/Tests/GitCommands/UpdateRefTests.cs @@ -3,6 +3,8 @@ namespace GVFS.FunctionalTests.Tests.GitCommands { [TestFixture] + [Category(Categories.GitCommands)] + [Category(Categories.MacTODO.M4)] public class UpdateRefTests : GitRepoTests { public UpdateRefTests() : base(enlistmentPerTest: true) diff --git a/GVFS/GVFS.FunctionalTests/Tests/MultiEnlistmentTests/ServiceVerbTests.cs b/GVFS/GVFS.FunctionalTests/Tests/MultiEnlistmentTests/ServiceVerbTests.cs index 71d4976d3..4015f54e0 100644 --- a/GVFS/GVFS.FunctionalTests/Tests/MultiEnlistmentTests/ServiceVerbTests.cs +++ b/GVFS/GVFS.FunctionalTests/Tests/MultiEnlistmentTests/ServiceVerbTests.cs @@ -8,6 +8,7 @@ namespace GVFS.FunctionalTests.Tests.MultiEnlistmentTests [TestFixture] [NonParallelizable] [Category(Categories.FullSuiteOnly)] + [Category(Categories.MacTODO.M4)] public class ServiceVerbTests : TestsWithMultiEnlistment { private static readonly string[] EmptyRepoList = new string[] { }; diff --git a/GVFS/GVFS.FunctionalTests/Tests/MultiEnlistmentTests/SharedCacheTests.cs b/GVFS/GVFS.FunctionalTests/Tests/MultiEnlistmentTests/SharedCacheTests.cs index eb120e920..7499293e6 100644 --- a/GVFS/GVFS.FunctionalTests/Tests/MultiEnlistmentTests/SharedCacheTests.cs +++ b/GVFS/GVFS.FunctionalTests/Tests/MultiEnlistmentTests/SharedCacheTests.cs @@ -14,6 +14,7 @@ namespace GVFS.FunctionalTests.Tests.MultiEnlistmentTests { [TestFixture] [Category(Categories.FullSuiteOnly)] + [Category(Categories.MacTODO.M3)] public class SharedCacheTests : TestsWithMultiEnlistment { private const string WellKnownFile = "Readme.md"; diff --git a/GVFS/GVFS.Platform.Mac/MacFileSystemVirtualizer.cs b/GVFS/GVFS.Platform.Mac/MacFileSystemVirtualizer.cs index f358f07bb..0a3942116 100644 --- a/GVFS/GVFS.Platform.Mac/MacFileSystemVirtualizer.cs +++ b/GVFS/GVFS.Platform.Mac/MacFileSystemVirtualizer.cs @@ -81,11 +81,16 @@ public override FileSystemResult UpdatePlaceholderIfNeeded( out UpdateFailureReason failureReason) { UpdateFailureCause failureCause = UpdateFailureCause.NoFailure; + + // TODO(Mac): Add functional tests that include mode changes between commits + ushort fileMode = this.FileSystemCallbacks.GitIndexProjection.GetFilePathMode(relativePath); + Result result = this.virtualizationInstance.UpdatePlaceholderIfNeeded( relativePath, - GetPlaceholderVersionId(), - ConvertShaToContentId(shaContentId), + ToVersionIdByteArray(GetPlaceholderVersionId()), + ToVersionIdByteArray(ConvertShaToContentId(shaContentId)), (ulong)endOfFile, + fileMode, (UpdateType)updateFlags, out failureCause); failureReason = (UpdateFailureReason)failureCause; @@ -397,7 +402,7 @@ private Result OnEnumerateDirectory( projectedItems = this.FileSystemCallbacks.GitIndexProjection.GetProjectedItems(CancellationToken.None, blobSizesConnection, relativePath); } - result = this.CreateEnumerationPlaceholders(relativePath, projectedItems); + result = this.CreateEnumerationPlaceholders(relativePath, projectedItems, triggeringProcessName); } catch (SizesUnavailableException e) { @@ -422,31 +427,33 @@ private Result OnEnumerateDirectory( return Result.EIOError; } - private Result CreateEnumerationPlaceholders(string relativePath, IEnumerable projectedItems) + private Result CreateEnumerationPlaceholders(string relativePath, IEnumerable projectedItems, string triggeringProcessName) { foreach (ProjectedFileInfo fileInfo in projectedItems) { Result result; + string sha = null; + string fullRelativePath = Path.Combine(relativePath, fileInfo.Name); if (fileInfo.IsFolder) { - result = this.virtualizationInstance.WritePlaceholderDirectory(Path.Combine(relativePath, fileInfo.Name)); + result = this.virtualizationInstance.WritePlaceholderDirectory(fullRelativePath); } else { // TODO(Mac): Add functional tests that validate file mode is set correctly - string filePath = Path.Combine(relativePath, fileInfo.Name); - ushort fileMode = this.FileSystemCallbacks.GitIndexProjection.GetFilePathMode(filePath); + ushort fileMode = this.FileSystemCallbacks.GitIndexProjection.GetFilePathMode(fullRelativePath); + sha = fileInfo.Sha.ToString(); result = this.virtualizationInstance.WritePlaceholderFile( - filePath, + fullRelativePath, ToVersionIdByteArray(FileSystemVirtualizer.GetPlaceholderVersionId()), - ToVersionIdByteArray(FileSystemVirtualizer.ConvertShaToContentId(fileInfo.Sha.ToString())), + ToVersionIdByteArray(FileSystemVirtualizer.ConvertShaToContentId(sha)), (ulong)fileInfo.Size, fileMode); } if (result != Result.Success) { - EventMetadata metadata = this.CreateEventMetadata(relativePath); + EventMetadata metadata = this.CreateEventMetadata(fullRelativePath); metadata.Add("fileInfo.Name", fileInfo.Name); metadata.Add("fileInfo.Size", fileInfo.Size); metadata.Add("fileInfo.IsFolder", fileInfo.IsFolder); @@ -454,6 +461,17 @@ private Result CreateEnumerationPlaceholders(string relativePath, IEnumerable Date: Wed, 29 Aug 2018 10:59:35 -0700 Subject: [PATCH 077/272] Fix StyleCop errors --- .../CaseOnlyFolderRenameTests.cs | 22 +++++++++---------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/GVFS/GVFS.FunctionalTests/Tests/EnlistmentPerTestCase/CaseOnlyFolderRenameTests.cs b/GVFS/GVFS.FunctionalTests/Tests/EnlistmentPerTestCase/CaseOnlyFolderRenameTests.cs index d5fb1e49a..cca045f73 100644 --- a/GVFS/GVFS.FunctionalTests/Tests/EnlistmentPerTestCase/CaseOnlyFolderRenameTests.cs +++ b/GVFS/GVFS.FunctionalTests/Tests/EnlistmentPerTestCase/CaseOnlyFolderRenameTests.cs @@ -30,11 +30,11 @@ public void CaseRenameFoldersAndRemountAndRenameAgain() string newGVFSSubFolderName = "gvfs"; string newGVFSSubFolderPath = Path.Combine(parentFolderName, newGVFSSubFolderName); - this.Enlistment.GetVirtualPathTo(oldGVFSSubFolderPath).ShouldBeADirectory(fileSystem).WithCaseMatchingName(oldGVFSSubFolderName); + this.Enlistment.GetVirtualPathTo(oldGVFSSubFolderPath).ShouldBeADirectory(this.fileSystem).WithCaseMatchingName(oldGVFSSubFolderName); this.fileSystem.MoveFile(this.Enlistment.GetVirtualPathTo(oldGVFSSubFolderPath), this.Enlistment.GetVirtualPathTo(newGVFSSubFolderPath)); - this.Enlistment.GetVirtualPathTo(newGVFSSubFolderPath).ShouldBeADirectory(fileSystem).WithCaseMatchingName(newGVFSSubFolderName); + this.Enlistment.GetVirtualPathTo(newGVFSSubFolderPath).ShouldBeADirectory(this.fileSystem).WithCaseMatchingName(newGVFSSubFolderName); // Projected folder with a physical folder string oldTestsSubFolderName = "GVFS.FunctionalTests"; @@ -45,33 +45,33 @@ public void CaseRenameFoldersAndRemountAndRenameAgain() string fileToAdd = "NewFile.txt"; string fileToAddContent = "This is new file text."; string fileToAddPath = this.Enlistment.GetVirtualPathTo(Path.Combine(oldTestsSubFolderPath, fileToAdd)); - fileSystem.WriteAllText(fileToAddPath, fileToAddContent); + this.fileSystem.WriteAllText(fileToAddPath, fileToAddContent); - this.Enlistment.GetVirtualPathTo(oldTestsSubFolderPath).ShouldBeADirectory(fileSystem).WithCaseMatchingName(oldTestsSubFolderName); + this.Enlistment.GetVirtualPathTo(oldTestsSubFolderPath).ShouldBeADirectory(this.fileSystem).WithCaseMatchingName(oldTestsSubFolderName); this.fileSystem.MoveFile(this.Enlistment.GetVirtualPathTo(oldTestsSubFolderPath), this.Enlistment.GetVirtualPathTo(newTestsSubFolderPath)); - this.Enlistment.GetVirtualPathTo(newTestsSubFolderPath).ShouldBeADirectory(fileSystem).WithCaseMatchingName(newTestsSubFolderName); + this.Enlistment.GetVirtualPathTo(newTestsSubFolderPath).ShouldBeADirectory(this.fileSystem).WithCaseMatchingName(newTestsSubFolderName); // Remount this.Enlistment.UnmountGVFS(); this.Enlistment.MountGVFS(); - this.Enlistment.GetVirtualPathTo(newGVFSSubFolderPath).ShouldBeADirectory(fileSystem).WithCaseMatchingName(newGVFSSubFolderName); - this.Enlistment.GetVirtualPathTo(newTestsSubFolderPath).ShouldBeADirectory(fileSystem).WithCaseMatchingName(newTestsSubFolderName); - this.Enlistment.GetVirtualPathTo(Path.Combine(newTestsSubFolderPath, fileToAdd)).ShouldBeAFile(fileSystem).WithContents().ShouldEqual(fileToAddContent); + this.Enlistment.GetVirtualPathTo(newGVFSSubFolderPath).ShouldBeADirectory(this.fileSystem).WithCaseMatchingName(newGVFSSubFolderName); + this.Enlistment.GetVirtualPathTo(newTestsSubFolderPath).ShouldBeADirectory(this.fileSystem).WithCaseMatchingName(newTestsSubFolderName); + this.Enlistment.GetVirtualPathTo(Path.Combine(newTestsSubFolderPath, fileToAdd)).ShouldBeAFile(this.fileSystem).WithContents().ShouldEqual(fileToAddContent); // Rename each folder again string finalGVFSSubFolderName = "gvFS"; string finalGVFSSubFolderPath = Path.Combine(parentFolderName, finalGVFSSubFolderName); this.fileSystem.MoveFile(this.Enlistment.GetVirtualPathTo(newGVFSSubFolderPath), this.Enlistment.GetVirtualPathTo(finalGVFSSubFolderPath)); - this.Enlistment.GetVirtualPathTo(finalGVFSSubFolderPath).ShouldBeADirectory(fileSystem).WithCaseMatchingName(finalGVFSSubFolderName); + this.Enlistment.GetVirtualPathTo(finalGVFSSubFolderPath).ShouldBeADirectory(this.fileSystem).WithCaseMatchingName(finalGVFSSubFolderName); string finalTestsSubFolderName = "gvfs.FunctionalTESTS"; string finalTestsSubFolderPath = Path.Combine(parentFolderName, finalTestsSubFolderName); this.fileSystem.MoveFile(this.Enlistment.GetVirtualPathTo(newTestsSubFolderPath), this.Enlistment.GetVirtualPathTo(finalTestsSubFolderPath)); - this.Enlistment.GetVirtualPathTo(finalTestsSubFolderPath).ShouldBeADirectory(fileSystem).WithCaseMatchingName(finalTestsSubFolderName); - this.Enlistment.GetVirtualPathTo(Path.Combine(finalTestsSubFolderPath, fileToAdd)).ShouldBeAFile(fileSystem).WithContents().ShouldEqual(fileToAddContent); + this.Enlistment.GetVirtualPathTo(finalTestsSubFolderPath).ShouldBeADirectory(this.fileSystem).WithCaseMatchingName(finalTestsSubFolderName); + this.Enlistment.GetVirtualPathTo(Path.Combine(finalTestsSubFolderPath, fileToAdd)).ShouldBeAFile(this.fileSystem).WithContents().ShouldEqual(fileToAddContent); } } } From d58b8b79b5c5ace2f923654181370d232bc181b4 Mon Sep 17 00:00:00 2001 From: Kevin Willford Date: Tue, 28 Aug 2018 11:10:18 -0600 Subject: [PATCH 078/272] Add enlistment id and mount id to data for ETW --- GVFS/FastFetch/FastFetchVerb.cs | 2 +- GVFS/GVFS.Common/GVFSPlatform.cs | 2 +- GVFS/GVFS.Common/Tracing/JsonTracer.cs | 13 ++++--- GVFS/GVFS.Mount/InProcessMountVerb.cs | 19 +++++++++- GVFS/GVFS.Platform.Mac/MacPlatform.cs | 2 +- .../ETWTelemetryEventListener.cs | 36 ++++++++++++++----- GVFS/GVFS.Platform.Windows/WindowsPlatform.cs | 6 ++-- .../Mock/Common/MockPlatform.cs | 2 +- 8 files changed, 62 insertions(+), 20 deletions(-) diff --git a/GVFS/FastFetch/FastFetchVerb.cs b/GVFS/FastFetch/FastFetchVerb.cs index 82cf59fcc..cd37e371b 100644 --- a/GVFS/FastFetch/FastFetchVerb.cs +++ b/GVFS/FastFetch/FastFetchVerb.cs @@ -206,7 +206,7 @@ private int ExecuteWithExitCode() Console.WriteLine("The ParentActivityId provided (" + this.ParentActivityId + ") is not a valid GUID."); } - using (JsonTracer tracer = new JsonTracer("Microsoft.Git.FastFetch", parentActivityId, "FastFetch", disableTelemetry: true)) + using (JsonTracer tracer = new JsonTracer("Microsoft.Git.FastFetch", parentActivityId, "FastFetch", enlistmentId: null, mountId: null, disableTelemetry: true)) { if (this.Verbose) { diff --git a/GVFS/GVFS.Common/GVFSPlatform.cs b/GVFS/GVFS.Common/GVFSPlatform.cs index acb0aa2f3..2297a75d9 100644 --- a/GVFS/GVFS.Common/GVFSPlatform.cs +++ b/GVFS/GVFS.Common/GVFSPlatform.cs @@ -50,7 +50,7 @@ public static void Register(GVFSPlatform platform) public abstract bool TryGetGVFSHooksPathAndVersion(out string hooksPaths, out string hooksVersion, out string error); public abstract bool TryInstallGitCommandHooks(GVFSContext context, string executingDirectory, string hookName, string commandHookPath, out string errorMessage); - public abstract InProcEventListener CreateTelemetryListenerIfEnabled(string providerName); + public abstract InProcEventListener CreateTelemetryListenerIfEnabled(string providerName, string enlistmentId, string mountId); public abstract Dictionary GetPhysicalDiskInfo(string path); diff --git a/GVFS/GVFS.Common/Tracing/JsonTracer.cs b/GVFS/GVFS.Common/Tracing/JsonTracer.cs index aad25dcd8..07e424b23 100644 --- a/GVFS/GVFS.Common/Tracing/JsonTracer.cs +++ b/GVFS/GVFS.Common/Tracing/JsonTracer.cs @@ -20,13 +20,17 @@ public class JsonTracer : ITracer private EventLevel startStopLevel; private Keywords startStopKeywords; - public JsonTracer(string providerName, string activityName, bool disableTelemetry = false) - : this(providerName, Guid.Empty, activityName, disableTelemetry) + : this(providerName, Guid.Empty, activityName, enlistmentId: null, mountId: null, disableTelemetry: disableTelemetry) { } - public JsonTracer(string providerName, Guid providerActivityId, string activityName, bool disableTelemetry = false) + public JsonTracer(string providerName, string activityName, string enlistmentId, string mountId, bool disableTelemetry = false) + : this(providerName, Guid.Empty, activityName, enlistmentId, mountId, disableTelemetry) + { + } + + public JsonTracer(string providerName, Guid providerActivityId, string activityName, string enlistmentId, string mountId, bool disableTelemetry = false) : this( new List(), providerActivityId, @@ -36,7 +40,7 @@ public JsonTracer(string providerName, Guid providerActivityId, string activityN { if (!disableTelemetry) { - InProcEventListener telemetryListener = GVFSPlatform.Instance.CreateTelemetryListenerIfEnabled(providerName); + InProcEventListener telemetryListener = GVFSPlatform.Instance.CreateTelemetryListenerIfEnabled(providerName, enlistmentId, mountId); if (telemetryListener != null) { this.listeners.Add(telemetryListener); @@ -252,7 +256,6 @@ private static string GetCategorizedErrorEventName(Keywords keywords) private void WriteEvent(string eventName, EventLevel level, Keywords keywords, EventMetadata metadata, EventOpcode opcode) { string jsonPayload = metadata != null ? JsonConvert.SerializeObject(metadata) : null; - foreach (InProcEventListener listener in this.listeners) { listener.RecordMessage(eventName, this.activityId, this.parentActivityId, level, keywords, opcode, jsonPayload); diff --git a/GVFS/GVFS.Mount/InProcessMountVerb.cs b/GVFS/GVFS.Mount/InProcessMountVerb.cs index b1c1e19ce..71e835004 100644 --- a/GVFS/GVFS.Mount/InProcessMountVerb.cs +++ b/GVFS/GVFS.Mount/InProcessMountVerb.cs @@ -1,5 +1,6 @@ using CommandLine; using GVFS.Common; +using GVFS.Common.Git; using GVFS.Common.Http; using GVFS.Common.Tracing; using System; @@ -124,7 +125,23 @@ private void UnhandledGVFSExceptionHandler(ITracer tracer, object sender, Unhand private JsonTracer CreateTracer(GVFSEnlistment enlistment, EventLevel verbosity, Keywords keywords) { - JsonTracer tracer = new JsonTracer(GVFSConstants.GVFSEtwProviderName, "GVFSMount"); + string enlistmentId = null; + string mountId = null; + + GitProcess git = new GitProcess(enlistment); + GitProcess.Result configResult = git.GetFromLocalConfig(GVFSConstants.GitConfig.EnlistmentId); + if (!configResult.HasErrors) + { + enlistmentId = configResult.Output.Trim(); + } + + configResult = git.GetFromLocalConfig(GVFSConstants.GitConfig.MountId); + if (!configResult.HasErrors) + { + mountId = configResult.Output.Trim(); + } + + JsonTracer tracer = new JsonTracer(GVFSConstants.GVFSEtwProviderName, "GVFSMount", enlistmentId: enlistmentId, mountId: mountId); tracer.AddLogFileEventListener( GVFSEnlistment.GetNewGVFSLogFileName(enlistment.GVFSLogsRoot, GVFSConstants.LogFileTypes.MountProcess), verbosity, diff --git a/GVFS/GVFS.Platform.Mac/MacPlatform.cs b/GVFS/GVFS.Platform.Mac/MacPlatform.cs index c5683de6b..9be9a5ed9 100644 --- a/GVFS/GVFS.Platform.Mac/MacPlatform.cs +++ b/GVFS/GVFS.Platform.Mac/MacPlatform.cs @@ -75,7 +75,7 @@ public override NamedPipeServerStream CreatePipeByName(string pipeName) return pipe; } - public override InProcEventListener CreateTelemetryListenerIfEnabled(string providerName) + public override InProcEventListener CreateTelemetryListenerIfEnabled(string providerName, string enlistmentId, string mountId) { return null; } diff --git a/GVFS/GVFS.Platform.Windows/ETWTelemetryEventListener.cs b/GVFS/GVFS.Platform.Windows/ETWTelemetryEventListener.cs index 9f4140867..3ca1adcd5 100644 --- a/GVFS/GVFS.Platform.Windows/ETWTelemetryEventListener.cs +++ b/GVFS/GVFS.Platform.Windows/ETWTelemetryEventListener.cs @@ -32,14 +32,18 @@ public class ETWTelemetryEventListener : InProcEventListener private const long MeasureKeyword = 0x400000000000; private EventSource eventSource; + private string enlistmentId; + private string mountId; - private ETWTelemetryEventListener(string providerName, string[] traitsList) + private ETWTelemetryEventListener(string providerName, string[] traitsList, string enlistmentId, string mountId) : base(EventLevel.Verbose, Keywords.Telemetry) { this.eventSource = new EventSource(providerName, EventSourceSettings.EtwSelfDescribingEventFormat, traitsList); + this.enlistmentId = enlistmentId; + this.mountId = mountId; } - public static ETWTelemetryEventListener CreateTelemetryListenerIfEnabled(string gitBinRoot, string providerName) + public static ETWTelemetryEventListener CreateTelemetryListenerIfEnabled(string gitBinRoot, string providerName, string enlistmentId, string mountId) { // This listener is disabled unless the user specifies the proper git config setting. @@ -52,7 +56,7 @@ public static ETWTelemetryEventListener CreateTelemetryListenerIfEnabled(string if (!result.HasErrors && !string.IsNullOrEmpty(result.Output.TrimEnd('\r', '\n'))) { string[] traitsList = result.Output.TrimEnd('\r', '\n').Split('|'); - return new ETWTelemetryEventListener(providerName, traitsList); + return new ETWTelemetryEventListener(providerName, traitsList, enlistmentId, mountId); } else { @@ -83,12 +87,12 @@ protected override void RecordMessageInternal( if (jsonPayload != null) { - JsonPayload payload = new JsonPayload(jsonPayload); + JsonPayload payload = new JsonPayload(jsonPayload, this.enlistmentId, this.mountId); this.eventSource.Write(eventName, ref options, ref activityId, ref parentActivityId, ref payload); } else { - EmptyStruct payload = new EmptyStruct(); + Payload payload = new Payload(this.enlistmentId, this.mountId); this.eventSource.Write(eventName, ref options, ref activityId, ref parentActivityId, ref payload); } } @@ -105,22 +109,38 @@ private EventSourceOptions CreateOptions(EventLevel level, Keywords keywords, Ev return options; } - // Needed to pass relatedId without metadata [EventData] - public struct EmptyStruct + public struct Payload { + public Payload(string enlistmentId, string mountId) + { + this.EnlistmentId = enlistmentId; + this.MountId = mountId; + } + + [EventField] + public string EnlistmentId { get; } + [EventField] + public string MountId { get; } } [EventData] public struct JsonPayload { - public JsonPayload(string payload) + public JsonPayload(string payload, string enlistmentId, string mountId) { this.Json = payload; + this.EnlistmentId = enlistmentId; + this.MountId = mountId; } [EventField] public string Json { get; } + + [EventField] + public string EnlistmentId { get; } + [EventField] + public string MountId { get; } } } } diff --git a/GVFS/GVFS.Platform.Windows/WindowsPlatform.cs b/GVFS/GVFS.Platform.Windows/WindowsPlatform.cs index 80dde743b..aa8c3953b 100644 --- a/GVFS/GVFS.Platform.Windows/WindowsPlatform.cs +++ b/GVFS/GVFS.Platform.Windows/WindowsPlatform.cs @@ -69,11 +69,13 @@ public static bool TrySetDWordInRegistry(RegistryHive registryHive, string key, return true; } - public override InProcEventListener CreateTelemetryListenerIfEnabled(string providerName) + public override InProcEventListener CreateTelemetryListenerIfEnabled(string providerName, string enlistmentId, string mountId) { return ETWTelemetryEventListener.CreateTelemetryListenerIfEnabled( this.GitInstallation.GetInstalledGitBinPath(), - providerName); + providerName, + enlistmentId, + mountId); } public override void InitializeEnlistmentACLs(string enlistmentPath) diff --git a/GVFS/GVFS.UnitTests/Mock/Common/MockPlatform.cs b/GVFS/GVFS.UnitTests/Mock/Common/MockPlatform.cs index 0062d7e08..b0cd6d43f 100644 --- a/GVFS/GVFS.UnitTests/Mock/Common/MockPlatform.cs +++ b/GVFS/GVFS.UnitTests/Mock/Common/MockPlatform.cs @@ -50,7 +50,7 @@ public override NamedPipeServerStream CreatePipeByName(string pipeName) throw new NotSupportedException(); } - public override InProcEventListener CreateTelemetryListenerIfEnabled(string providerName) + public override InProcEventListener CreateTelemetryListenerIfEnabled(string providerName, string enlistmentId, string mountId) { return new MockListener(EventLevel.Verbose, Keywords.Telemetry); } From aaaf05752c63edef264fe667ea82c6b927209120 Mon Sep 17 00:00:00 2001 From: William Baker Date: Wed, 29 Aug 2018 12:19:24 -0700 Subject: [PATCH 079/272] Fix failing functional test --- GVFS/GVFS.FunctionalTests/Tests/GitCommands/GitCommandsTests.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/GVFS/GVFS.FunctionalTests/Tests/GitCommands/GitCommandsTests.cs b/GVFS/GVFS.FunctionalTests/Tests/GitCommands/GitCommandsTests.cs index 3a550d801..40f4cd675 100644 --- a/GVFS/GVFS.FunctionalTests/Tests/GitCommands/GitCommandsTests.cs +++ b/GVFS/GVFS.FunctionalTests/Tests/GitCommands/GitCommandsTests.cs @@ -315,7 +315,7 @@ public void AddFileAndCommitOnNewBranchSwitchDeleteFolderAndSwitchBack() { // 663045 - Confirm that folder can be deleted after adding a file then changing branches string newFileParentFolderPath = Path.Combine("GVFS", "GVFS", "CommandLine"); - string newFilePath = Path.Combine(newFileParentFolderPath + "testfile.txt"); + string newFilePath = Path.Combine(newFileParentFolderPath, "testfile.txt"); string newFileContents = "test contents"; this.CommitChangesSwitchBranch( From fa6143f0fef38197b130f3d9a46f0747305b760f Mon Sep 17 00:00:00 2001 From: William Baker Date: Wed, 29 Aug 2018 14:03:10 -0700 Subject: [PATCH 080/272] Create FailsOnBuildAgent category and tag EditFileNeedingUtf8Encoding with that category --- GVFS/GVFS.FunctionalTests/Categories.cs | 4 ++++ .../Tests/EnlistmentPerFixture/WorkingDirectoryTests.cs | 3 +-- .../Tests/GitCommands/GitCommandsTests.cs | 1 + 3 files changed, 6 insertions(+), 2 deletions(-) diff --git a/GVFS/GVFS.FunctionalTests/Categories.cs b/GVFS/GVFS.FunctionalTests/Categories.cs index 328145b02..fe837c42b 100644 --- a/GVFS/GVFS.FunctionalTests/Categories.cs +++ b/GVFS/GVFS.FunctionalTests/Categories.cs @@ -11,6 +11,10 @@ public static class Categories public static class MacTODO { + // The FailsOnBuildAgent category is for tests that pass on dev + // machines but not on the build agents + public const string FailsOnBuildAgent = "FailsOnBuildAgent"; + public const string NeedsLockHolder = "NeedsDotCoreLockHolder"; public const string M2 = "M2_StaticViewGitCommands"; public const string M3 = "M3_AllGitCommands"; diff --git a/GVFS/GVFS.FunctionalTests/Tests/EnlistmentPerFixture/WorkingDirectoryTests.cs b/GVFS/GVFS.FunctionalTests/Tests/EnlistmentPerFixture/WorkingDirectoryTests.cs index 2e32f9dd2..d5088205e 100644 --- a/GVFS/GVFS.FunctionalTests/Tests/EnlistmentPerFixture/WorkingDirectoryTests.cs +++ b/GVFS/GVFS.FunctionalTests/Tests/EnlistmentPerFixture/WorkingDirectoryTests.cs @@ -447,9 +447,8 @@ public void FolderContentsCorrectAfterCreateNewFolderRenameAndCheckoutCommitWith (folder + @"\MoveUnhydratedFileToDotGitFolder\Program.cs").ShouldBeAFile(this.fileSystem).WithContents(MoveRenameFileTests.TestFileContents); } - // TODO(Mac) This *should* be working already, we need further investigation of why this test fails on build agents, but not on dev machines. [TestCase, Order(15)] - [Category(Categories.MacTODO.M2)] + [Category(Categories.MacTODO.FailsOnBuildAgent)] public void FilterNonUTF8FileName() { string encodingFilename = "ريلٌأكتوبرûمارسأغسطسºٰٰۂْٗ۵ريلٌأك.txt"; diff --git a/GVFS/GVFS.FunctionalTests/Tests/GitCommands/GitCommandsTests.cs b/GVFS/GVFS.FunctionalTests/Tests/GitCommands/GitCommandsTests.cs index 40f4cd675..509cfb03f 100644 --- a/GVFS/GVFS.FunctionalTests/Tests/GitCommands/GitCommandsTests.cs +++ b/GVFS/GVFS.FunctionalTests/Tests/GitCommands/GitCommandsTests.cs @@ -969,6 +969,7 @@ public void OpenFileThenCheckout() } [TestCase] + [Category(Categories.MacTODO.FailsOnBuildAgent)] public void EditFileNeedingUtf8Encoding() { this.ValidateGitCommand("checkout -b tests/functional/EditFileNeedingUtf8Encoding"); From 9b29bf0514057d4b1e0ad3a2643d91c419f957d1 Mon Sep 17 00:00:00 2001 From: William Baker Date: Wed, 29 Aug 2018 15:39:08 -0700 Subject: [PATCH 081/272] Changes for PR Feedback --- .../CaseOnlyFolderRenameTests.cs | 1 - .../MacFileSystemVirtualizer.cs | 24 ++++++++++--------- ProjFS.Mac/PrjFSLib.Mac.Managed/UpdateType.cs | 1 - ProjFS.Mac/PrjFSLib/PrjFSLib.cpp | 3 +++ ProjFS.Mac/PrjFSLib/PrjFSLib.h | 1 - 5 files changed, 16 insertions(+), 14 deletions(-) diff --git a/GVFS/GVFS.FunctionalTests/Tests/EnlistmentPerTestCase/CaseOnlyFolderRenameTests.cs b/GVFS/GVFS.FunctionalTests/Tests/EnlistmentPerTestCase/CaseOnlyFolderRenameTests.cs index cca045f73..3c4a9fc0c 100644 --- a/GVFS/GVFS.FunctionalTests/Tests/EnlistmentPerTestCase/CaseOnlyFolderRenameTests.cs +++ b/GVFS/GVFS.FunctionalTests/Tests/EnlistmentPerTestCase/CaseOnlyFolderRenameTests.cs @@ -13,7 +13,6 @@ public class CaseOnlyFolderRenameTests : TestsWithEnlistmentPerTestCase private FileSystemRunner fileSystem; public CaseOnlyFolderRenameTests() - : base() { this.fileSystem = new BashRunner(); } diff --git a/GVFS/GVFS.Platform.Mac/MacFileSystemVirtualizer.cs b/GVFS/GVFS.Platform.Mac/MacFileSystemVirtualizer.cs index 0a3942116..4c90f4b6a 100644 --- a/GVFS/GVFS.Platform.Mac/MacFileSystemVirtualizer.cs +++ b/GVFS/GVFS.Platform.Mac/MacFileSystemVirtualizer.cs @@ -82,7 +82,9 @@ public override FileSystemResult UpdatePlaceholderIfNeeded( { UpdateFailureCause failureCause = UpdateFailureCause.NoFailure; - // TODO(Mac): Add functional tests that include mode changes between commits + // TODO(Mac): Add functional tests that include: + // - Mode + content changes between commits + // - Mode only changes (without any change to content, see issue #223) ushort fileMode = this.FileSystemCallbacks.GitIndexProjection.GetFilePathMode(relativePath); Result result = this.virtualizationInstance.UpdatePlaceholderIfNeeded( @@ -402,7 +404,7 @@ private Result OnEnumerateDirectory( projectedItems = this.FileSystemCallbacks.GitIndexProjection.GetProjectedItems(CancellationToken.None, blobSizesConnection, relativePath); } - result = this.CreateEnumerationPlaceholders(relativePath, projectedItems, triggeringProcessName); + result = this.CreatePlaceholders(relativePath, projectedItems, triggeringProcessName); } catch (SizesUnavailableException e) { @@ -427,24 +429,24 @@ private Result OnEnumerateDirectory( return Result.EIOError; } - private Result CreateEnumerationPlaceholders(string relativePath, IEnumerable projectedItems, string triggeringProcessName) + private Result CreatePlaceholders(string directoryRelativePath, IEnumerable projectedItems, string triggeringProcessName) { foreach (ProjectedFileInfo fileInfo in projectedItems) { Result result; string sha = null; - string fullRelativePath = Path.Combine(relativePath, fileInfo.Name); + string childRelativePath = Path.Combine(directoryRelativePath, fileInfo.Name); if (fileInfo.IsFolder) { - result = this.virtualizationInstance.WritePlaceholderDirectory(fullRelativePath); + result = this.virtualizationInstance.WritePlaceholderDirectory(childRelativePath); } else { // TODO(Mac): Add functional tests that validate file mode is set correctly - ushort fileMode = this.FileSystemCallbacks.GitIndexProjection.GetFilePathMode(fullRelativePath); + ushort fileMode = this.FileSystemCallbacks.GitIndexProjection.GetFilePathMode(childRelativePath); sha = fileInfo.Sha.ToString(); result = this.virtualizationInstance.WritePlaceholderFile( - fullRelativePath, + childRelativePath, ToVersionIdByteArray(FileSystemVirtualizer.GetPlaceholderVersionId()), ToVersionIdByteArray(FileSystemVirtualizer.ConvertShaToContentId(sha)), (ulong)fileInfo.Size, @@ -453,11 +455,11 @@ private Result CreateEnumerationPlaceholders(string relativePath, IEnumerable Date: Wed, 29 Aug 2018 15:46:19 -0700 Subject: [PATCH 082/272] Exclude Categories.MacTODO.FailsOnBuildAgent tests on Mac --- GVFS/GVFS.FunctionalTests/Program.cs | 1 + 1 file changed, 1 insertion(+) diff --git a/GVFS/GVFS.FunctionalTests/Program.cs b/GVFS/GVFS.FunctionalTests/Program.cs index add16f3b3..b467b28d5 100644 --- a/GVFS/GVFS.FunctionalTests/Program.cs +++ b/GVFS/GVFS.FunctionalTests/Program.cs @@ -66,6 +66,7 @@ public static void Main(string[] args) if (RuntimeInformation.IsOSPlatform(OSPlatform.OSX)) { excludeCategories.Add(Categories.MacTODO.NeedsLockHolder); + excludeCategories.Add(Categories.MacTODO.FailsOnBuildAgent); excludeCategories.Add(Categories.MacTODO.M2); excludeCategories.Add(Categories.MacTODO.M3); excludeCategories.Add(Categories.MacTODO.M4); From b206f3ea4c58722272a7ea4a585af97d20d37e21 Mon Sep 17 00:00:00 2001 From: William Baker Date: Wed, 29 Aug 2018 15:58:33 -0700 Subject: [PATCH 083/272] Fix fileSystemCallbacks bug in UpdatePlaceholderIfNeeded unit test --- .../GVFS.UnitTests/Platform.Mac/MacFileSystemVirtualizerTests.cs | 1 + 1 file changed, 1 insertion(+) diff --git a/GVFS/GVFS.UnitTests/Platform.Mac/MacFileSystemVirtualizerTests.cs b/GVFS/GVFS.UnitTests/Platform.Mac/MacFileSystemVirtualizerTests.cs index e6c6535ef..840c0177f 100644 --- a/GVFS/GVFS.UnitTests/Platform.Mac/MacFileSystemVirtualizerTests.cs +++ b/GVFS/GVFS.UnitTests/Platform.Mac/MacFileSystemVirtualizerTests.cs @@ -156,6 +156,7 @@ public void UpdatePlaceholderIfNeeded() out failureReason) .ShouldEqual(new FileSystemResult(FSResult.IOError, (int)mockVirtualization.UpdatePlaceholderIfNeededResult)); failureReason.ShouldEqual((UpdateFailureReason)mockVirtualization.UpdatePlaceholderIfNeededFailureCause); + fileSystemCallbacks.Stop(); } } From 1732df95d813741484e397d4d4473ae04df43a6e Mon Sep 17 00:00:00 2001 From: William Baker Date: Thu, 30 Aug 2018 13:56:22 -0700 Subject: [PATCH 084/272] More changes for PR feedback, and fix erroneous logging in kext --- .../MacFileSystemVirtualizer.cs | 6 ++++-- .../WindowsFileSystemVirtualizer.cs | 16 +++++++++++++--- .../WindowsFileSystemVirtualizerTests.cs | 10 +++++----- .../MacFileSystemVirtualizerTests.cs | 4 ++-- .../FileSystem/FileSystemVirtualizer.cs | 10 ---------- ProjFS.Mac/PrjFSKext/PrjFSKext/KauthHandler.cpp | 1 + ProjFS.Mac/PrjFSLib/PrjFSLib.cpp | 2 +- 7 files changed, 26 insertions(+), 23 deletions(-) diff --git a/GVFS/GVFS.Platform.Mac/MacFileSystemVirtualizer.cs b/GVFS/GVFS.Platform.Mac/MacFileSystemVirtualizer.cs index 4c90f4b6a..29607f276 100644 --- a/GVFS/GVFS.Platform.Mac/MacFileSystemVirtualizer.cs +++ b/GVFS/GVFS.Platform.Mac/MacFileSystemVirtualizer.cs @@ -14,6 +14,8 @@ namespace GVFS.Platform.Mac { public class MacFileSystemVirtualizer : FileSystemVirtualizer { + public static readonly byte[] PlaceholderVersionId = ToVersionIdByteArray(new byte[] { PlaceholderVersion }); + private VirtualizationInstance virtualizationInstance; public MacFileSystemVirtualizer(GVFSContext context, GVFSGitObjects gitObjects) @@ -89,7 +91,7 @@ public override FileSystemResult UpdatePlaceholderIfNeeded( Result result = this.virtualizationInstance.UpdatePlaceholderIfNeeded( relativePath, - ToVersionIdByteArray(GetPlaceholderVersionId()), + PlaceholderVersionId, ToVersionIdByteArray(ConvertShaToContentId(shaContentId)), (ulong)endOfFile, fileMode, @@ -447,7 +449,7 @@ private Result CreatePlaceholders(string directoryRelativePath, IEnumerable + /// GVFS uses the first byte of the providerId field of placeholders to version + /// the data that it stores in the contentId (and providerId) fields of the placeholder + /// + public static readonly byte[] PlaceholderVersionId = new byte[] { PlaceholderVersion }; + private const string ClassName = nameof(WindowsFileSystemVirtualizer); private const int MaxBlobStreamBufferSize = 64 * 1024; private const int MinPrjLibThreads = 5; @@ -131,7 +137,7 @@ public override FileSystemResult UpdatePlaceholderIfNeeded( fileAttributes, endOfFile, ConvertShaToContentId(shaContentId), - GetPlaceholderVersionId(), + PlaceholderVersionId, (UpdateType)updateFlags, out failureCause); failureReason = (UpdateFailureReason)failureCause; @@ -757,7 +763,7 @@ private void GetPlaceholderInformationAsyncHandler( endOfFile: fileInfo.Size, isDirectory: fileInfo.IsFolder, contentId: FileSystemVirtualizer.ConvertShaToContentId(sha), - providerId: FileSystemVirtualizer.GetPlaceholderVersionId()); + providerId: PlaceholderVersionId); if (result != HResult.Ok) { @@ -1094,7 +1100,11 @@ private void NotifyNewFileCreatedHandler( if (gitCommand.IsValidGitCommand) { string directoryPath = Path.Combine(this.Context.Enlistment.WorkingDirectoryRoot, virtualPath); - HResult hr = this.virtualizationInstance.ConvertDirectoryToPlaceholder(directoryPath, ConvertShaToContentId(GVFSConstants.AllZeroSha), GetPlaceholderVersionId()); + HResult hr = this.virtualizationInstance.ConvertDirectoryToPlaceholder( + directoryPath, + ConvertShaToContentId(GVFSConstants.AllZeroSha), + PlaceholderVersionId); + if (hr == HResult.Ok) { this.FileSystemCallbacks.OnPlaceholderFolderCreated(virtualPath); diff --git a/GVFS/GVFS.UnitTests.Windows/Windows/Virtualization/WindowsFileSystemVirtualizerTests.cs b/GVFS/GVFS.UnitTests.Windows/Windows/Virtualization/WindowsFileSystemVirtualizerTests.cs index 35c208466..eafdab5b4 100644 --- a/GVFS/GVFS.UnitTests.Windows/Windows/Virtualization/WindowsFileSystemVirtualizerTests.cs +++ b/GVFS/GVFS.UnitTests.Windows/Windows/Virtualization/WindowsFileSystemVirtualizerTests.cs @@ -441,7 +441,7 @@ public void OnGetFileStreamReturnsInternalErrorWhenOffsetNonZero() Guid enumerationGuid = Guid.NewGuid(); byte[] contentId = FileSystemVirtualizer.ConvertShaToContentId("0123456789012345678901234567890123456789"); - byte[] placeholderVersion = FileSystemVirtualizer.GetPlaceholderVersionId(); + byte[] placeholderVersion = WindowsFileSystemVirtualizer.PlaceholderVersionId; mockVirtualization.OnGetFileStream( commandId: 1, @@ -657,7 +657,7 @@ public void OnGetFileStreamReturnsPendingAndCompletesWithSuccessWhenNoFailures() Guid enumerationGuid = Guid.NewGuid(); byte[] contentId = FileSystemVirtualizer.ConvertShaToContentId("0123456789012345678901234567890123456789"); - byte[] placeholderVersion = FileSystemVirtualizer.GetPlaceholderVersionId(); + byte[] placeholderVersion = WindowsFileSystemVirtualizer.PlaceholderVersionId; uint fileLength = 100; MockGVFSGitObjects mockGVFSGitObjects = this.Repo.GitObjects as MockGVFSGitObjects; @@ -709,7 +709,7 @@ public void OnGetFileStreamHandlesTryCopyBlobContentStreamThrowingOperationCance Guid enumerationGuid = Guid.NewGuid(); byte[] contentId = FileSystemVirtualizer.ConvertShaToContentId("0123456789012345678901234567890123456789"); - byte[] placeholderVersion = FileSystemVirtualizer.GetPlaceholderVersionId(); + byte[] placeholderVersion = WindowsFileSystemVirtualizer.PlaceholderVersionId; MockGVFSGitObjects mockGVFSGitObjects = this.Repo.GitObjects as MockGVFSGitObjects; @@ -760,7 +760,7 @@ public void OnGetFileStreamHandlesCancellationDuringWriteAction() Guid enumerationGuid = Guid.NewGuid(); byte[] contentId = FileSystemVirtualizer.ConvertShaToContentId("0123456789012345678901234567890123456789"); - byte[] placeholderVersion = FileSystemVirtualizer.GetPlaceholderVersionId(); + byte[] placeholderVersion = WindowsFileSystemVirtualizer.PlaceholderVersionId; uint fileLength = 100; MockGVFSGitObjects mockGVFSGitObjects = this.Repo.GitObjects as MockGVFSGitObjects; @@ -815,7 +815,7 @@ public void OnGetFileStreamHandlesWriteFailure() Guid enumerationGuid = Guid.NewGuid(); byte[] contentId = FileSystemVirtualizer.ConvertShaToContentId("0123456789012345678901234567890123456789"); - byte[] placeholderVersion = FileSystemVirtualizer.GetPlaceholderVersionId(); + byte[] placeholderVersion = WindowsFileSystemVirtualizer.PlaceholderVersionId; uint fileLength = 100; MockGVFSGitObjects mockGVFSGitObjects = this.Repo.GitObjects as MockGVFSGitObjects; diff --git a/GVFS/GVFS.UnitTests/Platform.Mac/MacFileSystemVirtualizerTests.cs b/GVFS/GVFS.UnitTests/Platform.Mac/MacFileSystemVirtualizerTests.cs index 840c0177f..8b430267d 100644 --- a/GVFS/GVFS.UnitTests/Platform.Mac/MacFileSystemVirtualizerTests.cs +++ b/GVFS/GVFS.UnitTests/Platform.Mac/MacFileSystemVirtualizerTests.cs @@ -288,7 +288,7 @@ public void OnGetFileStreamReturnsSuccessWhenFileStreamAvailable() fileSystemCallbacks.TryStart(out error).ShouldEqual(true); byte[] contentId = FileSystemVirtualizer.ConvertShaToContentId("0123456789012345678901234567890123456789"); - byte[] placeholderVersion = FileSystemVirtualizer.GetPlaceholderVersionId(); + byte[] placeholderVersion = MacFileSystemVirtualizer.PlaceholderVersionId; uint fileLength = 100; MockGVFSGitObjects mockGVFSGitObjects = this.Repo.GitObjects as MockGVFSGitObjects; @@ -330,7 +330,7 @@ public void OnGetFileStreamReturnsErrorWhenWriteFileContentsFails() fileSystemCallbacks.TryStart(out error).ShouldEqual(true); byte[] contentId = FileSystemVirtualizer.ConvertShaToContentId("0123456789012345678901234567890123456789"); - byte[] placeholderVersion = FileSystemVirtualizer.GetPlaceholderVersionId(); + byte[] placeholderVersion = MacFileSystemVirtualizer.PlaceholderVersionId; uint fileLength = 100; MockGVFSGitObjects mockGVFSGitObjects = this.Repo.GitObjects as MockGVFSGitObjects; diff --git a/GVFS/GVFS.Virtualization/FileSystem/FileSystemVirtualizer.cs b/GVFS/GVFS.Virtualization/FileSystem/FileSystemVirtualizer.cs index 4357d85a4..0bfc60e0d 100644 --- a/GVFS/GVFS.Virtualization/FileSystem/FileSystemVirtualizer.cs +++ b/GVFS/GVFS.Virtualization/FileSystem/FileSystemVirtualizer.cs @@ -63,16 +63,6 @@ public static byte[] ConvertShaToContentId(string sha) return Encoding.Unicode.GetBytes(sha); } - /// - /// GVFS uses the first byte of the providerId field of placeholders to version - /// the data that it stores in the contentId (and providerId) fields of the placeholder - /// - /// Byte array to set as placeholder version Id - public static byte[] GetPlaceholderVersionId() - { - return new byte[] { PlaceholderVersion }; - } - public virtual bool TryStart(FileSystemCallbacks fileSystemCallbacks, out string error) { this.FileSystemCallbacks = fileSystemCallbacks; diff --git a/ProjFS.Mac/PrjFSKext/PrjFSKext/KauthHandler.cpp b/ProjFS.Mac/PrjFSKext/PrjFSKext/KauthHandler.cpp index c5d749782..acce65bce 100644 --- a/ProjFS.Mac/PrjFSKext/PrjFSKext/KauthHandler.cpp +++ b/ProjFS.Mac/PrjFSKext/PrjFSKext/KauthHandler.cpp @@ -214,6 +214,7 @@ void KauthHandler_HandleKernelMessageResponse(uint64_t messageId, MessageType re } } Mutex_Release(s_outstandingMessagesMutex); + break; } // The follow are not valid responses to kernel messages diff --git a/ProjFS.Mac/PrjFSLib/PrjFSLib.cpp b/ProjFS.Mac/PrjFSLib/PrjFSLib.cpp index b71db03a6..a60aa038f 100644 --- a/ProjFS.Mac/PrjFSLib/PrjFSLib.cpp +++ b/ProjFS.Mac/PrjFSLib/PrjFSLib.cpp @@ -405,7 +405,7 @@ PrjFS_Result PrjFS_DeleteFile( return PrjFS_Result_EInvalidArgs; } - // TODO(Mac): Unless allowed by updateFlags, ensure file is not full before proceeding + // TODO(Mac): Ensure file is not full before proceeding char fullPath[PrjFSMaxPath]; CombinePaths(s_virtualizationRootFullPath.c_str(), relativePath, fullPath); From 5a0715ebe2a92871f53a3da9b26ca7e245100899 Mon Sep 17 00:00:00 2001 From: William Baker Date: Thu, 30 Aug 2018 14:53:19 -0700 Subject: [PATCH 085/272] Fix warning that breaks Release build --- ProjFS.Mac/PrjFSLib/PrjFSLib.cpp | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/ProjFS.Mac/PrjFSLib/PrjFSLib.cpp b/ProjFS.Mac/PrjFSLib/PrjFSLib.cpp index a60aa038f..2fec82b75 100644 --- a/ProjFS.Mac/PrjFSLib/PrjFSLib.cpp +++ b/ProjFS.Mac/PrjFSLib/PrjFSLib.cpp @@ -65,7 +65,9 @@ static Message ParseMessageMemory(const void* messageMemory, uint32_t size); static void ClearMachNotification(mach_port_t port); +#ifdef DEBUG static const char* NotificationTypeToString(PrjFS_NotificationType notificationType); +#endif // State static io_connect_t s_kernelServiceConnection = IO_OBJECT_NULL; @@ -856,6 +858,7 @@ static void ClearMachNotification(mach_port_t port) mach_msg(&msg.msgHdr, MACH_RCV_MSG | MACH_RCV_TIMEOUT, 0, sizeof(msg), port, 0, MACH_PORT_NULL); } +#ifdef DEBUG static const char* NotificationTypeToString(PrjFS_NotificationType notificationType) { switch(notificationType) @@ -884,3 +887,4 @@ static const char* NotificationTypeToString(PrjFS_NotificationType notificationT return STRINGIFY(PrjFS_NotificationType_FileDeleted); } } +#endif From 6ec68462261f8ff2b2b9476877f985b5d392a452 Mon Sep 17 00:00:00 2001 From: William Baker Date: Thu, 30 Aug 2018 08:58:22 -0700 Subject: [PATCH 086/272] Mac: Support projection changes that add new files and delete folder --- .../FileSystem/IPlatformFileSystem.cs | 1 + GVFS/GVFS.Common/GVFSConstants.cs | 1 + GVFS/GVFS.Common/PlaceholderListDatabase.cs | 13 +- .../Tests/GitCommands/CheckoutTests.cs | 18 +++ GVFS/GVFS.Platform.Mac/MacFileSystem.cs | 1 + .../MacFileSystemVirtualizer.cs | 49 +++--- .../WindowsFileSystem.cs | 1 + .../WindowsFileSystemVirtualizer.cs | 59 +++++--- .../Should/EnumerableShouldExtensions.cs | 5 + .../Common/GitStatusCacheTests.cs | 2 + .../Mock/FileSystem/MockPlatformFileSystem.cs | 4 + .../FileSystem/MockFileSystemVirtualizer.cs | 5 + .../Projection/MockGitIndexProjection.cs | 8 + .../MacFileSystemVirtualizerTests.cs | 3 + .../FileSystem/FileSystemVirtualizer.cs | 6 + .../FileSystemCallbacks.cs | 5 + .../Projection/GitIndexProjection.cs | 139 ++++++++++++++++-- ProjFS.Mac/PrjFSLib/PrjFSLib.cpp | 2 +- 18 files changed, 266 insertions(+), 56 deletions(-) diff --git a/GVFS/GVFS.Common/FileSystem/IPlatformFileSystem.cs b/GVFS/GVFS.Common/FileSystem/IPlatformFileSystem.cs index 93c8cb2e9..9e7d44174 100644 --- a/GVFS/GVFS.Common/FileSystem/IPlatformFileSystem.cs +++ b/GVFS/GVFS.Common/FileSystem/IPlatformFileSystem.cs @@ -3,6 +3,7 @@ public interface IPlatformFileSystem { bool SupportsFileMode { get; } + bool EnumerationExpandsDirectories { get; } void FlushFileBuffers(string path); void MoveAndOverwriteFile(string sourceFileName, string destinationFilename); void CreateHardLink(string newLinkFileName, string existingFileName); diff --git a/GVFS/GVFS.Common/GVFSConstants.cs b/GVFS/GVFS.Common/GVFSConstants.cs index ece894ef8..8559eeba9 100644 --- a/GVFS/GVFS.Common/GVFSConstants.cs +++ b/GVFS/GVFS.Common/GVFSConstants.cs @@ -7,6 +7,7 @@ public static partial class GVFSConstants public const int ShaStringLength = 40; public const int MaxPath = 260; public const string AllZeroSha = "0000000000000000000000000000000000000000"; + public const string ExpandedFolderSha = "XF00000000000000000000000000000000000000"; public const char GitPathSeparator = '/'; public const string GitPathSeparatorString = "/"; diff --git a/GVFS/GVFS.Common/PlaceholderListDatabase.cs b/GVFS/GVFS.Common/PlaceholderListDatabase.cs index 48971b239..73dabaea4 100644 --- a/GVFS/GVFS.Common/PlaceholderListDatabase.cs +++ b/GVFS/GVFS.Common/PlaceholderListDatabase.cs @@ -229,7 +229,18 @@ public PlaceholderData(string path, string sha) public bool IsFolder { - get { return this.Sha == GVFSConstants.AllZeroSha; } + get + { + return this.Sha == GVFSConstants.AllZeroSha || this.IsExpandedFolder; + } + } + + public bool IsExpandedFolder + { + get + { + return this.Sha == GVFSConstants.ExpandedFolderSha; + } } } diff --git a/GVFS/GVFS.FunctionalTests/Tests/GitCommands/CheckoutTests.cs b/GVFS/GVFS.FunctionalTests/Tests/GitCommands/CheckoutTests.cs index ed4066332..9ac6c68ef 100644 --- a/GVFS/GVFS.FunctionalTests/Tests/GitCommands/CheckoutTests.cs +++ b/GVFS/GVFS.FunctionalTests/Tests/GitCommands/CheckoutTests.cs @@ -87,6 +87,24 @@ private enum NativeFileAccess : uint GENERIC_READ = 2147483648 } + // WIP: This test occasionally fails + [TestCase] + [Category(Categories.MacTODO.M3)] + public void ReadDeepFilesAfterCheckout() + { + // In commit 8df701986dea0a5e78b742d2eaf9348825b14d35 the CheckoutNewBranchFromStartingPointTest files were not present + this.ValidateGitCommand("checkout 8df701986dea0a5e78b742d2eaf9348825b14d35"); + + // In commit cd5c55fea4d58252bb38058dd3818da75aff6685 the CheckoutNewBranchFromStartingPointTest files were present + this.ValidateGitCommand("checkout cd5c55fea4d58252bb38058dd3818da75aff6685"); + + this.FileShouldHaveContents("TestFile1 \r\n", "GitCommandsTests", "CheckoutNewBranchFromStartingPointTest", "test1.txt"); + this.FileShouldHaveContents("TestFile2 \r\n", "GitCommandsTests", "CheckoutNewBranchFromStartingPointTest", "test2.txt"); + + this.ValidateGitCommand("status"); + } + + // WIP: This test occasionally fails [TestCase] [Category(Categories.MacTODO.M3)] public void CheckoutNewBranchFromStartingPointTest() diff --git a/GVFS/GVFS.Platform.Mac/MacFileSystem.cs b/GVFS/GVFS.Platform.Mac/MacFileSystem.cs index 4e72868a9..55244b827 100644 --- a/GVFS/GVFS.Platform.Mac/MacFileSystem.cs +++ b/GVFS/GVFS.Platform.Mac/MacFileSystem.cs @@ -7,6 +7,7 @@ namespace GVFS.Platform.Mac public partial class MacFileSystem : IPlatformFileSystem { public bool SupportsFileMode { get; } = true; + public bool EnumerationExpandsDirectories { get; } = true; public void FlushFileBuffers(string path) { diff --git a/GVFS/GVFS.Platform.Mac/MacFileSystemVirtualizer.cs b/GVFS/GVFS.Platform.Mac/MacFileSystemVirtualizer.cs index 29607f276..51542537c 100644 --- a/GVFS/GVFS.Platform.Mac/MacFileSystemVirtualizer.cs +++ b/GVFS/GVFS.Platform.Mac/MacFileSystemVirtualizer.cs @@ -70,6 +70,33 @@ public override void Stop() this.Context.Tracer.RelatedEvent(EventLevel.Informational, $"{nameof(this.Stop)}_StopRequested", metadata: null); } + public override FileSystemResult WritePlaceholder( + string relativePath, + long endOfFile, + bool isDirectory, + string sha) + { + Result result; + + if (isDirectory) + { + result = this.virtualizationInstance.WritePlaceholderDirectory(relativePath); + } + else + { + // TODO(Mac): Add functional tests that validate file mode is set correctly + ushort fileMode = this.FileSystemCallbacks.GitIndexProjection.GetFilePathMode(relativePath); + result = this.virtualizationInstance.WritePlaceholderFile( + relativePath, + PlaceholderVersionId, + ToVersionIdByteArray(FileSystemVirtualizer.ConvertShaToContentId(sha)), + (ulong)endOfFile, + fileMode); + } + + return new FileSystemResult(ResultToFSResult(result), unchecked((int)result)); + } + public override FileSystemResult UpdatePlaceholderIfNeeded( string relativePath, DateTime creationTime, @@ -436,25 +463,11 @@ private Result CreatePlaceholders(string directoryRelativePath, IEnumerable ShouldMatchInOrder(this IEnumerable group, IE return group; } + public static IEnumerable ShouldMatchInOrder(this IEnumerable group, params T[] expectedValues) + { + return group.ShouldMatchInOrder(expectedValues, (t1, t2) => t1.Equals(t2)); + } + public static IEnumerable ShouldMatchInOrder(this IEnumerable group, IEnumerable expectedValues) { return group.ShouldMatchInOrder(expectedValues, (t1, t2) => t1.Equals(t2)); diff --git a/GVFS/GVFS.UnitTests/Common/GitStatusCacheTests.cs b/GVFS/GVFS.UnitTests/Common/GitStatusCacheTests.cs index a4ef83db6..405f22483 100644 --- a/GVFS/GVFS.UnitTests/Common/GitStatusCacheTests.cs +++ b/GVFS/GVFS.UnitTests/Common/GitStatusCacheTests.cs @@ -2,6 +2,7 @@ using GVFS.Common.Git; using GVFS.Common.NamedPipes; using GVFS.Tests.Should; +using GVFS.UnitTests.Category; using GVFS.UnitTests.Mock.Common; using GVFS.UnitTests.Mock.FileSystem; using GVFS.UnitTests.Mock.Git; @@ -108,6 +109,7 @@ public void CanInvalidateCleanCache() } [TestCase] + [Category(CategoryConstants.ExceptionExpected)] public void CacheFileErrorShouldBlock() { this.fileSystem.DeleteFileThrowsException = true; diff --git a/GVFS/GVFS.UnitTests/Mock/FileSystem/MockPlatformFileSystem.cs b/GVFS/GVFS.UnitTests/Mock/FileSystem/MockPlatformFileSystem.cs index 999f7efd9..d3e6f42d5 100644 --- a/GVFS/GVFS.UnitTests/Mock/FileSystem/MockPlatformFileSystem.cs +++ b/GVFS/GVFS.UnitTests/Mock/FileSystem/MockPlatformFileSystem.cs @@ -6,6 +6,10 @@ namespace GVFS.UnitTests.Mock.FileSystem public class MockPlatformFileSystem : IPlatformFileSystem { public bool SupportsFileMode { get; } = true; + public bool EnumerationExpandsDirectories + { + get { throw new NotSupportedException(); } + } public void FlushFileBuffers(string path) { diff --git a/GVFS/GVFS.UnitTests/Mock/Virtualization/FileSystem/MockFileSystemVirtualizer.cs b/GVFS/GVFS.UnitTests/Mock/Virtualization/FileSystem/MockFileSystemVirtualizer.cs index 4972f23a2..a99625079 100644 --- a/GVFS/GVFS.UnitTests/Mock/Virtualization/FileSystem/MockFileSystemVirtualizer.cs +++ b/GVFS/GVFS.UnitTests/Mock/Virtualization/FileSystem/MockFileSystemVirtualizer.cs @@ -27,6 +27,11 @@ public override void Stop() { } + public override FileSystemResult WritePlaceholder(string relativePath, long endOfFile, bool isDirectory, string shaContentId) + { + throw new NotImplementedException(); + } + public override FileSystemResult UpdatePlaceholderIfNeeded(string relativePath, DateTime creationTime, DateTime lastAccessTime, DateTime lastWriteTime, DateTime changeTime, uint fileAttributes, long endOfFile, string shaContentId, UpdatePlaceholderType updateFlags, out UpdateFailureReason failureReason) { throw new NotImplementedException(); diff --git a/GVFS/GVFS.UnitTests/Mock/Virtualization/Projection/MockGitIndexProjection.cs b/GVFS/GVFS.UnitTests/Mock/Virtualization/Projection/MockGitIndexProjection.cs index fe82cd2e3..904946610 100644 --- a/GVFS/GVFS.UnitTests/Mock/Virtualization/Projection/MockGitIndexProjection.cs +++ b/GVFS/GVFS.UnitTests/Mock/Virtualization/Projection/MockGitIndexProjection.cs @@ -35,6 +35,7 @@ public MockGitIndexProjection(IEnumerable projectedFiles) } this.PlaceholdersCreated = new ConcurrentHashSet(); + this.ExpandedFolders = new ConcurrentHashSet(); this.MockFileModes = new ConcurrentDictionary(); this.unblockGetProjectedItems = new ManualResetEvent(true); @@ -53,6 +54,8 @@ public MockGitIndexProjection(IEnumerable projectedFiles) public ConcurrentHashSet PlaceholdersCreated { get; } + public ConcurrentHashSet ExpandedFolders { get; } + public ConcurrentDictionary MockFileModes { get; } public bool ThrowOperationCanceledExceptionOnProjectionRequest { get; set; } @@ -231,6 +234,11 @@ public override ProjectedFileInfo GetProjectedFileInfo( return null; } + public override void OnPlaceholderFolderExpanded(string relativePath) + { + this.ExpandedFolders.Add(relativePath); + } + public override void OnPlaceholderFileCreated(string virtualPath, string sha) { this.PlaceholdersCreated.Add(virtualPath); diff --git a/GVFS/GVFS.UnitTests/Platform.Mac/MacFileSystemVirtualizerTests.cs b/GVFS/GVFS.UnitTests/Platform.Mac/MacFileSystemVirtualizerTests.cs index 8b430267d..675ae8a93 100644 --- a/GVFS/GVFS.UnitTests/Platform.Mac/MacFileSystemVirtualizerTests.cs +++ b/GVFS/GVFS.UnitTests/Platform.Mac/MacFileSystemVirtualizerTests.cs @@ -1,6 +1,7 @@ using GVFS.Common; using GVFS.Platform.Mac; using GVFS.Tests.Should; +using GVFS.UnitTests.Category; using GVFS.UnitTests.Mock.Git; using GVFS.UnitTests.Mock.Mac; using GVFS.UnitTests.Mock.Virtualization.Background; @@ -228,6 +229,7 @@ public void OnEnumerateDirectoryReturnsSuccessWhenResultsInMemory() mockVirtualization.OnEnumerateDirectory(1, "test", triggeringProcessId: 1, triggeringProcessName: "UnitTests").ShouldEqual(Result.Success); mockVirtualization.CreatedPlaceholders.ShouldContain( kvp => kvp.Key.Equals(Path.Combine("test", "test.txt"), StringComparison.OrdinalIgnoreCase) && kvp.Value == FileMode644); + gitIndexProjection.ExpandedFolders.ShouldMatchInOrder("test"); fileSystemCallbacks.Stop(); } } @@ -311,6 +313,7 @@ public void OnGetFileStreamReturnsSuccessWhenFileStreamAvailable() } [TestCase] + [Category(CategoryConstants.ExceptionExpected)] public void OnGetFileStreamReturnsErrorWhenWriteFileContentsFails() { using (MockBackgroundFileSystemTaskRunner backgroundTaskRunner = new MockBackgroundFileSystemTaskRunner()) diff --git a/GVFS/GVFS.Virtualization/FileSystem/FileSystemVirtualizer.cs b/GVFS/GVFS.Virtualization/FileSystem/FileSystemVirtualizer.cs index 0bfc60e0d..3c67d1692 100644 --- a/GVFS/GVFS.Virtualization/FileSystem/FileSystemVirtualizer.cs +++ b/GVFS/GVFS.Virtualization/FileSystem/FileSystemVirtualizer.cs @@ -93,6 +93,12 @@ public void PrepareToStop() public abstract FileSystemResult DeleteFile(string relativePath, UpdatePlaceholderType updateFlags, out UpdateFailureReason failureReason); + public abstract FileSystemResult WritePlaceholder( + string relativePath, + long endOfFile, + bool isDirectory, + string shaContentId); + public abstract FileSystemResult UpdatePlaceholderIfNeeded( string relativePath, DateTime creationTime, diff --git a/GVFS/GVFS.Virtualization/FileSystemCallbacks.cs b/GVFS/GVFS.Virtualization/FileSystemCallbacks.cs index 28dd595f6..ce1b8afe4 100644 --- a/GVFS/GVFS.Virtualization/FileSystemCallbacks.cs +++ b/GVFS/GVFS.Virtualization/FileSystemCallbacks.cs @@ -471,6 +471,11 @@ public void OnPlaceholderFolderCreated(string relativePath) this.GitIndexProjection.OnPlaceholderFolderCreated(relativePath); } + public void OnPlaceholderFolderExpanded(string relativePath) + { + this.GitIndexProjection.OnPlaceholderFolderExpanded(relativePath); + } + public FileProperties GetLogsHeadFileProperties() { // Use a temporary FileProperties in case another thread sets this.logsHeadFileProperties before this diff --git a/GVFS/GVFS.Virtualization/Projection/GitIndexProjection.cs b/GVFS/GVFS.Virtualization/Projection/GitIndexProjection.cs index c807016a8..0d506f688 100644 --- a/GVFS/GVFS.Virtualization/Projection/GitIndexProjection.cs +++ b/GVFS/GVFS.Virtualization/Projection/GitIndexProjection.cs @@ -320,6 +320,11 @@ public void OnPlaceholderFolderCreated(string virtualPath) this.placeholderList.AddAndFlush(virtualPath, GVFSConstants.AllZeroSha); } + public virtual void OnPlaceholderFolderExpanded(string relativePath) + { + this.placeholderList.AddAndFlush(relativePath, GVFSConstants.ExpandedFolderSha); + } + public virtual void OnPlaceholderFileCreated(string virtualPath, string sha) { this.placeholderList.AddAndFlush(virtualPath, sha); @@ -1084,7 +1089,7 @@ private void UpdatePlaceholders() using (ITracer activity = this.context.Tracer.StartActivity("UpdatePlaceholders", EventLevel.Informational, metadata)) { ConcurrentHashSet folderPlaceholdersToKeep = new ConcurrentHashSet(); - ConcurrentBag updatedPlaceholderList = new ConcurrentBag(); + ConcurrentDictionary updatedPlaceholderList = new ConcurrentDictionary(StringComparer.Ordinal); this.ProcessListOnThreads( placeholderListCopy.Where(x => !x.IsFolder).ToList(), (placeholderBatch, start, end, blobSizesConnection, availableSizes) => @@ -1094,14 +1099,25 @@ private void UpdatePlaceholders() this.blobSizes.Flush(); - // Waiting for the UpdateOrDeleteFilePlaceholder to fill the folderPlaceholdersToKeep before trying to remove folder placeholders - // so that we don't try to delete a folder placeholder that has file placeholders and just fails - foreach (PlaceholderListDatabase.PlaceholderData folderPlaceholder in placeholderListCopy.Where(x => x.IsFolder).OrderByDescending(x => x.Path)) - { - this.TryRemoveFolderPlaceholder(folderPlaceholder, updatedPlaceholderList, folderPlaceholdersToKeep); + using (BlobSizes.BlobSizesConnection blobSizesConnection = this.blobSizes.CreateConnection()) + { + // Waiting for the UpdateOrDeleteFilePlaceholder to fill the folderPlaceholdersToKeep before trying to remove folder placeholders + // so that we don't try to delete a folder placeholder that has file placeholders and just fails + IEnumerable folderPlaceholderData = placeholderListCopy.Where(x => x.IsFolder); + + HashSet folderPlaceholders = new HashSet(folderPlaceholderData.Select(x => x.Path), StringComparer.OrdinalIgnoreCase); + foreach (PlaceholderListDatabase.PlaceholderData folderPlaceholder in folderPlaceholderData.OrderByDescending(x => x.Path)) + { + if (GVFSPlatform.Instance.FileSystem.EnumerationExpandsDirectories && folderPlaceholder.IsExpandedFolder) + { + this.ReExpandFolder(blobSizesConnection, folderPlaceholder.Path, updatedPlaceholderList, folderPlaceholders, folderPlaceholdersToKeep); + } + + this.TryRemoveFolderPlaceholder(folderPlaceholder, updatedPlaceholderList, folderPlaceholdersToKeep); + } } - this.placeholderList.WriteAllEntriesAndFlush(updatedPlaceholderList); + this.placeholderList.WriteAllEntriesAndFlush(updatedPlaceholderList.Values); this.repoMetadata.SetPlaceholdersNeedUpdate(false); TimeSpan duration = activity.Stop(null); @@ -1257,17 +1273,107 @@ private string GetNewProjectedShaForPlaceholder(string path) return null; } + private void ReExpandFolder( + BlobSizes.BlobSizesConnection blobSizesConnection, + string relativeFolderPath, + ConcurrentDictionary updatedPlaceholderList, + HashSet existingFolderPlaceholders, + ConcurrentHashSet folderPlaceholdersToKeep) + { + FolderData folderData; + if (!this.TryGetOrAddFolderDataFromCache(relativeFolderPath, out folderData)) + { + // Folder is no longer in the projection + return; + } + + folderData.PopulateSizes( + this.context.Tracer, + this.gitObjects, + blobSizesConnection, + availableSizes: null, + cancellationToken: CancellationToken.None); + + for (int i = 0; i < folderData.ChildEntries.Count; i++) + { + FolderEntryData childEntry = folderData.ChildEntries[i]; + string childRelativePath; + if (relativeFolderPath.Length == 0) + { + childRelativePath = childEntry.Name.GetString(); + } + else + { + childRelativePath = relativeFolderPath + Path.DirectorySeparatorChar + childEntry.Name.GetString(); + } + + if (childEntry.IsFolder) + { + if (!existingFolderPlaceholders.Contains(childRelativePath)) + { + // TODO(Mac): Check return value of WritePlaceholder + this.fileSystemVirtualizer.WritePlaceholder( + childRelativePath, + endOfFile: 0, + isDirectory: true, + shaContentId: GVFSConstants.AllZeroSha); + + updatedPlaceholderList.TryAdd( + childRelativePath, + new PlaceholderListDatabase.PlaceholderData(childRelativePath, GVFSConstants.AllZeroSha)); + + folderPlaceholdersToKeep.Add(relativeFolderPath); + } + } + else + { + if (!updatedPlaceholderList.ContainsKey(childRelativePath)) + { + FileData childFileData = childEntry as FileData; + string sha = childFileData.Sha.ToString(); + + // TODO(Mac): Check return value of WritePlaceholder + this.fileSystemVirtualizer.WritePlaceholder(childRelativePath, childFileData.Size, isDirectory: false, shaContentId: sha); + + updatedPlaceholderList.TryAdd( + childRelativePath, + new PlaceholderListDatabase.PlaceholderData(childRelativePath, sha)); + + folderPlaceholdersToKeep.Add(relativeFolderPath); + } + } + } + } + private void TryRemoveFolderPlaceholder( PlaceholderListDatabase.PlaceholderData placeholder, - ConcurrentBag updatedPlaceholderList, + ConcurrentDictionary updatedPlaceholderList, ConcurrentHashSet folderPlaceholdersToKeep) { if (folderPlaceholdersToKeep.Contains(placeholder.Path)) { - updatedPlaceholderList.Add(placeholder); + updatedPlaceholderList.TryAdd(placeholder.Path, placeholder); return; } + if (GVFSPlatform.Instance.FileSystem.EnumerationExpandsDirectories) + { + // If enumeration expands directories we should exit early if the folder + // is still in the projection to avoid deleting folder placeholders that should + // still be on disk. However, if enumeration does not expand directories there + // may be folder tombstones on disk that need to get cleaned up (e.g. because git + // deleted a folder a folder that was not in ModifiedPaths.dat) and we should + // proceed with removing the folder placeholder. + + FolderData folderData; + if (this.TryGetOrAddFolderDataFromCache(placeholder.Path, out folderData)) + { + // Folder is still in the projection and should not be removed + updatedPlaceholderList.TryAdd(placeholder.Path, placeholder); + return; + } + } + UpdateFailureReason failureReason = UpdateFailureReason.NoFailure; FileSystemResult result = this.fileSystemVirtualizer.DeleteFile(placeholder.Path, FolderPlaceholderDeleteFlags, out failureReason); switch (result.Result) @@ -1276,7 +1382,7 @@ private void TryRemoveFolderPlaceholder( break; case FSResult.DirectoryNotEmpty: - updatedPlaceholderList.Add(new PlaceholderListDatabase.PlaceholderData(placeholder.Path, GVFSConstants.AllZeroSha)); + updatedPlaceholderList.TryAdd(placeholder.Path, placeholder); break; case FSResult.FileOrPathNotFound: @@ -1289,6 +1395,9 @@ private void TryRemoveFolderPlaceholder( metadata.Add("result.RawResult", result.RawResult); metadata.Add("UpdateFailureCause", failureReason.ToString()); this.context.Tracer.RelatedEvent(EventLevel.Informational, nameof(this.TryRemoveFolderPlaceholder) + "_DeleteFileFailure", metadata); + + updatedPlaceholderList.TryAdd(placeholder.Path, placeholder); + break; } } @@ -1296,7 +1405,7 @@ private void TryRemoveFolderPlaceholder( private void UpdateOrDeleteFilePlaceholder( BlobSizes.BlobSizesConnection blobSizesConnection, PlaceholderListDatabase.PlaceholderData placeholder, - ConcurrentBag updatedPlaceholderList, + ConcurrentDictionary updatedPlaceholderList, ConcurrentHashSet folderPlaceholdersToKeep, Dictionary availableSizes) { @@ -1364,7 +1473,7 @@ private void UpdateOrDeleteFilePlaceholder( } else { - updatedPlaceholderList.Add(placeholder); + updatedPlaceholderList.TryAdd(placeholder.Path, placeholder); this.AddParentFoldersToListToKeep(parentKey, folderPlaceholdersToKeep); } } @@ -1374,7 +1483,7 @@ private void ProcessGvUpdateDeletePlaceholderResult( PlaceholderListDatabase.PlaceholderData placeholder, string projectedSha, FileSystemResult result, - ConcurrentBag updatedPlaceholderList, + ConcurrentDictionary updatedPlaceholderList, UpdateFailureReason failureReason, string parentKey, ConcurrentHashSet folderPlaceholdersToKeep, @@ -1386,7 +1495,9 @@ private void ProcessGvUpdateDeletePlaceholderResult( case FSResult.Ok: if (!deleteOperation) { - updatedPlaceholderList.Add(new PlaceholderListDatabase.PlaceholderData(placeholder.Path, projectedSha)); + updatedPlaceholderList.TryAdd( + placeholder.Path, + new PlaceholderListDatabase.PlaceholderData(placeholder.Path, projectedSha)); this.AddParentFoldersToListToKeep(parentKey, folderPlaceholdersToKeep); } diff --git a/ProjFS.Mac/PrjFSLib/PrjFSLib.cpp b/ProjFS.Mac/PrjFSLib/PrjFSLib.cpp index 2fec82b75..a858a7fe3 100644 --- a/ProjFS.Mac/PrjFSLib/PrjFSLib.cpp +++ b/ProjFS.Mac/PrjFSLib/PrjFSLib.cpp @@ -411,7 +411,7 @@ PrjFS_Result PrjFS_DeleteFile( char fullPath[PrjFSMaxPath]; CombinePaths(s_virtualizationRootFullPath.c_str(), relativePath, fullPath); - if (0 != unlink(fullPath)) + if (0 != remove(fullPath)) { switch(errno) { From 9bc2b6cd28d3d009158a46e8655288cc27a374ac Mon Sep 17 00:00:00 2001 From: William Baker Date: Tue, 4 Sep 2018 08:58:03 -0700 Subject: [PATCH 087/272] Fix compilation error on Windows --- GVFS/GVFS.Platform.Windows/WindowsFileSystemVirtualizer.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/GVFS/GVFS.Platform.Windows/WindowsFileSystemVirtualizer.cs b/GVFS/GVFS.Platform.Windows/WindowsFileSystemVirtualizer.cs index a622c35a5..089c34eb0 100644 --- a/GVFS/GVFS.Platform.Windows/WindowsFileSystemVirtualizer.cs +++ b/GVFS/GVFS.Platform.Windows/WindowsFileSystemVirtualizer.cs @@ -142,7 +142,7 @@ public override FileSystemResult WritePlaceholder( endOfFile: endOfFile, isDirectory: isDirectory, contentId: FileSystemVirtualizer.ConvertShaToContentId(sha), - providerId: FPlaceholderVersionId); + providerId: PlaceholderVersionId); return new FileSystemResult(HResultToFSResult(result), unchecked((int)result)); } From 7ef673d8e4b7c750a091dc0c723f1d9f9a127c85 Mon Sep 17 00:00:00 2001 From: William Baker Date: Tue, 4 Sep 2018 16:58:31 -0700 Subject: [PATCH 088/272] Enable most reset soft functional tests --- GVFS/GVFS.FunctionalTests/Tests/GitCommands/ResetSoftTests.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/GVFS/GVFS.FunctionalTests/Tests/GitCommands/ResetSoftTests.cs b/GVFS/GVFS.FunctionalTests/Tests/GitCommands/ResetSoftTests.cs index 01ce8336b..e79cdf99b 100644 --- a/GVFS/GVFS.FunctionalTests/Tests/GitCommands/ResetSoftTests.cs +++ b/GVFS/GVFS.FunctionalTests/Tests/GitCommands/ResetSoftTests.cs @@ -4,7 +4,6 @@ namespace GVFS.FunctionalTests.Tests.GitCommands { [TestFixture] [Category(Categories.GitCommands)] - [Category(Categories.MacTODO.M3)] public class ResetSoftTests : GitRepoTests { public ResetSoftTests() : base(enlistmentPerTest: true) @@ -51,6 +50,7 @@ public void ResetSoftThenCheckoutNoConflicts() } [TestCase] + [Category(Categories.MacTODO.M3)] public void ResetSoftThenResetHeadThenCheckoutNoConflicts() { this.ValidateGitCommand("checkout " + GitRepoTests.ConflictTargetBranch); From e28849f3bd6eae3ae9446fe76df4ef620c40549a Mon Sep 17 00:00:00 2001 From: William Baker Date: Tue, 4 Sep 2018 17:15:23 -0700 Subject: [PATCH 089/272] Enable additional tests in CheckoutTests --- .../Tests/GitCommands/CheckoutTests.cs | 9 ++------- 1 file changed, 2 insertions(+), 7 deletions(-) diff --git a/GVFS/GVFS.FunctionalTests/Tests/GitCommands/CheckoutTests.cs b/GVFS/GVFS.FunctionalTests/Tests/GitCommands/CheckoutTests.cs index 9ac6c68ef..e5a7af56f 100644 --- a/GVFS/GVFS.FunctionalTests/Tests/GitCommands/CheckoutTests.cs +++ b/GVFS/GVFS.FunctionalTests/Tests/GitCommands/CheckoutTests.cs @@ -140,13 +140,12 @@ public void CheckoutOrhpanBranchFromStartingPointTest() } [TestCase] - [Category(Categories.MacTODO.M3)] public void MoveFileFromDotGitFolderToWorkingDirectoryAndAddAndCheckout() { string testFileContents = "Test file contents for MoveFileFromDotGitFolderToWorkingDirectoryAndAddAndCheckout"; string filename = "AddedBySource.txt"; - string dotGitFilePath = @".git\" + filename; - string targetPath = @"Test_ConflictTests\AddedFiles\" + filename; + string dotGitFilePath = Path.Combine(".git", filename); + string targetPath = Path.Combine("Test_ConflictTests", "AddedFiles", filename); // In commit db95d631e379d366d26d899523f8136a77441914 Test_ConflictTests\AddedFiles\AddedBySource.txt does not exist this.ControlGitRepo.Fetch(GitRepoTests.ConflictSourceBranch); @@ -173,7 +172,6 @@ public void MoveFileFromDotGitFolderToWorkingDirectoryAndAddAndCheckout() } [TestCase] - [Category(Categories.MacTODO.M3)] public void CheckoutBranchNoCrashOnStatus() { this.ControlGitRepo.Fetch("FunctionalTests/20170331_git_crash"); @@ -182,7 +180,6 @@ public void CheckoutBranchNoCrashOnStatus() } [TestCase] - [Category(Categories.MacTODO.M3)] public void CheckoutCommitWhereFileContentsChangeAfterRead() { this.ControlGitRepo.Fetch(GitRepoTests.ConflictSourceBranch); @@ -202,7 +199,6 @@ public void CheckoutCommitWhereFileContentsChangeAfterRead() } [TestCase] - [Category(Categories.MacTODO.M3)] public void CheckoutCommitWhereFileDeletedAfterRead() { this.ControlGitRepo.Fetch(GitRepoTests.ConflictSourceBranch); @@ -729,7 +725,6 @@ public void ResetMixedTwiceThenCheckoutWithRemovedFiles() } [TestCase] - [Category(Categories.MacTODO.M3)] public void DeleteFolderAndChangeBranchToFolderWithDifferentCase() { // 692765 - Recursive modified paths entries for folders should be case insensitive when From 981a155934532cea2b9605cec1e4cd6eee0fc080 Mon Sep 17 00:00:00 2001 From: Saeed Noursalehi Date: Wed, 5 Sep 2018 12:22:27 -0700 Subject: [PATCH 090/272] Throw if a message is written to a disposed tracer --- GVFS/GVFS.Common/Tracing/JsonTracer.cs | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/GVFS/GVFS.Common/Tracing/JsonTracer.cs b/GVFS/GVFS.Common/Tracing/JsonTracer.cs index 07e424b23..07fe4b01c 100644 --- a/GVFS/GVFS.Common/Tracing/JsonTracer.cs +++ b/GVFS/GVFS.Common/Tracing/JsonTracer.cs @@ -20,6 +20,9 @@ public class JsonTracer : ITracer private EventLevel startStopLevel; private Keywords startStopKeywords; + + private bool disposed; + public JsonTracer(string providerName, string activityName, bool disableTelemetry = false) : this(providerName, Guid.Empty, activityName, enlistmentId: null, mountId: null, disableTelemetry: disableTelemetry) { @@ -101,6 +104,8 @@ public void Dispose() this.listeners.Clear(); } + + this.disposed = true; } public virtual void RelatedEvent(EventLevel level, string eventName, EventMetadata metadata) @@ -256,6 +261,15 @@ private static string GetCategorizedErrorEventName(Keywords keywords) private void WriteEvent(string eventName, EventLevel level, Keywords keywords, EventMetadata metadata, EventOpcode opcode) { string jsonPayload = metadata != null ? JsonConvert.SerializeObject(metadata) : null; + + if (this.disposed) + { + Console.WriteLine("Writing to disposed tracer"); + Console.WriteLine(jsonPayload); + + throw new ObjectDisposedException(nameof(JsonTracer)); + } + foreach (InProcEventListener listener in this.listeners) { listener.RecordMessage(eventName, this.activityId, this.parentActivityId, level, keywords, opcode, jsonPayload); From 1a9e8c3edc32ab0224ee9c4806fde3e82b730c1f Mon Sep 17 00:00:00 2001 From: Saeed Noursalehi Date: Wed, 5 Sep 2018 12:23:32 -0700 Subject: [PATCH 091/272] Early exit if the named pipe is already disposed --- GVFS/GVFS.Common/NamedPipes/NamedPipeServer.cs | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/GVFS/GVFS.Common/NamedPipes/NamedPipeServer.cs b/GVFS/GVFS.Common/NamedPipes/NamedPipeServer.cs index 91350dfcd..3b89baaba 100644 --- a/GVFS/GVFS.Common/NamedPipes/NamedPipeServer.cs +++ b/GVFS/GVFS.Common/NamedPipes/NamedPipeServer.cs @@ -87,7 +87,10 @@ private void OpenListeningPipe() private void OnNewConnection(IAsyncResult ar) { - this.OnNewConnection(ar, createNewThreadIfSynchronous: true); + if (!this.isStopping) + { + this.OnNewConnection(ar, createNewThreadIfSynchronous: true); + } } private void OnNewConnection(IAsyncResult ar, bool createNewThreadIfSynchronous) From dc4a5f6075f2b23a4c4b0091e85c26b6bafb387f Mon Sep 17 00:00:00 2001 From: Saeed Noursalehi Date: Wed, 5 Sep 2018 12:24:02 -0700 Subject: [PATCH 092/272] Clarify a warning message that gets logged during prefetch if the mount process is not running --- GVFS/GVFS.Common/Prefetch/CommitPrefetcher.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/GVFS/GVFS.Common/Prefetch/CommitPrefetcher.cs b/GVFS/GVFS.Common/Prefetch/CommitPrefetcher.cs index eb2237cbc..ebe943ee0 100644 --- a/GVFS/GVFS.Common/Prefetch/CommitPrefetcher.cs +++ b/GVFS/GVFS.Common/Prefetch/CommitPrefetcher.cs @@ -155,7 +155,7 @@ private static bool TrySchedulePostFetchJob(ITracer tracer, string namedPipeName { tracer.RelatedWarning( metadata: null, - message: "Failed to connect to GVFS. Skipping post-fetch job request.", + message: "Failed to connect to GVFS.Mount process. Skipping post-fetch job request.", keywords: Keywords.Telemetry); return false; } From 3162792c53d792d4bf3c8934d88ad322084da871 Mon Sep 17 00:00:00 2001 From: Saeed Noursalehi Date: Wed, 5 Sep 2018 14:55:04 -0700 Subject: [PATCH 093/272] No longer need to handle ObjectDisposedException on Windows --- GVFS/GVFS.Common/NamedPipes/NamedPipeServer.cs | 7 ------- 1 file changed, 7 deletions(-) diff --git a/GVFS/GVFS.Common/NamedPipes/NamedPipeServer.cs b/GVFS/GVFS.Common/NamedPipes/NamedPipeServer.cs index 3b89baaba..d6eebe711 100644 --- a/GVFS/GVFS.Common/NamedPipes/NamedPipeServer.cs +++ b/GVFS/GVFS.Common/NamedPipes/NamedPipeServer.cs @@ -126,13 +126,6 @@ private void OnNewConnection(IAsyncResult ar, bool createNewThreadIfSynchronous) metadata.Add(TracingConstants.MessageKey.WarningMessage, "OnNewConnection: Connection broken"); this.tracer.RelatedEvent(EventLevel.Warning, "OnNewConnectionn_EndWaitForConnection_IOException", metadata); } - catch (ObjectDisposedException) - { - if (!this.isStopping) - { - throw; - } - } catch (Exception e) { this.LogErrorAndExit("OnNewConnection caught unhandled exception, exiting process", e); From 5041065091bf7acb1e1aa4ef077124e8d17765e7 Mon Sep 17 00:00:00 2001 From: William Baker Date: Wed, 5 Sep 2018 15:27:28 -0700 Subject: [PATCH 094/272] Enable more functional tests --- .../Tests/GitCommands/CheckoutTests.cs | 16 ++-------------- .../Tests/GitCommands/DeleteEmptyFolderTests.cs | 1 - .../Tests/GitCommands/EnumerationMergeTest.cs | 1 - .../Tests/GitCommands/GitCommandsTests.cs | 4 ---- .../Tests/GitCommands/GitRepoTests.cs | 2 +- .../Tests/GitCommands/UpdateRefTests.cs | 1 - 6 files changed, 3 insertions(+), 22 deletions(-) diff --git a/GVFS/GVFS.FunctionalTests/Tests/GitCommands/CheckoutTests.cs b/GVFS/GVFS.FunctionalTests/Tests/GitCommands/CheckoutTests.cs index e5a7af56f..0208cd188 100644 --- a/GVFS/GVFS.FunctionalTests/Tests/GitCommands/CheckoutTests.cs +++ b/GVFS/GVFS.FunctionalTests/Tests/GitCommands/CheckoutTests.cs @@ -219,7 +219,6 @@ public void CheckoutCommitWhereFileDeletedAfterRead() } [TestCase] - [Category(Categories.MacTODO.M3)] public void CheckoutBranchAfterReadingFileAndVerifyContentsCorrect() { this.ControlGitRepo.Fetch(GitRepoTests.ConflictSourceBranch); @@ -232,11 +231,10 @@ public void CheckoutBranchAfterReadingFileAndVerifyContentsCorrect() this.FilesShouldMatchCheckoutOfSourceBranch(); // Verify modified paths contents - GVFSHelpers.ModifiedPathsContentsShouldEqual(this.FileSystem, this.Enlistment.DotGVFSRoot, "A .gitattributes" + Environment.NewLine); + GVFSHelpers.ModifiedPathsContentsShouldEqual(this.FileSystem, this.Enlistment.DotGVFSRoot, "A .gitattributes\r\n"); } [TestCase] - [Category(Categories.MacTODO.M3)] public void CheckoutBranchAfterReadingAllFilesAndVerifyContentsCorrect() { this.ControlGitRepo.Fetch(GitRepoTests.ConflictSourceBranch); @@ -251,7 +249,7 @@ public void CheckoutBranchAfterReadingAllFilesAndVerifyContentsCorrect() .WithDeepStructure(this.FileSystem, this.ControlGitRepo.RootPath, compareContent: true); // Verify modified paths contents - GVFSHelpers.ModifiedPathsContentsShouldEqual(this.FileSystem, this.Enlistment.DotGVFSRoot, "A .gitattributes" + Environment.NewLine); + GVFSHelpers.ModifiedPathsContentsShouldEqual(this.FileSystem, this.Enlistment.DotGVFSRoot, "A .gitattributes\r\n"); } [TestCase] @@ -282,7 +280,6 @@ public void CheckoutBranchThatHasFolderShouldGetDeleted() } [TestCase] - [Category(Categories.MacTODO.M3)] public void CheckoutBranchThatDoesNotHaveFolderShouldNotHaveFolder() { // this.ControlGitRepo.Commitish should not have the folder Test_ConflictTests\AddedFiles @@ -308,7 +305,6 @@ public void CheckoutBranchThatDoesNotHaveFolderShouldNotHaveFolder() } [TestCase] - [Category(Categories.MacTODO.M3)] public void EditFileReadFileAndCheckoutConflict() { // editFilePath was changed on ConflictTargetBranch @@ -343,7 +339,6 @@ public void EditFileReadFileAndCheckoutConflict() } [TestCase] - [Category(Categories.MacTODO.M3)] public void MarkFileAsReadOnlyAndCheckoutCommitWhereFileIsDifferent() { string filePath = Path.Combine("Test_ConflictTests", "ModifiedFiles", "ConflictingChange.txt"); @@ -359,7 +354,6 @@ public void MarkFileAsReadOnlyAndCheckoutCommitWhereFileIsDifferent() } [TestCase] - [Category(Categories.MacTODO.M3)] public void MarkFileAsReadOnlyAndCheckoutCommitWhereFileIsDeleted() { string filePath = Path.Combine("Test_ConflictTests", "AddedFiles", "AddedBySource.txt"); @@ -375,7 +369,6 @@ public void MarkFileAsReadOnlyAndCheckoutCommitWhereFileIsDeleted() } [TestCase] - [Category(Categories.MacTODO.M3)] public void ModifyAndCheckoutFirstOfSeveralFilesWhoseNamesAppearBeforeDot() { // Commit cb2d05febf64e3b0df50bd8d3fe8f05c0e2caa47 has the files (a).txt and (z).txt @@ -460,7 +453,6 @@ public void ReadFileAfterTryingToReadFileAtCommitWhereFileDoesNotExist() } [TestCase] - [Category(Categories.MacTODO.M3)] public void CheckoutBranchWithOpenHandleBlockingRepoMetdataUpdate() { this.ControlGitRepo.Fetch(GitRepoTests.ConflictSourceBranch); @@ -566,7 +558,6 @@ public void CheckoutBranchWithOpenHandleBlockingProjectionDeleteAndRepoMetdataUp } [TestCase] - [Category(Categories.MacTODO.M3)] public void CheckoutBranchWithStaleRepoMetadataTmpFileOnDisk() { this.ControlGitRepo.Fetch(GitRepoTests.ConflictSourceBranch); @@ -888,7 +879,6 @@ public void CheckoutBranchWithDirectoryNameSameAsFileWithWrite() } [TestCase] - [Category(Categories.MacTODO.M3)] public void CheckoutBranchDirectoryWithOneFile() { this.SetupForFileDirectoryTest(commandBranch: GitRepoTests.DirectoryWithDifferentFileAfterBranch); @@ -896,14 +886,12 @@ public void CheckoutBranchDirectoryWithOneFile() } [TestCase] - [Category(Categories.MacTODO.M3)] public void CheckoutBranchDirectoryWithOneFileEnumerate() { this.RunFileDirectoryEnumerateTest("checkout", commandBranch: GitRepoTests.DirectoryWithDifferentFileAfterBranch); } [TestCase] - [Category(Categories.MacTODO.M3)] public void CheckoutBranchDirectoryWithOneFileRead() { this.RunFileDirectoryReadTest("checkout", commandBranch: GitRepoTests.DirectoryWithDifferentFileAfterBranch); diff --git a/GVFS/GVFS.FunctionalTests/Tests/GitCommands/DeleteEmptyFolderTests.cs b/GVFS/GVFS.FunctionalTests/Tests/GitCommands/DeleteEmptyFolderTests.cs index 7bfb99cde..9fd134313 100644 --- a/GVFS/GVFS.FunctionalTests/Tests/GitCommands/DeleteEmptyFolderTests.cs +++ b/GVFS/GVFS.FunctionalTests/Tests/GitCommands/DeleteEmptyFolderTests.cs @@ -6,7 +6,6 @@ namespace GVFS.FunctionalTests.Tests.GitCommands { [TestFixture] [Category(Categories.GitCommands)] - [Category(Categories.MacTODO.M4)] public class DeleteEmptyFolderTests : GitRepoTests { public DeleteEmptyFolderTests() : base(enlistmentPerTest: true) diff --git a/GVFS/GVFS.FunctionalTests/Tests/GitCommands/EnumerationMergeTest.cs b/GVFS/GVFS.FunctionalTests/Tests/GitCommands/EnumerationMergeTest.cs index 6b77a2fdf..4306529ab 100644 --- a/GVFS/GVFS.FunctionalTests/Tests/GitCommands/EnumerationMergeTest.cs +++ b/GVFS/GVFS.FunctionalTests/Tests/GitCommands/EnumerationMergeTest.cs @@ -4,7 +4,6 @@ namespace GVFS.FunctionalTests.Tests.GitCommands { [TestFixture] [Category(Categories.GitCommands)] - [Category(Categories.MacTODO.M3)] public class EnumerationMergeTest : GitRepoTests { // Commit that found GvFlt Bug 12258777: Entries are sometimes skipped during diff --git a/GVFS/GVFS.FunctionalTests/Tests/GitCommands/GitCommandsTests.cs b/GVFS/GVFS.FunctionalTests/Tests/GitCommands/GitCommandsTests.cs index 509cfb03f..f079e5464 100644 --- a/GVFS/GVFS.FunctionalTests/Tests/GitCommands/GitCommandsTests.cs +++ b/GVFS/GVFS.FunctionalTests/Tests/GitCommands/GitCommandsTests.cs @@ -310,7 +310,6 @@ public void DeleteFileWithNameAheadOfDotAndSwitchCommits() } [TestCase] - [Category(Categories.MacTODO.M3)] public void AddFileAndCommitOnNewBranchSwitchDeleteFolderAndSwitchBack() { // 663045 - Confirm that folder can be deleted after adding a file then changing branches @@ -710,7 +709,6 @@ public void AddFileAfterFolderRename() } [TestCase] - [Category(Categories.MacTODO.M3)] public void ResetSoft() { this.ValidateGitCommand("checkout -b tests/functional/ResetSoft"); @@ -742,7 +740,6 @@ public void ManuallyModifyHead() } [TestCase] - [Category(Categories.MacTODO.M3)] public void ResetSoftTwice() { this.ValidateGitCommand("checkout -b tests/functional/ResetSoftTwice"); @@ -754,7 +751,6 @@ public void ResetSoftTwice() } [TestCase] - [Category(Categories.MacTODO.M3)] public void ResetMixedTwice() { this.ValidateGitCommand("checkout -b tests/functional/ResetMixedTwice"); diff --git a/GVFS/GVFS.FunctionalTests/Tests/GitCommands/GitRepoTests.cs b/GVFS/GVFS.FunctionalTests/Tests/GitCommands/GitRepoTests.cs index 39301b73e..932ee1eaa 100644 --- a/GVFS/GVFS.FunctionalTests/Tests/GitCommands/GitRepoTests.cs +++ b/GVFS/GVFS.FunctionalTests/Tests/GitCommands/GitRepoTests.cs @@ -409,7 +409,7 @@ protected void RunFileDirectoryEnumerateTest(string command, string commandBranc protected void RunFileDirectoryReadTest(string command, string commandBranch = DirectoryWithFileAfterBranch) { this.SetupForFileDirectoryTest(commandBranch); - this.FileContentsShouldMatch("file.txt\\file.txt"); + this.FileContentsShouldMatch("file.txt", "file.txt"); this.ValidateFileDirectoryTest(command, commandBranch); } diff --git a/GVFS/GVFS.FunctionalTests/Tests/GitCommands/UpdateRefTests.cs b/GVFS/GVFS.FunctionalTests/Tests/GitCommands/UpdateRefTests.cs index 3ff64c14e..20281ecc8 100644 --- a/GVFS/GVFS.FunctionalTests/Tests/GitCommands/UpdateRefTests.cs +++ b/GVFS/GVFS.FunctionalTests/Tests/GitCommands/UpdateRefTests.cs @@ -4,7 +4,6 @@ namespace GVFS.FunctionalTests.Tests.GitCommands { [TestFixture] [Category(Categories.GitCommands)] - [Category(Categories.MacTODO.M4)] public class UpdateRefTests : GitRepoTests { public UpdateRefTests() : base(enlistmentPerTest: true) From 0c55441362b6c467a58216a8ce107e0dbc249b8b Mon Sep 17 00:00:00 2001 From: Saeed Noursalehi Date: Wed, 5 Sep 2018 15:28:39 -0700 Subject: [PATCH 095/272] Don't pass --no-mount and --no-prefetch flags to clone, because it can hide bugs --- GVFS/GVFS.FunctionalTests/Tools/GVFSFunctionalTestEnlistment.cs | 1 - GVFS/GVFS.FunctionalTests/Tools/GVFSProcess.cs | 2 +- Scripts/Mac/RunFunctionalTests.sh | 2 +- 3 files changed, 2 insertions(+), 3 deletions(-) diff --git a/GVFS/GVFS.FunctionalTests/Tools/GVFSFunctionalTestEnlistment.cs b/GVFS/GVFS.FunctionalTests/Tools/GVFSFunctionalTestEnlistment.cs index d1f1734b3..9ca2bc7e9 100644 --- a/GVFS/GVFS.FunctionalTests/Tools/GVFSFunctionalTestEnlistment.cs +++ b/GVFS/GVFS.FunctionalTests/Tools/GVFSFunctionalTestEnlistment.cs @@ -157,7 +157,6 @@ public void CloneAndMount() { this.gvfsProcess.Clone(this.RepoUrl, this.Commitish); - this.MountGVFS(); GitProcess.Invoke(this.RepoRoot, "checkout " + this.Commitish); GitProcess.Invoke(this.RepoRoot, "branch --unset-upstream"); GitProcess.Invoke(this.RepoRoot, "config core.abbrev 40"); diff --git a/GVFS/GVFS.FunctionalTests/Tools/GVFSProcess.cs b/GVFS/GVFS.FunctionalTests/Tools/GVFSProcess.cs index 1743670e2..b0d6aea85 100644 --- a/GVFS/GVFS.FunctionalTests/Tools/GVFSProcess.cs +++ b/GVFS/GVFS.FunctionalTests/Tools/GVFSProcess.cs @@ -21,7 +21,7 @@ public GVFSProcess(string pathToGVFS, string enlistmentRoot, string localCacheRo public void Clone(string repositorySource, string branchToCheckout) { string args = string.Format( - "clone \"{0}\" \"{1}\" --branch \"{2}\" --no-mount --no-prefetch --local-cache-path \"{3}\"", + "clone \"{0}\" \"{1}\" --branch \"{2}\" --local-cache-path \"{3}\"", repositorySource, this.enlistmentRoot, branchToCheckout, diff --git a/Scripts/Mac/RunFunctionalTests.sh b/Scripts/Mac/RunFunctionalTests.sh index bc84e8a18..627f8a3c7 100755 --- a/Scripts/Mac/RunFunctionalTests.sh +++ b/Scripts/Mac/RunFunctionalTests.sh @@ -15,4 +15,4 @@ sudo mkdir /GVFS.FT sudo chown $USER /GVFS.FT $SRCDIR/ProjFS.Mac/Scripts/LoadPrjFSKext.sh -$PUBLISHDIR/GVFS.FunctionalTests --full-suite +$PUBLISHDIR/GVFS.FunctionalTests --full-suite $2 From e6bd6b1262c87ed7ecddee6c18aaa7a6de4b316d Mon Sep 17 00:00:00 2001 From: Saeed Noursalehi Date: Wed, 5 Sep 2018 21:25:32 -0700 Subject: [PATCH 096/272] Allow the prefetch tests to skip prefetch during clone --- .../PrefetchVerbWithoutSharedCacheTests.cs | 2 +- .../TestsWithEnlistmentPerFixture.cs | 6 ++++-- .../Tools/GVFSFunctionalTestEnlistment.cs | 12 ++++++------ GVFS/GVFS.FunctionalTests/Tools/GVFSProcess.cs | 9 ++++----- 4 files changed, 15 insertions(+), 14 deletions(-) diff --git a/GVFS/GVFS.FunctionalTests/Tests/EnlistmentPerFixture/PrefetchVerbWithoutSharedCacheTests.cs b/GVFS/GVFS.FunctionalTests/Tests/EnlistmentPerFixture/PrefetchVerbWithoutSharedCacheTests.cs index a64bae950..2ee1da2f7 100644 --- a/GVFS/GVFS.FunctionalTests/Tests/EnlistmentPerFixture/PrefetchVerbWithoutSharedCacheTests.cs +++ b/GVFS/GVFS.FunctionalTests/Tests/EnlistmentPerFixture/PrefetchVerbWithoutSharedCacheTests.cs @@ -22,7 +22,7 @@ public class PrefetchVerbWithoutSharedCacheTests : TestsWithEnlistmentPerFixture // Set forcePerRepoObjectCache to true to avoid any of the tests inadvertently corrupting // the cache public PrefetchVerbWithoutSharedCacheTests() - : base(forcePerRepoObjectCache: true) + : base(forcePerRepoObjectCache: true, skipPrefetchDuringClone: true) { this.fileSystem = new SystemIORunner(); } diff --git a/GVFS/GVFS.FunctionalTests/Tests/EnlistmentPerFixture/TestsWithEnlistmentPerFixture.cs b/GVFS/GVFS.FunctionalTests/Tests/EnlistmentPerFixture/TestsWithEnlistmentPerFixture.cs index b8bc7a9c3..fd5d798fd 100644 --- a/GVFS/GVFS.FunctionalTests/Tests/EnlistmentPerFixture/TestsWithEnlistmentPerFixture.cs +++ b/GVFS/GVFS.FunctionalTests/Tests/EnlistmentPerFixture/TestsWithEnlistmentPerFixture.cs @@ -7,10 +7,12 @@ namespace GVFS.FunctionalTests.Tests.EnlistmentPerFixture public abstract class TestsWithEnlistmentPerFixture { private readonly bool forcePerRepoObjectCache; + private readonly bool skipPrefetchDuringClone; - public TestsWithEnlistmentPerFixture(bool forcePerRepoObjectCache = false) + public TestsWithEnlistmentPerFixture(bool forcePerRepoObjectCache = false, bool skipPrefetchDuringClone = false) { this.forcePerRepoObjectCache = forcePerRepoObjectCache; + this.skipPrefetchDuringClone = skipPrefetchDuringClone; } public GVFSFunctionalTestEnlistment Enlistment @@ -23,7 +25,7 @@ public virtual void CreateEnlistment() { if (this.forcePerRepoObjectCache) { - this.Enlistment = GVFSFunctionalTestEnlistment.CloneAndMountWithPerRepoCache(GVFSTestConfig.PathToGVFS); + this.Enlistment = GVFSFunctionalTestEnlistment.CloneAndMountWithPerRepoCache(GVFSTestConfig.PathToGVFS, this.skipPrefetchDuringClone); } else { diff --git a/GVFS/GVFS.FunctionalTests/Tools/GVFSFunctionalTestEnlistment.cs b/GVFS/GVFS.FunctionalTests/Tools/GVFSFunctionalTestEnlistment.cs index 9ca2bc7e9..427fef7b2 100644 --- a/GVFS/GVFS.FunctionalTests/Tools/GVFSFunctionalTestEnlistment.cs +++ b/GVFS/GVFS.FunctionalTests/Tools/GVFSFunctionalTestEnlistment.cs @@ -82,11 +82,11 @@ public string Commitish get; private set; } - public static GVFSFunctionalTestEnlistment CloneAndMountWithPerRepoCache(string pathToGvfs, string commitish = null) + public static GVFSFunctionalTestEnlistment CloneAndMountWithPerRepoCache(string pathToGvfs, bool skipPrefetch) { string enlistmentRoot = GVFSFunctionalTestEnlistment.GetUniqueEnlistmentRoot(); string localCache = GVFSFunctionalTestEnlistment.GetRepoSpecificLocalCacheRoot(enlistmentRoot); - return CloneAndMount(pathToGvfs, enlistmentRoot, commitish, localCache); + return CloneAndMount(pathToGvfs, enlistmentRoot, null, localCache, skipPrefetch); } public static GVFSFunctionalTestEnlistment CloneAndMount(string pathToGvfs, string commitish = null, string localCacheRoot = null) @@ -153,9 +153,9 @@ public void DeleteEnlistment() } } - public void CloneAndMount() + public void CloneAndMount(bool skipPrefetch) { - this.gvfsProcess.Clone(this.RepoUrl, this.Commitish); + this.gvfsProcess.Clone(this.RepoUrl, this.Commitish, skipPrefetch); GitProcess.Invoke(this.RepoRoot, "checkout " + this.Commitish); GitProcess.Invoke(this.RepoRoot, "branch --unset-upstream"); @@ -264,7 +264,7 @@ public string GetObjectPathTo(string objectHash) objectHash.Substring(2)); } - private static GVFSFunctionalTestEnlistment CloneAndMount(string pathToGvfs, string enlistmentRoot, string commitish, string localCacheRoot) + private static GVFSFunctionalTestEnlistment CloneAndMount(string pathToGvfs, string enlistmentRoot, string commitish, string localCacheRoot, bool skipPrefetch = false) { GVFSFunctionalTestEnlistment enlistment = new GVFSFunctionalTestEnlistment( pathToGvfs, @@ -275,7 +275,7 @@ private static GVFSFunctionalTestEnlistment CloneAndMount(string pathToGvfs, str try { - enlistment.CloneAndMount(); + enlistment.CloneAndMount(skipPrefetch); } catch (Exception e) { diff --git a/GVFS/GVFS.FunctionalTests/Tools/GVFSProcess.cs b/GVFS/GVFS.FunctionalTests/Tools/GVFSProcess.cs index b0d6aea85..5bcfd9049 100644 --- a/GVFS/GVFS.FunctionalTests/Tools/GVFSProcess.cs +++ b/GVFS/GVFS.FunctionalTests/Tools/GVFSProcess.cs @@ -1,7 +1,5 @@ using GVFS.Tests.Should; -using System; using System.Diagnostics; -using System.Runtime.InteropServices; namespace GVFS.FunctionalTests.Tools { @@ -18,14 +16,15 @@ public GVFSProcess(string pathToGVFS, string enlistmentRoot, string localCacheRo this.localCacheRoot = localCacheRoot; } - public void Clone(string repositorySource, string branchToCheckout) + public void Clone(string repositorySource, string branchToCheckout, bool skipPrefetch) { string args = string.Format( - "clone \"{0}\" \"{1}\" --branch \"{2}\" --local-cache-path \"{3}\"", + "clone \"{0}\" \"{1}\" --branch \"{2}\" --local-cache-path \"{3}\" {4}", repositorySource, this.enlistmentRoot, branchToCheckout, - this.localCacheRoot); + this.localCacheRoot, + skipPrefetch ? "--no-prefetch" : string.Empty); this.CallGVFS(args, failOnError: true); } From 76eb25ec2c7ffdd6bac0b0c476ca3c5de5940783 Mon Sep 17 00:00:00 2001 From: William Baker Date: Thu, 6 Sep 2018 08:50:14 -0700 Subject: [PATCH 097/272] Enable a subset of the SharedCacheTests --- .../MultiEnlistmentTests/SharedCacheTests.cs | 29 +++++++++++++++---- 1 file changed, 23 insertions(+), 6 deletions(-) diff --git a/GVFS/GVFS.FunctionalTests/Tests/MultiEnlistmentTests/SharedCacheTests.cs b/GVFS/GVFS.FunctionalTests/Tests/MultiEnlistmentTests/SharedCacheTests.cs index 7499293e6..fe0d7784a 100644 --- a/GVFS/GVFS.FunctionalTests/Tests/MultiEnlistmentTests/SharedCacheTests.cs +++ b/GVFS/GVFS.FunctionalTests/Tests/MultiEnlistmentTests/SharedCacheTests.cs @@ -7,6 +7,7 @@ using System.Collections.Generic; using System.IO; using System.Linq; +using System.Runtime.InteropServices; using System.Threading; using System.Threading.Tasks; @@ -14,7 +15,6 @@ namespace GVFS.FunctionalTests.Tests.MultiEnlistmentTests { [TestFixture] [Category(Categories.FullSuiteOnly)] - [Category(Categories.MacTODO.M3)] public class SharedCacheTests : TestsWithMultiEnlistment { private const string WellKnownFile = "Readme.md"; @@ -61,6 +61,7 @@ public void SecondCloneDoesNotDownloadAdditionalObjects() } [TestCase] + [Category(Categories.MacTODO.M4)] public void RepairFixesCorruptBlobSizesDatabase() { GVFSFunctionalTestEnlistment enlistment = this.CloneAndMountEnlistment(); @@ -80,6 +81,7 @@ public void RepairFixesCorruptBlobSizesDatabase() } [TestCase] + [Category(Categories.MacTODO.M4)] public void CloneCleansUpStaleMetadataLock() { GVFSFunctionalTestEnlistment enlistment1 = this.CloneAndMountEnlistment(); @@ -124,6 +126,7 @@ public void ParallelReadsInASharedCache() } [TestCase] + [Category(Categories.MacTODO.M3)] public void DeleteObjectsCacheAndCacheMappingBeforeMount() { GVFSFunctionalTestEnlistment enlistment1 = this.CloneAndMountEnlistment(); @@ -133,7 +136,7 @@ public void DeleteObjectsCacheAndCacheMappingBeforeMount() string objectsRoot = GVFSHelpers.GetPersistedGitObjectsRoot(enlistment1.DotGVFSRoot).ShouldNotBeNull(); objectsRoot.ShouldBeADirectory(this.fileSystem); - CmdRunner.DeleteDirectoryWithUnlimitedRetries(objectsRoot); + this.DeleteDirectoryWithUnlimitedRetries(objectsRoot); string metadataPath = Path.Combine(this.localCachePath, "mapping.dat"); metadataPath.ShouldBeAFile(this.fileSystem); @@ -156,6 +159,7 @@ public void DeleteObjectsCacheAndCacheMappingBeforeMount() } [TestCase] + [Category(Categories.MacTODO.M3)] public void DeleteCacheDuringHydrations() { GVFSFunctionalTestEnlistment enlistment1 = this.CloneAndMountEnlistment(); @@ -173,7 +177,7 @@ public void DeleteCacheDuringHydrations() try { // Delete objectsRoot rather than this.localCachePath as the blob sizes database cannot be deleted while GVFS is mounted - CmdRunner.DeleteDirectoryWithUnlimitedRetries(objectsRoot); + this.DeleteDirectoryWithUnlimitedRetries(objectsRoot); Thread.Sleep(100); } catch (IOException) @@ -213,7 +217,7 @@ public void MountReusesLocalCacheKeyWhenGitObjectsRootDeleted() mappingFileContents.Length.ShouldNotEqual(0, "mapping.dat should not be empty"); // Delete the git objects root folder, mount should re-create it and the mapping.dat file should not change - CmdRunner.DeleteDirectoryWithUnlimitedRetries(objectsRoot); + this.DeleteDirectoryWithUnlimitedRetries(objectsRoot); enlistment.MountGVFS(); @@ -240,7 +244,7 @@ public void MountUsesNewLocalCacheKeyWhenLocalCacheDeleted() mappingFileContents.Length.ShouldNotEqual(0, "mapping.dat should not be empty"); // Delete the local cache folder, mount should re-create it and generate a new mapping file and local cache key - CmdRunner.DeleteDirectoryWithUnlimitedRetries(enlistment.LocalCacheRoot); + this.DeleteDirectoryWithUnlimitedRetries(enlistment.LocalCacheRoot); enlistment.MountGVFS(); @@ -268,7 +272,7 @@ public void MountUsesNewLocalCacheKeyWhenLocalCacheDeleted() // localCacheParentPath can be deleted (as the SQLite blob sizes database cannot be deleted while GVFS is mounted) protected override void OnTearDownEnlistmentsDeleted() { - CmdRunner.DeleteDirectoryWithUnlimitedRetries(this.localCacheParentPath); + this.DeleteDirectoryWithUnlimitedRetries(this.localCacheParentPath); } private GVFSFunctionalTestEnlistment CloneAndMountEnlistment(string branch = null) @@ -303,5 +307,18 @@ private void HydrateEntireRepo(GVFSFunctionalTestEnlistment enlistment) } } } + + public void DeleteDirectoryWithUnlimitedRetries(string path) + { + if (RuntimeInformation.IsOSPlatform(OSPlatform.Windows)) + { + CmdRunner.DeleteDirectoryWithUnlimitedRetries(path); + } + else + { + // TODO(Mac): See if we can use BashRunner.DeleteDirectoryWithRetry on Windows as well + BashRunner.DeleteDirectoryWithUnlimitedRetries(path); + } + } } } From 947a8ce463359868c7d274eb3332adf0ccd18c77 Mon Sep 17 00:00:00 2001 From: William Baker Date: Thu, 6 Sep 2018 09:16:44 -0700 Subject: [PATCH 098/272] Enable ResetHardTests --- GVFS/GVFS.FunctionalTests/Tests/GitCommands/ResetHardTests.cs | 1 - 1 file changed, 1 deletion(-) diff --git a/GVFS/GVFS.FunctionalTests/Tests/GitCommands/ResetHardTests.cs b/GVFS/GVFS.FunctionalTests/Tests/GitCommands/ResetHardTests.cs index 8b3972b16..17ccf772f 100644 --- a/GVFS/GVFS.FunctionalTests/Tests/GitCommands/ResetHardTests.cs +++ b/GVFS/GVFS.FunctionalTests/Tests/GitCommands/ResetHardTests.cs @@ -5,7 +5,6 @@ namespace GVFS.FunctionalTests.Tests.GitCommands { [TestFixture] [Category(Categories.GitCommands)] - [Category(Categories.MacTODO.M3)] public class ResetHardTests : GitRepoTests { private const string ResetHardCommand = "reset --hard"; From 73837291d2aa976a1d5b768c85bafd1427f19ba2 Mon Sep 17 00:00:00 2001 From: William Baker Date: Thu, 6 Sep 2018 09:44:03 -0700 Subject: [PATCH 099/272] Code cleanup and add comments --- GVFS/GVFS.Common/GVFSConstants.cs | 2 +- .../PrefetchVerbWithoutSharedCacheTests.cs | 2 ++ GVFS/GVFS.Virtualization/Projection/GitIndexProjection.cs | 6 +++--- 3 files changed, 6 insertions(+), 4 deletions(-) diff --git a/GVFS/GVFS.Common/GVFSConstants.cs b/GVFS/GVFS.Common/GVFSConstants.cs index 8559eeba9..7e687da1c 100644 --- a/GVFS/GVFS.Common/GVFSConstants.cs +++ b/GVFS/GVFS.Common/GVFSConstants.cs @@ -7,7 +7,7 @@ public static partial class GVFSConstants public const int ShaStringLength = 40; public const int MaxPath = 260; public const string AllZeroSha = "0000000000000000000000000000000000000000"; - public const string ExpandedFolderSha = "XF00000000000000000000000000000000000000"; + public const string ExpandedFolderSha = "X_EXPANDED_FOLDER00000000000000000000000"; public const char GitPathSeparator = '/'; public const string GitPathSeparatorString = "/"; diff --git a/GVFS/GVFS.FunctionalTests/Tests/EnlistmentPerFixture/PrefetchVerbWithoutSharedCacheTests.cs b/GVFS/GVFS.FunctionalTests/Tests/EnlistmentPerFixture/PrefetchVerbWithoutSharedCacheTests.cs index a64bae950..e5419e315 100644 --- a/GVFS/GVFS.FunctionalTests/Tests/EnlistmentPerFixture/PrefetchVerbWithoutSharedCacheTests.cs +++ b/GVFS/GVFS.FunctionalTests/Tests/EnlistmentPerFixture/PrefetchVerbWithoutSharedCacheTests.cs @@ -9,6 +9,8 @@ namespace GVFS.FunctionalTests.Tests.EnlistmentPerFixture { + // TODO(Mac): Before these tests can be enabled PostFetchJobShouldComplete needs + // to work on Mac (where post-fetch.lock is not removed from disk) [TestFixture] [Category(Categories.FullSuiteOnly)] [Category(Categories.MacTODO.M4)] diff --git a/GVFS/GVFS.Virtualization/Projection/GitIndexProjection.cs b/GVFS/GVFS.Virtualization/Projection/GitIndexProjection.cs index 0d506f688..70b42d5ae 100644 --- a/GVFS/GVFS.Virtualization/Projection/GitIndexProjection.cs +++ b/GVFS/GVFS.Virtualization/Projection/GitIndexProjection.cs @@ -1311,9 +1311,9 @@ private void ReExpandFolder( { if (!existingFolderPlaceholders.Contains(childRelativePath)) { - // TODO(Mac): Check return value of WritePlaceholder + // TODO(Mac): Issue #245, handle failures of WritePlaceholder this.fileSystemVirtualizer.WritePlaceholder( - childRelativePath, + childRelativePath, endOfFile: 0, isDirectory: true, shaContentId: GVFSConstants.AllZeroSha); @@ -1332,7 +1332,7 @@ private void ReExpandFolder( FileData childFileData = childEntry as FileData; string sha = childFileData.Sha.ToString(); - // TODO(Mac): Check return value of WritePlaceholder + // TODO(Mac): Issue #245, handle failures of WritePlaceholder this.fileSystemVirtualizer.WritePlaceholder(childRelativePath, childFileData.Size, isDirectory: false, shaContentId: sha); updatedPlaceholderList.TryAdd( From 0d8310c4baf0eb79f8f0784f2b110ba1ed647046 Mon Sep 17 00:00:00 2001 From: William Baker Date: Thu, 6 Sep 2018 09:52:11 -0700 Subject: [PATCH 100/272] Fix StyleCop issues --- .../Tests/MultiEnlistmentTests/SharedCacheTests.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/GVFS/GVFS.FunctionalTests/Tests/MultiEnlistmentTests/SharedCacheTests.cs b/GVFS/GVFS.FunctionalTests/Tests/MultiEnlistmentTests/SharedCacheTests.cs index fe0d7784a..de37e9070 100644 --- a/GVFS/GVFS.FunctionalTests/Tests/MultiEnlistmentTests/SharedCacheTests.cs +++ b/GVFS/GVFS.FunctionalTests/Tests/MultiEnlistmentTests/SharedCacheTests.cs @@ -308,7 +308,7 @@ private void HydrateEntireRepo(GVFSFunctionalTestEnlistment enlistment) } } - public void DeleteDirectoryWithUnlimitedRetries(string path) + private void DeleteDirectoryWithUnlimitedRetries(string path) { if (RuntimeInformation.IsOSPlatform(OSPlatform.Windows)) { From f781b9e79a23fcebf5f725b2cdfc62a7e233963f Mon Sep 17 00:00:00 2001 From: Saeed Noursalehi Date: Thu, 6 Sep 2018 10:08:59 -0700 Subject: [PATCH 101/272] Plumb the internal service name to all GVFS verbs. Otherwise automount during clone talks to the installed service, not the test instance. --- .../MultiEnlistmentTests/ServiceVerbTests.cs | 1 - .../TestsWithMultiEnlistment.cs | 1 - .../GVFS.FunctionalTests/Tools/GVFSProcess.cs | 24 ++++--------------- GVFS/GVFS/CommandLine/GVFSVerb.cs | 1 - 4 files changed, 5 insertions(+), 22 deletions(-) diff --git a/GVFS/GVFS.FunctionalTests/Tests/MultiEnlistmentTests/ServiceVerbTests.cs b/GVFS/GVFS.FunctionalTests/Tests/MultiEnlistmentTests/ServiceVerbTests.cs index 4015f54e0..161a3ab54 100644 --- a/GVFS/GVFS.FunctionalTests/Tests/MultiEnlistmentTests/ServiceVerbTests.cs +++ b/GVFS/GVFS.FunctionalTests/Tests/MultiEnlistmentTests/ServiceVerbTests.cs @@ -1,7 +1,6 @@ using GVFS.FunctionalTests.Tools; using GVFS.Tests.Should; using NUnit.Framework; -using System.IO; namespace GVFS.FunctionalTests.Tests.MultiEnlistmentTests { diff --git a/GVFS/GVFS.FunctionalTests/Tests/MultiEnlistmentTests/TestsWithMultiEnlistment.cs b/GVFS/GVFS.FunctionalTests/Tests/MultiEnlistmentTests/TestsWithMultiEnlistment.cs index 5becccdcb..f0afd9399 100644 --- a/GVFS/GVFS.FunctionalTests/Tests/MultiEnlistmentTests/TestsWithMultiEnlistment.cs +++ b/GVFS/GVFS.FunctionalTests/Tests/MultiEnlistmentTests/TestsWithMultiEnlistment.cs @@ -1,7 +1,6 @@ using GVFS.FunctionalTests.Tools; using NUnit.Framework; using System.Collections.Generic; -using System.IO; namespace GVFS.FunctionalTests.Tests.MultiEnlistmentTests { diff --git a/GVFS/GVFS.FunctionalTests/Tools/GVFSProcess.cs b/GVFS/GVFS.FunctionalTests/Tools/GVFSProcess.cs index 5bcfd9049..51f466bb4 100644 --- a/GVFS/GVFS.FunctionalTests/Tools/GVFSProcess.cs +++ b/GVFS/GVFS.FunctionalTests/Tools/GVFSProcess.cs @@ -37,10 +37,8 @@ public void Mount() public bool TryMount(out string output) { - string mountCommand = "mount \"" + this.enlistmentRoot + "\" --internal_use_only_service_name " + GVFSServiceProcess.TestServiceName; - this.IsEnlistmentMounted().ShouldEqual(false, "GVFS is already mounted"); - output = this.CallGVFS(mountCommand); + output = this.CallGVFS("mount \"" + this.enlistmentRoot + "\""); return this.IsEnlistmentMounted(); } @@ -58,11 +56,7 @@ public void Repair() public string Diagnose() { - string diagnoseArgs = string.Join( - " ", - "diagnose \"" + this.enlistmentRoot + "\"", - "--internal_use_only_service_name " + GVFSServiceProcess.TestServiceName); - return this.CallGVFS(diagnoseArgs); + return this.CallGVFS("diagnose \"" + this.enlistmentRoot + "\""); } public string Status() @@ -79,11 +73,7 @@ public void Unmount() { if (this.IsEnlistmentMounted()) { - string unmountArgs = string.Join( - " ", - "unmount \"" + this.enlistmentRoot + "\"", - "--internal_use_only_service_name " + GVFSServiceProcess.TestServiceName); - string result = this.CallGVFS(unmountArgs, failOnError: true); + string result = this.CallGVFS("unmount \"" + this.enlistmentRoot + "\"", failOnError: true); this.IsEnlistmentMounted().ShouldEqual(false, "GVFS did not unmount: " + result); } } @@ -96,18 +86,14 @@ public bool IsEnlistmentMounted() public string RunServiceVerb(string argument) { - string serviceVerbArgs = string.Join( - " ", - "service " + argument, - "--internal_use_only_service_name " + GVFSServiceProcess.TestServiceName); - return this.CallGVFS(serviceVerbArgs, failOnError: true); + return this.CallGVFS("service " + argument, failOnError: true); } private string CallGVFS(string args, bool failOnError = false) { ProcessStartInfo processInfo = null; processInfo = new ProcessStartInfo(this.pathToGVFS); - processInfo.Arguments = args; + processInfo.Arguments = args + " --internal_use_only_service_name " + GVFSServiceProcess.TestServiceName; processInfo.WindowStyle = ProcessWindowStyle.Hidden; processInfo.UseShellExecute = false; diff --git a/GVFS/GVFS/CommandLine/GVFSVerb.cs b/GVFS/GVFS/CommandLine/GVFSVerb.cs index 945682c40..dbb9440d6 100644 --- a/GVFS/GVFS/CommandLine/GVFSVerb.cs +++ b/GVFS/GVFS/CommandLine/GVFSVerb.cs @@ -7,7 +7,6 @@ using GVFS.Common.Tracing; using System; using System.Collections.Generic; -using System.Diagnostics; using System.IO; using System.Linq; using System.Security; From 3b16d1ec91af0cdbf09d3d48f4b9c58bac6334f0 Mon Sep 17 00:00:00 2001 From: William Baker Date: Thu, 6 Sep 2018 12:06:57 -0700 Subject: [PATCH 102/272] Increase maximum number of roots to 128 --- ProjFS.Mac/PrjFSKext/PrjFSKext/VirtualizationRoots.cpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ProjFS.Mac/PrjFSKext/PrjFSKext/VirtualizationRoots.cpp b/ProjFS.Mac/PrjFSKext/PrjFSKext/VirtualizationRoots.cpp index 48c98edb8..d37281361 100644 --- a/ProjFS.Mac/PrjFSKext/PrjFSKext/VirtualizationRoots.cpp +++ b/ProjFS.Mac/PrjFSKext/PrjFSKext/VirtualizationRoots.cpp @@ -16,7 +16,7 @@ static RWLock s_rwLock = {}; // Arbitrary choice, but prevents user space attacker from causing // allocation of too much wired kernel memory. -static const size_t MaxVirtualizationRoots = 64; +static const size_t MaxVirtualizationRoots = 128; static VirtualizationRoot s_virtualizationRoots[MaxVirtualizationRoots] = {}; From 4674617f001b88991e988c571041c96a52559093 Mon Sep 17 00:00:00 2001 From: William Baker Date: Thu, 6 Sep 2018 12:53:34 -0700 Subject: [PATCH 103/272] Add NeedsCachePoisonFix test category --- GVFS/GVFS.FunctionalTests/Categories.cs | 4 ++-- GVFS/GVFS.FunctionalTests/Program.cs | 1 + .../Tests/GitCommands/CheckoutTests.cs | 8 +++----- 3 files changed, 6 insertions(+), 7 deletions(-) diff --git a/GVFS/GVFS.FunctionalTests/Categories.cs b/GVFS/GVFS.FunctionalTests/Categories.cs index fe837c42b..181d671ad 100644 --- a/GVFS/GVFS.FunctionalTests/Categories.cs +++ b/GVFS/GVFS.FunctionalTests/Categories.cs @@ -14,8 +14,8 @@ public static class MacTODO // The FailsOnBuildAgent category is for tests that pass on dev // machines but not on the build agents public const string FailsOnBuildAgent = "FailsOnBuildAgent"; - - public const string NeedsLockHolder = "NeedsDotCoreLockHolder"; + public const string NeedsLockHolder = "NeedsDotCoreLockHolder"; + public const string NeedsCachePoisonFix = "NeedsCachePoisonFix"; public const string M2 = "M2_StaticViewGitCommands"; public const string M3 = "M3_AllGitCommands"; public const string M4 = "M4_All"; diff --git a/GVFS/GVFS.FunctionalTests/Program.cs b/GVFS/GVFS.FunctionalTests/Program.cs index b467b28d5..a9cf972c0 100644 --- a/GVFS/GVFS.FunctionalTests/Program.cs +++ b/GVFS/GVFS.FunctionalTests/Program.cs @@ -67,6 +67,7 @@ public static void Main(string[] args) { excludeCategories.Add(Categories.MacTODO.NeedsLockHolder); excludeCategories.Add(Categories.MacTODO.FailsOnBuildAgent); + excludeCategories.Add(Categories.MacTODO.NeedsCachePoisonFix); excludeCategories.Add(Categories.MacTODO.M2); excludeCategories.Add(Categories.MacTODO.M3); excludeCategories.Add(Categories.MacTODO.M4); diff --git a/GVFS/GVFS.FunctionalTests/Tests/GitCommands/CheckoutTests.cs b/GVFS/GVFS.FunctionalTests/Tests/GitCommands/CheckoutTests.cs index 0208cd188..4c5518c0d 100644 --- a/GVFS/GVFS.FunctionalTests/Tests/GitCommands/CheckoutTests.cs +++ b/GVFS/GVFS.FunctionalTests/Tests/GitCommands/CheckoutTests.cs @@ -87,9 +87,8 @@ private enum NativeFileAccess : uint GENERIC_READ = 2147483648 } - // WIP: This test occasionally fails [TestCase] - [Category(Categories.MacTODO.M3)] + [Category(Categories.MacTODO.NeedsCachePoisonFix)] public void ReadDeepFilesAfterCheckout() { // In commit 8df701986dea0a5e78b742d2eaf9348825b14d35 the CheckoutNewBranchFromStartingPointTest files were not present @@ -104,9 +103,8 @@ public void ReadDeepFilesAfterCheckout() this.ValidateGitCommand("status"); } - // WIP: This test occasionally fails [TestCase] - [Category(Categories.MacTODO.M3)] + [Category(Categories.MacTODO.NeedsCachePoisonFix)] public void CheckoutNewBranchFromStartingPointTest() { // In commit 8df701986dea0a5e78b742d2eaf9348825b14d35 the CheckoutNewBranchFromStartingPointTest files were not present @@ -123,7 +121,7 @@ public void CheckoutNewBranchFromStartingPointTest() } [TestCase] - [Category(Categories.MacTODO.M3)] + [Category(Categories.MacTODO.NeedsCachePoisonFix)] public void CheckoutOrhpanBranchFromStartingPointTest() { // In commit 8df701986dea0a5e78b742d2eaf9348825b14d35 the CheckoutOrhpanBranchFromStartingPointTest files were not present From 804effc3e7615329465ec791d74852311a127b83 Mon Sep 17 00:00:00 2001 From: nicrd Date: Sun, 9 Sep 2018 16:49:16 -0400 Subject: [PATCH 104/272] Fix console logging typo in MirrorProvider --- .../MirrorProvider.Windows/WindowsFileSystemVirtualizer.cs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/MirrorProvider/MirrorProvider.Windows/WindowsFileSystemVirtualizer.cs b/MirrorProvider/MirrorProvider.Windows/WindowsFileSystemVirtualizer.cs index 5100884cf..aebdcd7ba 100644 --- a/MirrorProvider/MirrorProvider.Windows/WindowsFileSystemVirtualizer.cs +++ b/MirrorProvider/MirrorProvider.Windows/WindowsFileSystemVirtualizer.cs @@ -317,7 +317,7 @@ private void OnFileModifiedOrDeleted(string relativePath, bool isDirectory, bool // NotificationType.FileHandleClosedFileModified and so this method will only be called for modifications. // Once MacFileSystemVirtualizer supports delete notifications we'll register for // NotificationType.FileHandleClosedFileDeleted and this method will be called for both modifications and deletions. - Console.WriteLine($"OnFileModifiedOrDeleted: `{relativePath}`, isDirectory: {isDirectory}, isModfied: {isFileDeleted}, isDeleted: {isFileDeleted}"); + Console.WriteLine($"OnFileModifiedOrDeleted: `{relativePath}`, isDirectory: {isDirectory}, isModfied: {isFileModified}, isDeleted: {isFileDeleted}"); } private void OnFileRenamed( @@ -345,4 +345,4 @@ private static HResult HResultFromWin32(int win32error) return win32error <= 0 ? (HResult)win32error : (HResult)unchecked((win32error & 0x0000FFFF) | (FacilityWin32 << 16) | 0x80000000); } } -} \ No newline at end of file +} From 67a53a18afd7c18e47c5e8c69f9a8ab63b2434c4 Mon Sep 17 00:00:00 2001 From: William Baker Date: Fri, 7 Sep 2018 09:17:38 -0700 Subject: [PATCH 105/272] Changes for first round of PR feedback --- GVFS/GVFS.Common/FileSystem/IKernelDriver.cs | 2 +- .../FileSystem/IPlatformFileSystem.cs | 1 - GVFS/GVFS.Common/GVFSConstants.cs | 1 - GVFS/GVFS.Common/PlaceholderListDatabase.cs | 104 +++++++++--- GVFS/GVFS.Common/RepoMetadata.cs | 2 +- .../Windows/Tests/DiskLayoutUpgradeTests.cs | 152 +++++++++++------ .../MultithreadedReadWriteTests.cs | 2 +- .../Tests/GitCommands/CheckoutTests.cs | 6 +- .../GVFS.FunctionalTests/Tools/GVFSHelpers.cs | 6 +- .../Tools/TestConstants.cs | 1 + GVFS/GVFS.Platform.Mac/MacFileSystem.cs | 1 - .../MacFileSystemVirtualizer.cs | 51 +++--- GVFS/GVFS.Platform.Mac/ProjFSKext.cs | 2 + ...ut16to17Upgrade_FolderPlaceholderValues.cs | 75 +++++++++ ...BackgroundAndPlaceholderListToFileBased.cs | 2 +- .../WindowsDiskLayoutUpgradeData.cs | 1 + .../GVFS.Platform.Windows.csproj | 1 + GVFS/GVFS.Platform.Windows/ProjFSFilter.cs | 1 + .../WindowsFileSystem.cs | 1 - .../WindowsFileSystemVirtualizer.cs | 52 +++--- .../Should/EnumerableShouldExtensions.cs | 2 +- .../Common/PlaceholderDatabaseTests.cs | 50 ++++-- .../Mock/FileSystem/MockPlatformFileSystem.cs | 4 - .../FileSystem/MockFileSystemVirtualizer.cs | 7 +- .../FileSystem/FileSystemVirtualizer.cs | 9 +- .../Projection/GitIndexProjection.cs | 153 ++++++++++-------- 26 files changed, 472 insertions(+), 217 deletions(-) create mode 100644 GVFS/GVFS.Platform.Windows/DiskLayoutUpgrades/DiskLayout16to17Upgrade_FolderPlaceholderValues.cs diff --git a/GVFS/GVFS.Common/FileSystem/IKernelDriver.cs b/GVFS/GVFS.Common/FileSystem/IKernelDriver.cs index f10aba1f7..270019da4 100644 --- a/GVFS/GVFS.Common/FileSystem/IKernelDriver.cs +++ b/GVFS/GVFS.Common/FileSystem/IKernelDriver.cs @@ -4,8 +4,8 @@ namespace GVFS.Common.FileSystem { public interface IKernelDriver { + bool EnumerationExpandsDirectories { get; } string DriverLogFolderName { get; } - bool IsSupported(string normalizedEnlistmentRootPath, out string warning, out string error); string FlushDriverLogs(); bool TryPrepareFolderForCallbacks(string folderPath, out string error); diff --git a/GVFS/GVFS.Common/FileSystem/IPlatformFileSystem.cs b/GVFS/GVFS.Common/FileSystem/IPlatformFileSystem.cs index 9e7d44174..93c8cb2e9 100644 --- a/GVFS/GVFS.Common/FileSystem/IPlatformFileSystem.cs +++ b/GVFS/GVFS.Common/FileSystem/IPlatformFileSystem.cs @@ -3,7 +3,6 @@ public interface IPlatformFileSystem { bool SupportsFileMode { get; } - bool EnumerationExpandsDirectories { get; } void FlushFileBuffers(string path); void MoveAndOverwriteFile(string sourceFileName, string destinationFilename); void CreateHardLink(string newLinkFileName, string existingFileName); diff --git a/GVFS/GVFS.Common/GVFSConstants.cs b/GVFS/GVFS.Common/GVFSConstants.cs index 7e687da1c..ece894ef8 100644 --- a/GVFS/GVFS.Common/GVFSConstants.cs +++ b/GVFS/GVFS.Common/GVFSConstants.cs @@ -7,7 +7,6 @@ public static partial class GVFSConstants public const int ShaStringLength = 40; public const int MaxPath = 260; public const string AllZeroSha = "0000000000000000000000000000000000000000"; - public const string ExpandedFolderSha = "X_EXPANDED_FOLDER00000000000000000000000"; public const char GitPathSeparator = '/'; public const string GitPathSeparatorString = "/"; diff --git a/GVFS/GVFS.Common/PlaceholderListDatabase.cs b/GVFS/GVFS.Common/PlaceholderListDatabase.cs index 73dabaea4..cf63018e5 100644 --- a/GVFS/GVFS.Common/PlaceholderListDatabase.cs +++ b/GVFS/GVFS.Common/PlaceholderListDatabase.cs @@ -8,7 +8,12 @@ namespace GVFS.Common { public class PlaceholderListDatabase : FileBasedCollection { + public const string PartialFolderValue = "@FPARTIAL_000000000000000000000000000000"; + public const string ExpandedFolderValue = "@FEXPANDED_00000000000000000000000000000"; + private const char PathTerminator = '\0'; + private const string FolderValuePrefix = "@F"; + private const string ExpandedFolderPrefix = "@FE"; // This list holds entries that would otherwise be lost because WriteAllEntriesAndFlush has not been called, but a file // snapshot has been taken using GetAllEntries. @@ -51,19 +56,29 @@ public static bool TryCreate(ITracer tracer, string dataFilePath, PhysicalFileSy output = temp; return true; } - - public void AddAndFlush(string path, string sha) + + public void AddAndFlushFile(string path, string sha) + { + this.AddAndFlush(path, sha); + } + + public void AddAndFlushFolder(string path, bool isExpanded) + { + this.AddAndFlush(path, isExpanded ? ExpandedFolderValue : PartialFolderValue); + } + + public void RemoveAndFlush(string path) { try { - this.WriteAddEntry( - path + PathTerminator + sha, + this.WriteRemoveEntry( + path, () => { - this.EstimatedCount++; + this.EstimatedCount--; if (this.placeholderDataEntries != null) { - this.placeholderDataEntries.Add(new PlaceholderDataEntry(path, sha)); + this.placeholderDataEntries.Add(new PlaceholderDataEntry(path)); } }); } @@ -73,20 +88,32 @@ public void AddAndFlush(string path, string sha) } } - public void RemoveAndFlush(string path) + public List GetAllEntries() { try { - this.WriteRemoveEntry( - path, + List placeholders = new List(Math.Max(1, this.EstimatedCount)); + + string error; + if (!this.TryLoadFromDisk( + this.TryParseAddLine, + this.TryParseRemoveLine, + (key, value) => placeholders.Add(new PlaceholderData(path: key, fileShaOrFolderValue: value)), + out error, () => { - this.EstimatedCount--; if (this.placeholderDataEntries != null) { - this.placeholderDataEntries.Add(new PlaceholderDataEntry(path)); + throw new InvalidOperationException("PlaceholderListDatabase should always flush queue placeholders using WriteAllEntriesAndFlush before calling GetAllEntries again."); } - }); + + this.placeholderDataEntries = new List(); + })) + { + throw new InvalidDataException(error); + } + + return placeholders; } catch (Exception e) { @@ -94,17 +121,28 @@ public void RemoveAndFlush(string path) } } - public List GetAllEntries() + public void GetAllEntries(out List filePlaceholders, out List folderPlaceholders) { try { - List output = new List(Math.Max(1, this.EstimatedCount)); + List filePlaceholdersFromDisk = new List(Math.Max(1, this.EstimatedCount)); + List folderPlaceholdersFromDisk = new List(Math.Max(1, (int)(this.EstimatedCount * .3))); string error; if (!this.TryLoadFromDisk( this.TryParseAddLine, this.TryParseRemoveLine, - (key, value) => output.Add(new PlaceholderData(path: key, sha: value)), + (key, value) => + { + if (value.StartsWith(FolderValuePrefix, StringComparison.Ordinal)) + { + folderPlaceholdersFromDisk.Add(new PlaceholderData(path: key, fileShaOrFolderValue: value)); + } + else + { + filePlaceholdersFromDisk.Add(new PlaceholderData(path: key, fileShaOrFolderValue: value)); + } + }, out error, () => { @@ -119,7 +157,8 @@ public List GetAllEntries() throw new InvalidDataException(error); } - return output; + filePlaceholders = filePlaceholdersFromDisk; + folderPlaceholders = folderPlaceholdersFromDisk; } catch (Exception e) { @@ -181,6 +220,27 @@ private IEnumerable GenerateDataLines(IEnumerable updat } } + private void AddAndFlush(string path, string sha) + { + try + { + this.WriteAddEntry( + path + PathTerminator + sha, + () => + { + this.EstimatedCount++; + if (this.placeholderDataEntries != null) + { + this.placeholderDataEntries.Add(new PlaceholderDataEntry(path, sha)); + } + }); + } + catch (Exception e) + { + throw new FileBasedCollectionException(e); + } + } + private bool TryParseAddLine(string line, out string key, out string value, out string error) { // Expected: \0<40-Char-SHA1> @@ -197,7 +257,7 @@ private bool TryParseAddLine(string line, out string key, out string value, out { key = null; value = null; - error = "Invalid SHA1 length: " + line; + error = $"Invalid SHA1 length {line.Length - idx - 1}: " + line; return false; } @@ -218,10 +278,10 @@ private bool TryParseRemoveLine(string line, out string key, out string error) public class PlaceholderData { - public PlaceholderData(string path, string sha) + public PlaceholderData(string path, string fileShaOrFolderValue) { this.Path = path; - this.Sha = sha; + this.Sha = fileShaOrFolderValue; } public string Path { get; } @@ -230,8 +290,8 @@ public PlaceholderData(string path, string sha) public bool IsFolder { get - { - return this.Sha == GVFSConstants.AllZeroSha || this.IsExpandedFolder; + { + return this.Sha.StartsWith(FolderValuePrefix, StringComparison.Ordinal); } } @@ -239,7 +299,7 @@ public bool IsExpandedFolder { get { - return this.Sha == GVFSConstants.ExpandedFolderSha; + return this.Sha.StartsWith(ExpandedFolderPrefix, StringComparison.Ordinal); } } } diff --git a/GVFS/GVFS.Common/RepoMetadata.cs b/GVFS/GVFS.Common/RepoMetadata.cs index db28c223d..eeefb78dc 100644 --- a/GVFS/GVFS.Common/RepoMetadata.cs +++ b/GVFS/GVFS.Common/RepoMetadata.cs @@ -316,7 +316,7 @@ public static class DiskLayoutVersion // The major version should be bumped whenever there is an on-disk format change that requires a one-way upgrade. // Increasing this version will make older versions of GVFS unable to mount a repo that has been mounted by a newer // version of GVFS. - public const int CurrentMajorVersion = 16; + public const int CurrentMajorVersion = 17; // The minor version should be bumped whenever there is an upgrade that can be safely ignored by older versions of GVFS. // For example, this allows an upgrade step that sets a default value for some new config setting. diff --git a/GVFS/GVFS.FunctionalTests.Windows/Windows/Tests/DiskLayoutUpgradeTests.cs b/GVFS/GVFS.FunctionalTests.Windows/Windows/Tests/DiskLayoutUpgradeTests.cs index 6495a9cd6..2d407de3b 100644 --- a/GVFS/GVFS.FunctionalTests.Windows/Windows/Tests/DiskLayoutUpgradeTests.cs +++ b/GVFS/GVFS.FunctionalTests.Windows/Windows/Tests/DiskLayoutUpgradeTests.cs @@ -17,7 +17,7 @@ namespace GVFS.FunctionalTests.Windows.Tests [Category(Categories.WindowsOnly)] public class DiskLayoutUpgradeTests : TestsWithEnlistmentPerTestCase { - public const int CurrentDiskLayoutMajorVersion = 16; + public const int CurrentDiskLayoutMajorVersion = 17; public const int CurrentDiskLayoutMinorVersion = 0; public const string BlobSizesCacheName = "blobSizes"; @@ -153,34 +153,34 @@ public void MountSucceedsIfMinorVersionHasAdvancedButNotMajorVersion() [TestCase] public void MountWritesFolderPlaceholdersToPlaceholderDatabase() { - // Create some placeholder data - this.fileSystem.ReadAllText(Path.Combine(this.Enlistment.RepoRoot, "Readme.md")); - this.fileSystem.ReadAllText(Path.Combine(this.Enlistment.RepoRoot, "Scripts\\RunUnitTests.bat")); - this.fileSystem.ReadAllText(Path.Combine(this.Enlistment.RepoRoot, "GVFS\\GVFS.Common\\Git\\GitRefs.cs")); + this.PerformIOBeforePlaceholderDatabaseUpgradeTest(); - // Create a full folder - this.fileSystem.CreateDirectory(Path.Combine(this.Enlistment.RepoRoot, "GVFS\\FullFolder")); - this.fileSystem.WriteAllText(Path.Combine(this.Enlistment.RepoRoot, "GVFS\\FullFolder\\test.txt"), "Test contents"); + this.Enlistment.UnmountGVFS(); - // Create a tombstone - this.fileSystem.DeleteDirectory(Path.Combine(this.Enlistment.RepoRoot, "GVFS\\GVFS.Tests\\Properties")); + // Delete the existing folder placeholder data + string placeholderDatabasePath = Path.Combine(this.Enlistment.DotGVFSRoot, GVFSHelpers.PlaceholderListFile); + string[] lines = this.GetPlaceholderDatabaseLinesBeforeUpgrade(placeholderDatabasePath); - string junctionTarget = Path.Combine(this.Enlistment.EnlistmentRoot, "DirJunction"); - string symlinkTarget = Path.Combine(this.Enlistment.EnlistmentRoot, "DirSymlink"); - Directory.CreateDirectory(junctionTarget); - Directory.CreateDirectory(symlinkTarget); + // Placeholder database file should only have file placeholders + this.fileSystem.WriteAllText( + placeholderDatabasePath, + string.Join(Environment.NewLine, lines.Where(x => !x.EndsWith(TestConstants.PartialFolderPlaceholderDatabaseValue))) + Environment.NewLine); - string junctionLink = Path.Combine(this.Enlistment.RepoRoot, "DirJunction"); - string symlink = Path.Combine(this.Enlistment.RepoRoot, "DirLink"); - ProcessHelper.Run("CMD.exe", "/C mklink /J " + junctionLink + " " + junctionTarget); - ProcessHelper.Run("CMD.exe", "/C mklink /D " + symlink + " " + symlinkTarget); + GVFSHelpers.SaveDiskLayoutVersion(this.Enlistment.DotGVFSRoot, "12", "1"); - string target = Path.Combine(this.Enlistment.EnlistmentRoot, "GVFS", "GVFS", "GVFS.UnitTests"); - string link = Path.Combine(this.Enlistment.RepoRoot, "UnitTests"); - ProcessHelper.Run("CMD.exe", "/C mklink /J " + link + " " + target); - target = Path.Combine(this.Enlistment.EnlistmentRoot, "GVFS", "GVFS", "GVFS.Installer"); - link = Path.Combine(this.Enlistment.RepoRoot, "Installer"); - ProcessHelper.Run("CMD.exe", "/C mklink /D " + link + " " + target); + this.Enlistment.MountGVFS(); + this.Enlistment.UnmountGVFS(); + + // Validate the folder placeholders are in the placeholder database now + this.GetPlaceholderDatabaseLinesAfterUpgradeFrom12_1(placeholderDatabasePath); + + this.ValidatePersistedVersionMatchesCurrentVersion(); + } + + [TestCase] + public void MountUpdatesAllZeroShaFolderPlaceholderEntriesToPartialFolderSpecialValue() + { + this.PerformIOBeforePlaceholderDatabaseUpgradeTest(); this.Enlistment.UnmountGVFS(); @@ -188,16 +188,22 @@ public void MountWritesFolderPlaceholdersToPlaceholderDatabase() string placeholderDatabasePath = Path.Combine(this.Enlistment.DotGVFSRoot, GVFSHelpers.PlaceholderListFile); string[] lines = this.GetPlaceholderDatabaseLinesBeforeUpgrade(placeholderDatabasePath); - // Placeholder database file should only have file placeholders - this.fileSystem.WriteAllText(placeholderDatabasePath, string.Join(Environment.NewLine, lines.Where(x => !x.EndsWith(TestConstants.AllZeroSha))) + Environment.NewLine); + // Update the placeholder file so that folders have an all zero SHA + this.fileSystem.WriteAllText( + placeholderDatabasePath, + string.Join( + Environment.NewLine, + lines.Select(x => x.Replace(TestConstants.PartialFolderPlaceholderDatabaseValue, TestConstants.AllZeroSha))) + Environment.NewLine); - GVFSHelpers.SaveDiskLayoutVersion(this.Enlistment.DotGVFSRoot, "12", "1"); + GVFSHelpers.SaveDiskLayoutVersion(this.Enlistment.DotGVFSRoot, "16", "0"); this.Enlistment.MountGVFS(); this.Enlistment.UnmountGVFS(); - // Validate the folder placeholders are in the placeholder database now - this.GetPlaceholderDatabaseLinesAfterUpgrade(placeholderDatabasePath); + // Validate the folder placeholders in the database have PartialFolderPlaceholderDatabaseValue values + this.GetPlaceholderDatabaseLinesAfterUpgradeFrom16(placeholderDatabasePath); + + this.ValidatePersistedVersionMatchesCurrentVersion(); } [TestCase] @@ -232,17 +238,7 @@ public void MountUpgradesPreSharedCacheLocalSizes() this.Enlistment.MountGVFS(); - string majorVersion; - string minorVersion; - GVFSHelpers.GetPersistedDiskLayoutVersion(this.Enlistment.DotGVFSRoot, out majorVersion, out minorVersion); - - majorVersion - .ShouldBeAnInt("Disk layout version should always be an int") - .ShouldEqual(CurrentDiskLayoutMajorVersion, "Disk layout version should be upgraded to the latest"); - - minorVersion - .ShouldBeAnInt("Disk layout version should always be an int") - .ShouldEqual(CurrentDiskLayoutMinorVersion, "Disk layout version should be upgraded to the latest"); + this.ValidatePersistedVersionMatchesCurrentVersion(); GVFSHelpers.GetPersistedLocalCacheRoot(this.Enlistment.DotGVFSRoot) .ShouldEqual(string.Empty, "LocalCacheRoot should be an empty string when upgrading from a version prior to 12"); @@ -335,6 +331,40 @@ A tools/perllib/MS/Somefile.txt "; modifiedPathsDatabasePath.ShouldBeAFile(this.fileSystem).WithContents(expectedModifiedPaths); + + this.ValidatePersistedVersionMatchesCurrentVersion(); + } + + private void PerformIOBeforePlaceholderDatabaseUpgradeTest() + { + // Create some placeholder data + this.fileSystem.ReadAllText(Path.Combine(this.Enlistment.RepoRoot, "Readme.md")); + this.fileSystem.ReadAllText(Path.Combine(this.Enlistment.RepoRoot, "Scripts\\RunUnitTests.bat")); + this.fileSystem.ReadAllText(Path.Combine(this.Enlistment.RepoRoot, "GVFS\\GVFS.Common\\Git\\GitRefs.cs")); + + // Create a full folder + this.fileSystem.CreateDirectory(Path.Combine(this.Enlistment.RepoRoot, "GVFS\\FullFolder")); + this.fileSystem.WriteAllText(Path.Combine(this.Enlistment.RepoRoot, "GVFS\\FullFolder\\test.txt"), "Test contents"); + + // Create a tombstone + this.fileSystem.DeleteDirectory(Path.Combine(this.Enlistment.RepoRoot, "GVFS\\GVFS.Tests\\Properties")); + + string junctionTarget = Path.Combine(this.Enlistment.EnlistmentRoot, "DirJunction"); + string symlinkTarget = Path.Combine(this.Enlistment.EnlistmentRoot, "DirSymlink"); + Directory.CreateDirectory(junctionTarget); + Directory.CreateDirectory(symlinkTarget); + + string junctionLink = Path.Combine(this.Enlistment.RepoRoot, "DirJunction"); + string symlink = Path.Combine(this.Enlistment.RepoRoot, "DirLink"); + ProcessHelper.Run("CMD.exe", "/C mklink /J " + junctionLink + " " + junctionTarget); + ProcessHelper.Run("CMD.exe", "/C mklink /D " + symlink + " " + symlinkTarget); + + string target = Path.Combine(this.Enlistment.EnlistmentRoot, "GVFS", "GVFS", "GVFS.UnitTests"); + string link = Path.Combine(this.Enlistment.RepoRoot, "UnitTests"); + ProcessHelper.Run("CMD.exe", "/C mklink /J " + link + " " + target); + target = Path.Combine(this.Enlistment.EnlistmentRoot, "GVFS", "GVFS", "GVFS.Installer"); + link = Path.Combine(this.Enlistment.RepoRoot, "Installer"); + ProcessHelper.Run("CMD.exe", "/C mklink /D " + link + " " + target); } private string[] GetPlaceholderDatabaseLinesBeforeUpgrade(string placeholderDatabasePath) @@ -348,16 +378,16 @@ private string[] GetPlaceholderDatabaseLinesBeforeUpgrade(string placeholderData lines.ShouldContain(x => x.Contains("A GVFS\\GVFS.Tests\\Properties\\AssemblyInfo.cs")); lines.ShouldContain(x => x.Contains("A .gitignore")); lines.ShouldContain(x => x == "D GVFS\\GVFS.Tests\\Properties\\AssemblyInfo.cs"); - lines.ShouldContain(x => x == "A Scripts\0" + TestConstants.AllZeroSha); - lines.ShouldContain(x => x == "A GVFS\0" + TestConstants.AllZeroSha); - lines.ShouldContain(x => x == "A GVFS\\GVFS.Common\0" + TestConstants.AllZeroSha); - lines.ShouldContain(x => x == "A GVFS\\GVFS.Common\\Git\0" + TestConstants.AllZeroSha); - lines.ShouldContain(x => x == "A GVFS\\GVFS.Tests\0" + TestConstants.AllZeroSha); - lines.ShouldContain(x => x == "A GVFS\\GVFS.Tests\\Properties\0" + TestConstants.AllZeroSha); + lines.ShouldContain(x => x == "A Scripts\0" + TestConstants.PartialFolderPlaceholderDatabaseValue); + lines.ShouldContain(x => x == "A GVFS\0" + TestConstants.PartialFolderPlaceholderDatabaseValue); + lines.ShouldContain(x => x == "A GVFS\\GVFS.Common\0" + TestConstants.PartialFolderPlaceholderDatabaseValue); + lines.ShouldContain(x => x == "A GVFS\\GVFS.Common\\Git\0" + TestConstants.PartialFolderPlaceholderDatabaseValue); + lines.ShouldContain(x => x == "A GVFS\\GVFS.Tests\0" + TestConstants.PartialFolderPlaceholderDatabaseValue); + lines.ShouldContain(x => x == "A GVFS\\GVFS.Tests\\Properties\0" + TestConstants.PartialFolderPlaceholderDatabaseValue); return lines; } - private string[] GetPlaceholderDatabaseLinesAfterUpgrade(string placeholderDatabasePath) + private string[] GetPlaceholderDatabaseLinesAfterUpgradeFrom12_1(string placeholderDatabasePath) { placeholderDatabasePath.ShouldBeAFile(this.fileSystem); string[] lines = this.fileSystem.ReadAllText(placeholderDatabasePath).Split(new char[] { '\r', '\n' }, StringSplitOptions.RemoveEmptyEntries); @@ -366,11 +396,29 @@ private string[] GetPlaceholderDatabaseLinesAfterUpgrade(string placeholderDatab lines.ShouldContain(x => x.Contains("Scripts\\RunUnitTests.bat")); lines.ShouldContain(x => x.Contains("GVFS\\GVFS.Common\\Git\\GitRefs.cs")); lines.ShouldContain(x => x.Contains("A .gitignore")); - lines.ShouldContain(x => x == "A Scripts\0" + TestConstants.AllZeroSha); - lines.ShouldContain(x => x == "A GVFS\0" + TestConstants.AllZeroSha); - lines.ShouldContain(x => x == "A GVFS\\GVFS.Common\0" + TestConstants.AllZeroSha); - lines.ShouldContain(x => x == "A GVFS\\GVFS.Common\\Git\0" + TestConstants.AllZeroSha); - lines.ShouldContain(x => x == "A GVFS\\GVFS.Tests\0" + TestConstants.AllZeroSha); + lines.ShouldContain(x => x == "A Scripts\0" + TestConstants.PartialFolderPlaceholderDatabaseValue); + lines.ShouldContain(x => x == "A GVFS\0" + TestConstants.PartialFolderPlaceholderDatabaseValue); + lines.ShouldContain(x => x == "A GVFS\\GVFS.Common\0" + TestConstants.PartialFolderPlaceholderDatabaseValue); + lines.ShouldContain(x => x == "A GVFS\\GVFS.Common\\Git\0" + TestConstants.PartialFolderPlaceholderDatabaseValue); + lines.ShouldContain(x => x == "A GVFS\\GVFS.Tests\0" + TestConstants.PartialFolderPlaceholderDatabaseValue); + return lines; + } + + private string[] GetPlaceholderDatabaseLinesAfterUpgradeFrom16(string placeholderDatabasePath) + { + placeholderDatabasePath.ShouldBeAFile(this.fileSystem); + string[] lines = this.fileSystem.ReadAllText(placeholderDatabasePath).Split(new char[] { '\r', '\n' }, StringSplitOptions.RemoveEmptyEntries); + lines.Length.ShouldEqual(10); + lines.ShouldContain(x => x.Contains("Readme.md")); + lines.ShouldContain(x => x.Contains("Scripts\\RunUnitTests.bat")); + lines.ShouldContain(x => x.Contains("GVFS\\GVFS.Common\\Git\\GitRefs.cs")); + lines.ShouldContain(x => x.Contains("A .gitignore")); + lines.ShouldContain(x => x == "A Scripts\0" + TestConstants.PartialFolderPlaceholderDatabaseValue); + lines.ShouldContain(x => x == "A GVFS\0" + TestConstants.PartialFolderPlaceholderDatabaseValue); + lines.ShouldContain(x => x == "A GVFS\\GVFS.Common\0" + TestConstants.PartialFolderPlaceholderDatabaseValue); + lines.ShouldContain(x => x == "A GVFS\\GVFS.Common\\Git\0" + TestConstants.PartialFolderPlaceholderDatabaseValue); + lines.ShouldContain(x => x == "A GVFS\\GVFS.Tests\0" + TestConstants.PartialFolderPlaceholderDatabaseValue); + lines.ShouldContain(x => x == "A GVFS\\GVFS.Tests\\Properties\0" + TestConstants.PartialFolderPlaceholderDatabaseValue); return lines; } diff --git a/GVFS/GVFS.FunctionalTests/Tests/EnlistmentPerFixture/MultithreadedReadWriteTests.cs b/GVFS/GVFS.FunctionalTests/Tests/EnlistmentPerFixture/MultithreadedReadWriteTests.cs index 21924e706..9846642af 100644 --- a/GVFS/GVFS.FunctionalTests/Tests/EnlistmentPerFixture/MultithreadedReadWriteTests.cs +++ b/GVFS/GVFS.FunctionalTests/Tests/EnlistmentPerFixture/MultithreadedReadWriteTests.cs @@ -14,7 +14,7 @@ namespace GVFS.FunctionalTests.Tests.EnlistmentPerFixture public class MultithreadedReadWriteTests : TestsWithEnlistmentPerFixture { [TestCase, Order(1)] - [Category(Categories.WindowsOnly)] + [Category(Categories.MacTODO.NeedsCachePoisonFix)] public void CanReadVirtualFileInParallel() { // Note: This test MUST go first, or else it needs to ensure that it is reading a unique path compared to the diff --git a/GVFS/GVFS.FunctionalTests/Tests/GitCommands/CheckoutTests.cs b/GVFS/GVFS.FunctionalTests/Tests/GitCommands/CheckoutTests.cs index 4c5518c0d..0fb311fc6 100644 --- a/GVFS/GVFS.FunctionalTests/Tests/GitCommands/CheckoutTests.cs +++ b/GVFS/GVFS.FunctionalTests/Tests/GitCommands/CheckoutTests.cs @@ -229,7 +229,7 @@ public void CheckoutBranchAfterReadingFileAndVerifyContentsCorrect() this.FilesShouldMatchCheckoutOfSourceBranch(); // Verify modified paths contents - GVFSHelpers.ModifiedPathsContentsShouldEqual(this.FileSystem, this.Enlistment.DotGVFSRoot, "A .gitattributes\r\n"); + GVFSHelpers.ModifiedPathsContentsShouldEqual(this.FileSystem, this.Enlistment.DotGVFSRoot, "A .gitattributes" + GVFSHelpers.ModifiedPathsNewLine); } [TestCase] @@ -247,7 +247,7 @@ public void CheckoutBranchAfterReadingAllFilesAndVerifyContentsCorrect() .WithDeepStructure(this.FileSystem, this.ControlGitRepo.RootPath, compareContent: true); // Verify modified paths contents - GVFSHelpers.ModifiedPathsContentsShouldEqual(this.FileSystem, this.Enlistment.DotGVFSRoot, "A .gitattributes\r\n"); + GVFSHelpers.ModifiedPathsContentsShouldEqual(this.FileSystem, this.Enlistment.DotGVFSRoot, "A .gitattributes" + GVFSHelpers.ModifiedPathsNewLine); } [TestCase] @@ -734,7 +734,7 @@ public void DeleteFolderAndChangeBranchToFolderWithDifferentCase() } [TestCase] - [Category(Categories.MacTODO.M3)] + [Category(Categories.MacTODO.NeedsCachePoisonFix)] public void SuccessfullyChecksOutDirectoryToFileToDirectory() { // This test switches between two branches and verifies specific transitions occured diff --git a/GVFS/GVFS.FunctionalTests/Tools/GVFSHelpers.cs b/GVFS/GVFS.FunctionalTests/Tools/GVFSHelpers.cs index 17fd64369..139fb0a06 100644 --- a/GVFS/GVFS.FunctionalTests/Tools/GVFSHelpers.cs +++ b/GVFS/GVFS.FunctionalTests/Tools/GVFSHelpers.cs @@ -13,6 +13,8 @@ namespace GVFS.FunctionalTests.Tools { public static class GVFSHelpers { + public const string ModifiedPathsNewLine = "\r\n"; + public static readonly string BackgroundOpsFile = Path.Combine("databases", "BackgroundGitOperations.dat"); public static readonly string PlaceholderListFile = Path.Combine("databases", "PlaceholderList.dat"); public static readonly string RepoMetadataName = Path.Combine("databases", "RepoMetadata.dat"); @@ -21,9 +23,7 @@ public static class GVFSHelpers private const string DiskLayoutMinorVersionKey = "DiskLayoutMinorVersion"; private const string LocalCacheRootKey = "LocalCacheRoot"; private const string GitObjectsRootKey = "GitObjectsRoot"; - private const string BlobSizesRootKey = "BlobSizesRoot"; - - private const string ModifiedPathsNewLine = "\r\n"; + private const string BlobSizesRootKey = "BlobSizesRoot"; public static void SaveDiskLayoutVersion(string dotGVFSRoot, string majorVersion, string minorVersion) { diff --git a/GVFS/GVFS.FunctionalTests/Tools/TestConstants.cs b/GVFS/GVFS.FunctionalTests/Tools/TestConstants.cs index 7f04e129a..ca6d8cc9f 100644 --- a/GVFS/GVFS.FunctionalTests/Tools/TestConstants.cs +++ b/GVFS/GVFS.FunctionalTests/Tools/TestConstants.cs @@ -5,6 +5,7 @@ namespace GVFS.FunctionalTests.Tools public static class TestConstants { public const string AllZeroSha = "0000000000000000000000000000000000000000"; + public const string PartialFolderPlaceholderDatabaseValue = "@FPARTIAL_000000000000000000000000000000"; public static class DotGit { diff --git a/GVFS/GVFS.Platform.Mac/MacFileSystem.cs b/GVFS/GVFS.Platform.Mac/MacFileSystem.cs index 55244b827..4e72868a9 100644 --- a/GVFS/GVFS.Platform.Mac/MacFileSystem.cs +++ b/GVFS/GVFS.Platform.Mac/MacFileSystem.cs @@ -7,7 +7,6 @@ namespace GVFS.Platform.Mac public partial class MacFileSystem : IPlatformFileSystem { public bool SupportsFileMode { get; } = true; - public bool EnumerationExpandsDirectories { get; } = true; public void FlushFileBuffers(string path) { diff --git a/GVFS/GVFS.Platform.Mac/MacFileSystemVirtualizer.cs b/GVFS/GVFS.Platform.Mac/MacFileSystemVirtualizer.cs index 51542537c..bc13595cd 100644 --- a/GVFS/GVFS.Platform.Mac/MacFileSystemVirtualizer.cs +++ b/GVFS/GVFS.Platform.Mac/MacFileSystemVirtualizer.cs @@ -70,30 +70,26 @@ public override void Stop() this.Context.Tracer.RelatedEvent(EventLevel.Informational, $"{nameof(this.Stop)}_StopRequested", metadata: null); } - public override FileSystemResult WritePlaceholder( + public override FileSystemResult WritePlaceholderFile( string relativePath, long endOfFile, - bool isDirectory, string sha) { - Result result; + // TODO(Mac): Add functional tests that validate file mode is set correctly + ushort fileMode = this.FileSystemCallbacks.GitIndexProjection.GetFilePathMode(relativePath); + Result result = this.virtualizationInstance.WritePlaceholderFile( + relativePath, + PlaceholderVersionId, + ToVersionIdByteArray(FileSystemVirtualizer.ConvertShaToContentId(sha)), + (ulong)endOfFile, + fileMode); - if (isDirectory) - { - result = this.virtualizationInstance.WritePlaceholderDirectory(relativePath); - } - else - { - // TODO(Mac): Add functional tests that validate file mode is set correctly - ushort fileMode = this.FileSystemCallbacks.GitIndexProjection.GetFilePathMode(relativePath); - result = this.virtualizationInstance.WritePlaceholderFile( - relativePath, - PlaceholderVersionId, - ToVersionIdByteArray(FileSystemVirtualizer.ConvertShaToContentId(sha)), - (ulong)endOfFile, - fileMode); - } + return new FileSystemResult(ResultToFSResult(result), unchecked((int)result)); + } + public override FileSystemResult WritePlaceholderDirectory(string relativePath) + { + Result result = this.virtualizationInstance.WritePlaceholderDirectory(relativePath); return new FileSystemResult(ResultToFSResult(result), unchecked((int)result)); } @@ -462,18 +458,29 @@ private Result CreatePlaceholders(string directoryRelativePath, IEnumerable throw new NotImplementedException(); public bool IsSupported(string normalizedEnlistmentRootPath, out string warning, out string error) diff --git a/GVFS/GVFS.Platform.Windows/DiskLayoutUpgrades/DiskLayout16to17Upgrade_FolderPlaceholderValues.cs b/GVFS/GVFS.Platform.Windows/DiskLayoutUpgrades/DiskLayout16to17Upgrade_FolderPlaceholderValues.cs new file mode 100644 index 000000000..24893770a --- /dev/null +++ b/GVFS/GVFS.Platform.Windows/DiskLayoutUpgrades/DiskLayout16to17Upgrade_FolderPlaceholderValues.cs @@ -0,0 +1,75 @@ +using GVFS.Common; +using GVFS.Common.FileSystem; +using GVFS.Common.Tracing; +using GVFS.DiskLayoutUpgrades; +using System; +using System.Collections.Generic; +using System.IO; + +namespace GVFS.Platform.Windows.DiskLayoutUpgrades +{ + /// + /// Updates the values for folder placeholders from AllZeroSha to PlaceholderListDatabase.PartialFolderValue + /// + public class DiskLayout16to17Upgrade_FolderPlaceholderValues : DiskLayoutUpgrade.MajorUpgrade + { + protected override int SourceMajorVersion => 16; + + public override bool TryUpgrade(ITracer tracer, string enlistmentRoot) + { + string dotGVFSRoot = Path.Combine(enlistmentRoot, GVFSConstants.DotGVFS.Root); + try + { + string error; + PlaceholderListDatabase placeholders; + if (!PlaceholderListDatabase.TryCreate( + tracer, + Path.Combine(dotGVFSRoot, GVFSConstants.DotGVFS.Databases.PlaceholderList), + new PhysicalFileSystem(), + out placeholders, + out error)) + { + tracer.RelatedError("Failed to open placeholder database: " + error); + return false; + } + + using (placeholders) + { + List oldPlaceholderEntries = placeholders.GetAllEntries(); + List newPlaceholderEntries = new List(); + + foreach (PlaceholderListDatabase.PlaceholderData entry in oldPlaceholderEntries) + { + if (entry.Sha == GVFSConstants.AllZeroSha) + { + newPlaceholderEntries.Add(new PlaceholderListDatabase.PlaceholderData(entry.Path, PlaceholderListDatabase.PartialFolderValue)); + } + else + { + newPlaceholderEntries.Add(entry); + } + } + + placeholders.WriteAllEntriesAndFlush(newPlaceholderEntries); + } + } + catch (IOException ex) + { + tracer.RelatedError("Could not write to placeholder database: " + ex.ToString()); + return false; + } + catch (Exception ex) + { + tracer.RelatedError("Error updating placeholder database folder entries: " + ex.ToString()); + return false; + } + + if (!this.TryIncrementMajorVersion(tracer, enlistmentRoot)) + { + return false; + } + + return true; + } + } +} diff --git a/GVFS/GVFS.Platform.Windows/DiskLayoutUpgrades/DiskLayout9to10Upgrade_BackgroundAndPlaceholderListToFileBased.cs b/GVFS/GVFS.Platform.Windows/DiskLayoutUpgrades/DiskLayout9to10Upgrade_BackgroundAndPlaceholderListToFileBased.cs index 464d2c1db..397738439 100644 --- a/GVFS/GVFS.Platform.Windows/DiskLayoutUpgrades/DiskLayout9to10Upgrade_BackgroundAndPlaceholderListToFileBased.cs +++ b/GVFS/GVFS.Platform.Windows/DiskLayoutUpgrades/DiskLayout9to10Upgrade_BackgroundAndPlaceholderListToFileBased.cs @@ -75,7 +75,7 @@ private bool UpdatePlaceholderList(ITracer tracer, string dotGVFSRoot) foreach (KeyValuePair kvp in oldPlaceholders) { tracer.RelatedInfo("Copying ESENT entry: {0} = {1}", kvp.Key, kvp.Value); - data.Add(new PlaceholderListDatabase.PlaceholderData(path: kvp.Key, sha: kvp.Value)); + data.Add(new PlaceholderListDatabase.PlaceholderData(path: kvp.Key, fileShaOrFolderValue: kvp.Value)); } newPlaceholders.WriteAllEntriesAndFlush(data); diff --git a/GVFS/GVFS.Platform.Windows/DiskLayoutUpgrades/WindowsDiskLayoutUpgradeData.cs b/GVFS/GVFS.Platform.Windows/DiskLayoutUpgrades/WindowsDiskLayoutUpgradeData.cs index 36b6bafc4..bd656bae3 100644 --- a/GVFS/GVFS.Platform.Windows/DiskLayoutUpgrades/WindowsDiskLayoutUpgradeData.cs +++ b/GVFS/GVFS.Platform.Windows/DiskLayoutUpgrades/WindowsDiskLayoutUpgradeData.cs @@ -27,6 +27,7 @@ public DiskLayoutUpgrade[] Upgrades new DiskLayout13to14Upgrade_BlobSizes(), new DiskLayout14to15Upgrade_ModifiedPaths(), new DiskLayout15to16Upgrade_GitStatusCache(), + new DiskLayout16to17Upgrade_FolderPlaceholderValues() }; } } diff --git a/GVFS/GVFS.Platform.Windows/GVFS.Platform.Windows.csproj b/GVFS/GVFS.Platform.Windows/GVFS.Platform.Windows.csproj index e3deda70c..c461a584b 100644 --- a/GVFS/GVFS.Platform.Windows/GVFS.Platform.Windows.csproj +++ b/GVFS/GVFS.Platform.Windows/GVFS.Platform.Windows.csproj @@ -79,6 +79,7 @@ + diff --git a/GVFS/GVFS.Platform.Windows/ProjFSFilter.cs b/GVFS/GVFS.Platform.Windows/ProjFSFilter.cs index 6b9f20493..544b5678a 100644 --- a/GVFS/GVFS.Platform.Windows/ProjFSFilter.cs +++ b/GVFS/GVFS.Platform.Windows/ProjFSFilter.cs @@ -34,6 +34,7 @@ public class ProjFSFilter : IKernelDriver private const uint OkResult = 0; private const uint NameCollisionErrorResult = 0x801F0012; + public bool EnumerationExpandsDirectories { get; } = false; public string DriverLogFolderName { get; } = ProjFSFilter.ServiceName; public static bool TryAttach(ITracer tracer, string enlistmentRoot, out string errorMessage) diff --git a/GVFS/GVFS.Platform.Windows/WindowsFileSystem.cs b/GVFS/GVFS.Platform.Windows/WindowsFileSystem.cs index e6e5c3770..89aa726b4 100644 --- a/GVFS/GVFS.Platform.Windows/WindowsFileSystem.cs +++ b/GVFS/GVFS.Platform.Windows/WindowsFileSystem.cs @@ -6,7 +6,6 @@ namespace GVFS.Platform.Windows public partial class WindowsFileSystem : IPlatformFileSystem { public bool SupportsFileMode { get; } = false; - public bool EnumerationExpandsDirectories { get; } = false; public void FlushFileBuffers(string path) { diff --git a/GVFS/GVFS.Platform.Windows/WindowsFileSystemVirtualizer.cs b/GVFS/GVFS.Platform.Windows/WindowsFileSystemVirtualizer.cs index 089c34eb0..6418fda14 100644 --- a/GVFS/GVFS.Platform.Windows/WindowsFileSystemVirtualizer.cs +++ b/GVFS/GVFS.Platform.Windows/WindowsFileSystemVirtualizer.cs @@ -115,22 +115,11 @@ public override FileSystemResult DeleteFile(string relativePath, UpdatePlacehold return new FileSystemResult(HResultToFSResult(result), unchecked((int)result)); } - public override FileSystemResult WritePlaceholder( + public override FileSystemResult WritePlaceholderFile( string relativePath, long endOfFile, - bool isDirectory, string sha) { - uint fileAttributes; - if (isDirectory) - { - fileAttributes = (uint)NativeMethods.FileAttributes.FILE_ATTRIBUTE_DIRECTORY; - } - else - { - fileAttributes = (uint)NativeMethods.FileAttributes.FILE_ATTRIBUTE_ARCHIVE; - } - FileProperties properties = this.FileSystemCallbacks.GetLogsHeadFileProperties(); HResult result = this.virtualizationInstance.WritePlaceholderInformation( relativePath, @@ -138,15 +127,33 @@ public override FileSystemResult WritePlaceholder( properties.LastAccessTimeUTC, properties.LastWriteTimeUTC, changeTime: properties.LastWriteTimeUTC, - fileAttributes: fileAttributes, + fileAttributes: (uint)NativeMethods.FileAttributes.FILE_ATTRIBUTE_ARCHIVE, endOfFile: endOfFile, - isDirectory: isDirectory, + isDirectory: false, contentId: FileSystemVirtualizer.ConvertShaToContentId(sha), providerId: PlaceholderVersionId); return new FileSystemResult(HResultToFSResult(result), unchecked((int)result)); } + public override FileSystemResult WritePlaceholderDirectory(string relativePath) + { + FileProperties properties = this.FileSystemCallbacks.GetLogsHeadFileProperties(); + HResult result = this.virtualizationInstance.WritePlaceholderInformation( + relativePath, + properties.CreationTimeUTC, + properties.LastAccessTimeUTC, + properties.LastWriteTimeUTC, + changeTime: properties.LastWriteTimeUTC, + fileAttributes: (uint)NativeMethods.FileAttributes.FILE_ATTRIBUTE_DIRECTORY, + endOfFile: 0, + isDirectory: true, + contentId: FolderContentId, + providerId: PlaceholderVersionId); + + return new FileSystemResult(HResultToFSResult(result), unchecked((int)result)); + } + public override FileSystemResult UpdatePlaceholderIfNeeded( string relativePath, DateTime creationTime, @@ -772,11 +779,20 @@ private void GetPlaceholderInformationAsyncHandler( // with proper case. string gitCaseVirtualPath = Path.Combine(parentFolderPath, fileInfo.Name); - string sha = fileInfo.IsFolder ? string.Empty : fileInfo.Sha.ToString(); + string sha; + FileSystemResult fileSystemResult; + if (fileInfo.IsFolder) + { + sha = string.Empty; + fileSystemResult = this.WritePlaceholderDirectory(gitCaseVirtualPath); + } + else + { + sha = fileInfo.Sha.ToString(); + fileSystemResult = this.WritePlaceholderFile(gitCaseVirtualPath, fileInfo.Size, sha); + } - FileSystemResult fileSystemResult = this.WritePlaceholder(gitCaseVirtualPath, fileInfo.Size, fileInfo.IsFolder, sha); result = (HResult)fileSystemResult.RawResult; - if (result != HResult.Ok) { EventMetadata metadata = this.CreateEventMetadata(virtualPath); @@ -1115,7 +1131,7 @@ private void NotifyNewFileCreatedHandler( string directoryPath = Path.Combine(this.Context.Enlistment.WorkingDirectoryRoot, virtualPath); HResult hr = this.virtualizationInstance.ConvertDirectoryToPlaceholder( directoryPath, - ConvertShaToContentId(GVFSConstants.AllZeroSha), + FolderContentId, PlaceholderVersionId); if (hr == HResult.Ok) diff --git a/GVFS/GVFS.Tests/Should/EnumerableShouldExtensions.cs b/GVFS/GVFS.Tests/Should/EnumerableShouldExtensions.cs index 6f3ff9412..913af43b8 100644 --- a/GVFS/GVFS.Tests/Should/EnumerableShouldExtensions.cs +++ b/GVFS/GVFS.Tests/Should/EnumerableShouldExtensions.cs @@ -124,7 +124,7 @@ public static IEnumerable ShouldMatchInOrder(this IEnumerable group, IE public static IEnumerable ShouldMatchInOrder(this IEnumerable group, params T[] expectedValues) { - return group.ShouldMatchInOrder(expectedValues, (t1, t2) => t1.Equals(t2)); + return group.ShouldMatchInOrder((IEnumerable)expectedValues); } public static IEnumerable ShouldMatchInOrder(this IEnumerable group, IEnumerable expectedValues) diff --git a/GVFS/GVFS.UnitTests/Common/PlaceholderDatabaseTests.cs b/GVFS/GVFS.UnitTests/Common/PlaceholderDatabaseTests.cs index ea02b8ca3..465e4b907 100644 --- a/GVFS/GVFS.UnitTests/Common/PlaceholderDatabaseTests.cs +++ b/GVFS/GVFS.UnitTests/Common/PlaceholderDatabaseTests.cs @@ -21,8 +21,9 @@ public class PlaceholderDatabaseTests private const string InputThirdFilePath = "thirdFile"; private const string InputThirdFileSHA = "ff9630E00F715315FC90D4AEC98E6A7398F8BF11"; - private const string ExpectedGitIgnoreEntry = "A " + InputGitIgnorePath + "\0" + InputGitIgnoreSHA + "\r\n"; - private const string ExpectedGitAttributesEntry = "A " + InputGitAttributesPath + "\0" + InputGitAttributesSHA + "\r\n"; + private const string PlaceholderDatabaseNewLine = "\r\n"; + private const string ExpectedGitIgnoreEntry = "A " + InputGitIgnorePath + "\0" + InputGitIgnoreSHA + PlaceholderDatabaseNewLine; + private const string ExpectedGitAttributesEntry = "A " + InputGitAttributesPath + "\0" + InputGitAttributesSHA + PlaceholderDatabaseNewLine; private const string ExpectedTwoEntries = ExpectedGitIgnoreEntry + ExpectedGitAttributesEntry; @@ -50,11 +51,11 @@ public void WritesPlaceholderAddToFile() { ConfigurableFileSystem fs = new ConfigurableFileSystem(); PlaceholderListDatabase dut = CreatePlaceholderListDatabase(fs, string.Empty); - dut.AddAndFlush(InputGitIgnorePath, InputGitIgnoreSHA); + dut.AddAndFlushFile(InputGitIgnorePath, InputGitIgnoreSHA); fs.ExpectedFiles[MockEntryFileName].ReadAsString().ShouldEqual(ExpectedGitIgnoreEntry); - dut.AddAndFlush(InputGitAttributesPath, InputGitAttributesSHA); + dut.AddAndFlushFile(InputGitAttributesPath, InputGitAttributesSHA); fs.ExpectedFiles[MockEntryFileName].ReadAsString().ShouldEqual(ExpectedTwoEntries); } @@ -65,9 +66,9 @@ public void GetAllEntriesReturnsCorrectEntries() ConfigurableFileSystem fs = new ConfigurableFileSystem(); using (PlaceholderListDatabase dut1 = CreatePlaceholderListDatabase(fs, string.Empty)) { - dut1.AddAndFlush(InputGitIgnorePath, InputGitIgnoreSHA); - dut1.AddAndFlush(InputGitAttributesPath, InputGitAttributesSHA); - dut1.AddAndFlush(InputThirdFilePath, InputThirdFileSHA); + dut1.AddAndFlushFile(InputGitIgnorePath, InputGitIgnoreSHA); + dut1.AddAndFlushFile(InputGitAttributesPath, InputGitAttributesSHA); + dut1.AddAndFlushFile(InputThirdFilePath, InputThirdFileSHA); dut1.RemoveAndFlush(InputThirdFilePath); } @@ -78,6 +79,37 @@ public void GetAllEntriesReturnsCorrectEntries() allData.Count.ShouldEqual(2); } + [TestCase] + public void GetAllEntriesSplitsFilesAndFoldersCorrectly() + { + ConfigurableFileSystem fs = new ConfigurableFileSystem(); + using (PlaceholderListDatabase dut1 = CreatePlaceholderListDatabase(fs, string.Empty)) + { + dut1.AddAndFlushFile(InputGitIgnorePath, InputGitIgnoreSHA); + dut1.AddAndFlushFolder("partialFolder", isExpanded: false); + dut1.AddAndFlushFile(InputGitAttributesPath, InputGitAttributesSHA); + dut1.AddAndFlushFolder("expandedFolder", isExpanded: true); + dut1.AddAndFlushFile(InputThirdFilePath, InputThirdFileSHA); + dut1.RemoveAndFlush(InputThirdFilePath); + } + + string error; + PlaceholderListDatabase dut2; + PlaceholderListDatabase.TryCreate(null, MockEntryFileName, fs, out dut2, out error).ShouldEqual(true, error); + List fileData; + List folderData; + dut2.GetAllEntries(out fileData, out folderData); + fileData.Count.ShouldEqual(2); + folderData.Count.ShouldEqual(2); + folderData.ShouldContain( + new[] + { + new PlaceholderListDatabase.PlaceholderData("partialFolder", PlaceholderListDatabase.PartialFolderValue), + new PlaceholderListDatabase.PlaceholderData("expandedFolder", PlaceholderListDatabase.ExpandedFolderValue) + }, + (data1, data2) => data1.Path == data2.Path && data1.Sha == data2.Sha); + } + [TestCase] public void WriteAllEntriesCorrectlyWritesFile() { @@ -106,7 +138,7 @@ public void HandlesRaceBetweenAddAndWriteAllEntries() List existingEntries = dut.GetAllEntries(); - dut.AddAndFlush(InputGitAttributesPath, InputGitAttributesSHA); + dut.AddAndFlushFile(InputGitAttributesPath, InputGitAttributesSHA); dut.WriteAllEntriesAndFlush(existingEntries); fs.ExpectedFiles[MockEntryFileName].ReadAsString().ShouldEqual(ExpectedTwoEntries); @@ -115,7 +147,7 @@ public void HandlesRaceBetweenAddAndWriteAllEntries() [TestCase] public void HandlesRaceBetweenRemoveAndWriteAllEntries() { - const string DeleteGitAttributesEntry = "D .gitattributes\r\n"; + const string DeleteGitAttributesEntry = "D .gitattributes" + PlaceholderDatabaseNewLine; ConfigurableFileSystem fs = new ConfigurableFileSystem(); fs.ExpectedFiles.Add(MockEntryFileName + ".tmp", new ReusableMemoryStream(string.Empty)); diff --git a/GVFS/GVFS.UnitTests/Mock/FileSystem/MockPlatformFileSystem.cs b/GVFS/GVFS.UnitTests/Mock/FileSystem/MockPlatformFileSystem.cs index d3e6f42d5..999f7efd9 100644 --- a/GVFS/GVFS.UnitTests/Mock/FileSystem/MockPlatformFileSystem.cs +++ b/GVFS/GVFS.UnitTests/Mock/FileSystem/MockPlatformFileSystem.cs @@ -6,10 +6,6 @@ namespace GVFS.UnitTests.Mock.FileSystem public class MockPlatformFileSystem : IPlatformFileSystem { public bool SupportsFileMode { get; } = true; - public bool EnumerationExpandsDirectories - { - get { throw new NotSupportedException(); } - } public void FlushFileBuffers(string path) { diff --git a/GVFS/GVFS.UnitTests/Mock/Virtualization/FileSystem/MockFileSystemVirtualizer.cs b/GVFS/GVFS.UnitTests/Mock/Virtualization/FileSystem/MockFileSystemVirtualizer.cs index a99625079..b5223be47 100644 --- a/GVFS/GVFS.UnitTests/Mock/Virtualization/FileSystem/MockFileSystemVirtualizer.cs +++ b/GVFS/GVFS.UnitTests/Mock/Virtualization/FileSystem/MockFileSystemVirtualizer.cs @@ -27,11 +27,16 @@ public override void Stop() { } - public override FileSystemResult WritePlaceholder(string relativePath, long endOfFile, bool isDirectory, string shaContentId) + public override FileSystemResult WritePlaceholderFile(string relativePath, long endOfFile, string sha) { throw new NotImplementedException(); } + public override FileSystemResult WritePlaceholderDirectory(string relativePath) + { + throw new NotImplementedException(); + } + public override FileSystemResult UpdatePlaceholderIfNeeded(string relativePath, DateTime creationTime, DateTime lastAccessTime, DateTime lastWriteTime, DateTime changeTime, uint fileAttributes, long endOfFile, string shaContentId, UpdatePlaceholderType updateFlags, out UpdateFailureReason failureReason) { throw new NotImplementedException(); diff --git a/GVFS/GVFS.Virtualization/FileSystem/FileSystemVirtualizer.cs b/GVFS/GVFS.Virtualization/FileSystem/FileSystemVirtualizer.cs index 3c67d1692..4f811ce76 100644 --- a/GVFS/GVFS.Virtualization/FileSystem/FileSystemVirtualizer.cs +++ b/GVFS/GVFS.Virtualization/FileSystem/FileSystemVirtualizer.cs @@ -13,6 +13,8 @@ public abstract class FileSystemVirtualizer : IDisposable { public const byte PlaceholderVersion = 1; + protected static readonly byte[] FolderContentId = Encoding.Unicode.GetBytes(GVFSConstants.AllZeroSha); + protected static readonly GitCommandLineParser.Verbs CanCreatePlaceholderVerbs = GitCommandLineParser.Verbs.AddOrStage | GitCommandLineParser.Verbs.Move | GitCommandLineParser.Verbs.Status; @@ -93,11 +95,8 @@ public void PrepareToStop() public abstract FileSystemResult DeleteFile(string relativePath, UpdatePlaceholderType updateFlags, out UpdateFailureReason failureReason); - public abstract FileSystemResult WritePlaceholder( - string relativePath, - long endOfFile, - bool isDirectory, - string shaContentId); + public abstract FileSystemResult WritePlaceholderFile(string relativePath, long endOfFile, string sha); + public abstract FileSystemResult WritePlaceholderDirectory(string relativePath); public abstract FileSystemResult UpdatePlaceholderIfNeeded( string relativePath, diff --git a/GVFS/GVFS.Virtualization/Projection/GitIndexProjection.cs b/GVFS/GVFS.Virtualization/Projection/GitIndexProjection.cs index 70b42d5ae..1d485e48e 100644 --- a/GVFS/GVFS.Virtualization/Projection/GitIndexProjection.cs +++ b/GVFS/GVFS.Virtualization/Projection/GitIndexProjection.cs @@ -317,17 +317,17 @@ public void ClearNegativePathCacheIfPollutedByGit() public void OnPlaceholderFolderCreated(string virtualPath) { - this.placeholderList.AddAndFlush(virtualPath, GVFSConstants.AllZeroSha); + this.placeholderList.AddAndFlushFolder(virtualPath, isExpanded: false); } public virtual void OnPlaceholderFolderExpanded(string relativePath) { - this.placeholderList.AddAndFlush(relativePath, GVFSConstants.ExpandedFolderSha); + this.placeholderList.AddAndFlushFolder(relativePath, isExpanded: true); } public virtual void OnPlaceholderFileCreated(string virtualPath, string sha) { - this.placeholderList.AddAndFlush(virtualPath, sha); + this.placeholderList.AddAndFlushFile(virtualPath, sha); } public virtual bool TryGetProjectedItemsFromMemory(string folderPath, out List projectedItems) @@ -899,7 +899,11 @@ private bool TryGetOrAddFolderDataFromCache( { EventMetadata metadata = CreateEventMetadata(); metadata.Add("folderPath", folderPath); - this.context.Tracer.RelatedWarning(metadata, "GitIndexProjection_TryGetOrAddFolderDataFromCacheFoundFile: Found a file when expecting a folder"); + metadata.Add(TracingConstants.MessageKey.InfoMessage, "Found file at path"); + this.context.Tracer.RelatedEvent( + EventLevel.Informational, + $"{nameof(this.TryGetOrAddFolderDataFromCache)}_FileAtPath", + metadata); folderPath = null; return false; @@ -1083,15 +1087,20 @@ private void UpdatePlaceholders() { this.ClearUpdatePlaceholderErrors(); - List placeholderListCopy = this.placeholderList.GetAllEntries(); + List placeholderFilesListCopy; + List placeholderFoldersListCopy; + this.placeholderList.GetAllEntries(out placeholderFilesListCopy, out placeholderFoldersListCopy); + EventMetadata metadata = new EventMetadata(); - metadata.Add("Count", placeholderListCopy.Count); + metadata.Add("File placeholder count", placeholderFilesListCopy.Count); + metadata.Add("Folder placeholders count", placeholderFoldersListCopy.Count); + using (ITracer activity = this.context.Tracer.StartActivity("UpdatePlaceholders", EventLevel.Informational, metadata)) { ConcurrentHashSet folderPlaceholdersToKeep = new ConcurrentHashSet(); ConcurrentDictionary updatedPlaceholderList = new ConcurrentDictionary(StringComparer.Ordinal); this.ProcessListOnThreads( - placeholderListCopy.Where(x => !x.IsFolder).ToList(), + placeholderFilesListCopy, (placeholderBatch, start, end, blobSizesConnection, availableSizes) => this.BatchPopulateMissingSizesFromRemote(blobSizesConnection, placeholderBatch, start, end, availableSizes), (placeholder, blobSizesConnection, availableSizes) => @@ -1100,20 +1109,24 @@ private void UpdatePlaceholders() this.blobSizes.Flush(); using (BlobSizes.BlobSizesConnection blobSizesConnection = this.blobSizes.CreateConnection()) - { - // Waiting for the UpdateOrDeleteFilePlaceholder to fill the folderPlaceholdersToKeep before trying to remove folder placeholders - // so that we don't try to delete a folder placeholder that has file placeholders and just fails - IEnumerable folderPlaceholderData = placeholderListCopy.Where(x => x.IsFolder); - - HashSet folderPlaceholders = new HashSet(folderPlaceholderData.Select(x => x.Path), StringComparer.OrdinalIgnoreCase); - foreach (PlaceholderListDatabase.PlaceholderData folderPlaceholder in folderPlaceholderData.OrderByDescending(x => x.Path)) - { - if (GVFSPlatform.Instance.FileSystem.EnumerationExpandsDirectories && folderPlaceholder.IsExpandedFolder) - { - this.ReExpandFolder(blobSizesConnection, folderPlaceholder.Path, updatedPlaceholderList, folderPlaceholders, folderPlaceholdersToKeep); - } - - this.TryRemoveFolderPlaceholder(folderPlaceholder, updatedPlaceholderList, folderPlaceholdersToKeep); + { + // A hash of the folder placeholders is only required if the platform expands directories + HashSet folderPlaceholders = + GVFSPlatform.Instance.KernelDriver.EnumerationExpandsDirectories ? + new HashSet(placeholderFoldersListCopy.Select(x => x.Path), StringComparer.OrdinalIgnoreCase) : + null; + + foreach (PlaceholderListDatabase.PlaceholderData folderPlaceholder in placeholderFoldersListCopy.OrderByDescending(x => x.Path)) + { + // Remove folder placeholders before re-expansion to ensure that projection changes that convert a folder to a file work + // properly + if (!this.RemoveFolderPlaceholderIfEmpty(folderPlaceholder, updatedPlaceholderList, folderPlaceholdersToKeep)) + { + if (GVFSPlatform.Instance.KernelDriver.EnumerationExpandsDirectories && folderPlaceholder.IsExpandedFolder) + { + this.ReExpandFolder(blobSizesConnection, folderPlaceholder.Path, updatedPlaceholderList, folderPlaceholders); + } + } } } @@ -1275,10 +1288,9 @@ private string GetNewProjectedShaForPlaceholder(string path) private void ReExpandFolder( BlobSizes.BlobSizesConnection blobSizesConnection, - string relativeFolderPath, + string relativeFolderPath, ConcurrentDictionary updatedPlaceholderList, - HashSet existingFolderPlaceholders, - ConcurrentHashSet folderPlaceholdersToKeep) + HashSet existingFolderPlaceholders) { FolderData folderData; if (!this.TryGetOrAddFolderDataFromCache(relativeFolderPath, out folderData)) @@ -1304,48 +1316,47 @@ private void ReExpandFolder( } else { - childRelativePath = relativeFolderPath + Path.DirectorySeparatorChar + childEntry.Name.GetString(); + childRelativePath = relativeFolderPath + Path.DirectorySeparatorChar + childEntry.Name.GetString(); } + // TODO(Mac): Issue #245, handle failures of WritePlaceholderDirectory and WritePlaceholderFile if (childEntry.IsFolder) { if (!existingFolderPlaceholders.Contains(childRelativePath)) - { - // TODO(Mac): Issue #245, handle failures of WritePlaceholder - this.fileSystemVirtualizer.WritePlaceholder( - childRelativePath, - endOfFile: 0, - isDirectory: true, - shaContentId: GVFSConstants.AllZeroSha); - - updatedPlaceholderList.TryAdd( - childRelativePath, - new PlaceholderListDatabase.PlaceholderData(childRelativePath, GVFSConstants.AllZeroSha)); - - folderPlaceholdersToKeep.Add(relativeFolderPath); + { + this.fileSystemVirtualizer.WritePlaceholderDirectory(childRelativePath); + updatedPlaceholderList.TryAdd( + childRelativePath, + new PlaceholderListDatabase.PlaceholderData(childRelativePath, PlaceholderListDatabase.PartialFolderValue)); } } else { if (!updatedPlaceholderList.ContainsKey(childRelativePath)) - { - FileData childFileData = childEntry as FileData; - string sha = childFileData.Sha.ToString(); - - // TODO(Mac): Issue #245, handle failures of WritePlaceholder - this.fileSystemVirtualizer.WritePlaceholder(childRelativePath, childFileData.Size, isDirectory: false, shaContentId: sha); - - updatedPlaceholderList.TryAdd( - childRelativePath, - new PlaceholderListDatabase.PlaceholderData(childRelativePath, sha)); - - folderPlaceholdersToKeep.Add(relativeFolderPath); + { + FileData childFileData = childEntry as FileData; + string sha = childFileData.Sha.ToString(); + + this.fileSystemVirtualizer.WritePlaceholderFile(childRelativePath, childFileData.Size, sha); + updatedPlaceholderList.TryAdd( + childRelativePath, + new PlaceholderListDatabase.PlaceholderData(childRelativePath, sha)); } } } } - private void TryRemoveFolderPlaceholder( + /// + /// Removes the folder placeholder from disk if it's empty. + /// + /// + /// trueIf the folder placeholder was deleted + /// falseIf RemoveFolderPlaceholderIfEmpty did not attempt to remove the folder placeholder + /// + /// + /// If the platform expands on enumeration the folder will only be removed if it's not in the projection + /// + private bool RemoveFolderPlaceholderIfEmpty( PlaceholderListDatabase.PlaceholderData placeholder, ConcurrentDictionary updatedPlaceholderList, ConcurrentHashSet folderPlaceholdersToKeep) @@ -1353,24 +1364,25 @@ private void TryRemoveFolderPlaceholder( if (folderPlaceholdersToKeep.Contains(placeholder.Path)) { updatedPlaceholderList.TryAdd(placeholder.Path, placeholder); - return; + return false; } - if (GVFSPlatform.Instance.FileSystem.EnumerationExpandsDirectories) - { - // If enumeration expands directories we should exit early if the folder - // is still in the projection to avoid deleting folder placeholders that should - // still be on disk. However, if enumeration does not expand directories there - // may be folder tombstones on disk that need to get cleaned up (e.g. because git - // deleted a folder a folder that was not in ModifiedPaths.dat) and we should - // proceed with removing the folder placeholder. - - FolderData folderData; - if (this.TryGetOrAddFolderDataFromCache(placeholder.Path, out folderData)) - { - // Folder is still in the projection and should not be removed - updatedPlaceholderList.TryAdd(placeholder.Path, placeholder); - return; + if (GVFSPlatform.Instance.KernelDriver.EnumerationExpandsDirectories) + { + // If enumeration expands directories we should leave folder placeholders + // that are still in the projection on disk (even if they are empty). + // + // If enumeration does not expand directories there is no harm in deleting empty + // folder placeholders that are in the projection as they will be re-projected during + // enumeration. Additionally, there may be folder tombstones on disk that need to be + // cleaned up (e.g. git might have deleted a folder placeholder that was not in + // ModifiedPaths.dat, resulting in a tombstone getting created). + + FolderData folderData; + if (this.TryGetOrAddFolderDataFromCache(placeholder.Path, out folderData)) + { + updatedPlaceholderList.TryAdd(placeholder.Path, placeholder); + return false; } } @@ -1394,12 +1406,15 @@ private void TryRemoveFolderPlaceholder( metadata.Add("result.Result", result.Result.ToString()); metadata.Add("result.RawResult", result.RawResult); metadata.Add("UpdateFailureCause", failureReason.ToString()); - this.context.Tracer.RelatedEvent(EventLevel.Informational, nameof(this.TryRemoveFolderPlaceholder) + "_DeleteFileFailure", metadata); + this.context.Tracer.RelatedEvent(EventLevel.Informational, nameof(this.RemoveFolderPlaceholderIfEmpty) + "_DeleteFileFailure", metadata); - updatedPlaceholderList.TryAdd(placeholder.Path, placeholder); + // TODO(Mac): Issue #245, handle failures DeleteFile on Mac. If we don't do anything we could leave an untracked folder + // placeholder on disk that will never be updated by Git or VFSForGit break; } + + return true; } private void UpdateOrDeleteFilePlaceholder( From 1ac75ba6478f50c333f32c18960e35ba6488ffe2 Mon Sep 17 00:00:00 2001 From: William Baker Date: Mon, 10 Sep 2018 08:56:02 -0700 Subject: [PATCH 106/272] Add named constant for new line separator in FileBasedCollection --- GVFS/GVFS.Common/FileBasedCollection.cs | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/GVFS/GVFS.Common/FileBasedCollection.cs b/GVFS/GVFS.Common/FileBasedCollection.cs index 3600fbbfa..8fcf5a93a 100644 --- a/GVFS/GVFS.Common/FileBasedCollection.cs +++ b/GVFS/GVFS.Common/FileBasedCollection.cs @@ -15,6 +15,7 @@ public abstract class FileBasedCollection : IDisposable private const string AddEntryPrefix = "A "; private const string RemoveEntryPrefix = "D "; + private const string NewLine = "\r\n"; private const int IoFailureRetryDelayMS = 50; private const int IoFailureLoggingThreshold = 500; @@ -390,7 +391,7 @@ private void WriteToDisk(string value) throw new InvalidOperationException(nameof(this.WriteToDisk) + " requires that collectionAppendsDirectlyToFile be true"); } - byte[] bytes = Encoding.UTF8.GetBytes(value + "\r\n"); + byte[] bytes = Encoding.UTF8.GetBytes(value + NewLine); lock (this.fileLock) { this.dataFileHandle.Write(bytes, 0, bytes.Length); @@ -399,7 +400,7 @@ private void WriteToDisk(string value) } /// - /// Reads entries from dataFileHandle, removing any data after the last \r\n. Requires fileLock. + /// Reads entries from dataFileHandle, removing any data after the last NewLine ("\r\n"). Requires fileLock. /// private void RemoveLastEntryIfInvalid() { @@ -442,7 +443,7 @@ private bool TryWriteTempFile(Func> getDataLines, out Except { foreach (string line in getDataLines()) { - writer.Write(line + "\r\n"); + writer.Write(line + NewLine); } tempFile.Flush(); From cdf91d6f5eb145a9e979ef39c0134d90991e05b8 Mon Sep 17 00:00:00 2001 From: William Baker Date: Mon, 10 Sep 2018 13:17:58 -0700 Subject: [PATCH 107/272] Use ConcurrentBag when updating projection on Windows --- .../Projection/GitIndexProjection.cs | 71 +++++++++++++------ 1 file changed, 51 insertions(+), 20 deletions(-) diff --git a/GVFS/GVFS.Virtualization/Projection/GitIndexProjection.cs b/GVFS/GVFS.Virtualization/Projection/GitIndexProjection.cs index 1d485e48e..275092af7 100644 --- a/GVFS/GVFS.Virtualization/Projection/GitIndexProjection.cs +++ b/GVFS/GVFS.Virtualization/Projection/GitIndexProjection.cs @@ -1097,14 +1097,41 @@ private void UpdatePlaceholders() using (ITracer activity = this.context.Tracer.StartActivity("UpdatePlaceholders", EventLevel.Informational, metadata)) { + int minItemsPerThread = 10; + int numThreads = Math.Max(8, Environment.ProcessorCount); + numThreads = Math.Min(numThreads, placeholderFilesListCopy.Count / minItemsPerThread); + numThreads = Math.Max(numThreads, 1); + ConcurrentHashSet folderPlaceholdersToKeep = new ConcurrentHashSet(); - ConcurrentDictionary updatedPlaceholderList = new ConcurrentDictionary(StringComparer.Ordinal); + + // On platforms that expand on enumeration we need to keep the updated placeholders in a collection + // that supports fast searching (for ReExpandFolder) + ConcurrentDictionary updatedPlaceholderDictionary; + ConcurrentBag updatedPlaceholderBag; + Action addToUpdatePlaceholders; + if (GVFSPlatform.Instance.KernelDriver.EnumerationExpandsDirectories) + { + updatedPlaceholderDictionary = new ConcurrentDictionary( + concurrencyLevel: numThreads, + capacity: placeholderFilesListCopy.Count + placeholderFoldersListCopy.Count, + comparer: StringComparer.Ordinal); + updatedPlaceholderBag = null; + addToUpdatePlaceholders = (data) => updatedPlaceholderDictionary.TryAdd(data.Path, data); + } + else + { + updatedPlaceholderDictionary = null; + updatedPlaceholderBag = new ConcurrentBag(); + addToUpdatePlaceholders = (data) => updatedPlaceholderBag.Add(data); + } + this.ProcessListOnThreads( + numThreads, placeholderFilesListCopy, (placeholderBatch, start, end, blobSizesConnection, availableSizes) => this.BatchPopulateMissingSizesFromRemote(blobSizesConnection, placeholderBatch, start, end, availableSizes), (placeholder, blobSizesConnection, availableSizes) => - this.UpdateOrDeleteFilePlaceholder(blobSizesConnection, placeholder, updatedPlaceholderList, folderPlaceholdersToKeep, availableSizes)); + this.UpdateOrDeleteFilePlaceholder(blobSizesConnection, placeholder, addToUpdatePlaceholders, folderPlaceholdersToKeep, availableSizes)); this.blobSizes.Flush(); @@ -1120,17 +1147,25 @@ private void UpdatePlaceholders() { // Remove folder placeholders before re-expansion to ensure that projection changes that convert a folder to a file work // properly - if (!this.RemoveFolderPlaceholderIfEmpty(folderPlaceholder, updatedPlaceholderList, folderPlaceholdersToKeep)) + if (!this.RemoveFolderPlaceholderIfEmpty(folderPlaceholder, addToUpdatePlaceholders, folderPlaceholdersToKeep)) { if (GVFSPlatform.Instance.KernelDriver.EnumerationExpandsDirectories && folderPlaceholder.IsExpandedFolder) { - this.ReExpandFolder(blobSizesConnection, folderPlaceholder.Path, updatedPlaceholderList, folderPlaceholders); + this.ReExpandFolder(blobSizesConnection, folderPlaceholder.Path, updatedPlaceholderDictionary, folderPlaceholders); } } } } - this.placeholderList.WriteAllEntriesAndFlush(updatedPlaceholderList.Values); + if (GVFSPlatform.Instance.KernelDriver.EnumerationExpandsDirectories) + { + this.placeholderList.WriteAllEntriesAndFlush(updatedPlaceholderDictionary.Values); + } + else + { + this.placeholderList.WriteAllEntriesAndFlush(updatedPlaceholderBag); + } + this.repoMetadata.SetPlaceholdersNeedUpdate(false); TimeSpan duration = activity.Stop(null); @@ -1139,13 +1174,11 @@ private void UpdatePlaceholders() } private void ProcessListOnThreads( + int numThreads, List list, Action, int, int, BlobSizes.BlobSizesConnection, Dictionary> preProcessBatch, Action> processItem) { - int minItemsPerThread = 10; - int numThreads = Math.Max(8, Environment.ProcessorCount); - numThreads = Math.Min(numThreads, list.Count / minItemsPerThread); if (numThreads > 1) { Thread[] processThreads = new Thread[numThreads]; @@ -1358,12 +1391,12 @@ private void ReExpandFolder( /// private bool RemoveFolderPlaceholderIfEmpty( PlaceholderListDatabase.PlaceholderData placeholder, - ConcurrentDictionary updatedPlaceholderList, + Action addToUpdatePlaceholders, ConcurrentHashSet folderPlaceholdersToKeep) { if (folderPlaceholdersToKeep.Contains(placeholder.Path)) { - updatedPlaceholderList.TryAdd(placeholder.Path, placeholder); + addToUpdatePlaceholders(placeholder); return false; } @@ -1381,7 +1414,7 @@ private bool RemoveFolderPlaceholderIfEmpty( FolderData folderData; if (this.TryGetOrAddFolderDataFromCache(placeholder.Path, out folderData)) { - updatedPlaceholderList.TryAdd(placeholder.Path, placeholder); + addToUpdatePlaceholders(placeholder); return false; } } @@ -1394,7 +1427,7 @@ private bool RemoveFolderPlaceholderIfEmpty( break; case FSResult.DirectoryNotEmpty: - updatedPlaceholderList.TryAdd(placeholder.Path, placeholder); + addToUpdatePlaceholders(placeholder); break; case FSResult.FileOrPathNotFound: @@ -1420,7 +1453,7 @@ private bool RemoveFolderPlaceholderIfEmpty( private void UpdateOrDeleteFilePlaceholder( BlobSizes.BlobSizesConnection blobSizesConnection, PlaceholderListDatabase.PlaceholderData placeholder, - ConcurrentDictionary updatedPlaceholderList, + Action addToUpdatePlaceholders, ConcurrentHashSet folderPlaceholdersToKeep, Dictionary availableSizes) { @@ -1437,7 +1470,7 @@ private void UpdateOrDeleteFilePlaceholder( placeholder, string.Empty, result, - updatedPlaceholderList, + addToUpdatePlaceholders, failureReason, parentKey, folderPlaceholdersToKeep, @@ -1480,7 +1513,7 @@ private void UpdateOrDeleteFilePlaceholder( placeholder, projectedSha, result, - updatedPlaceholderList, + addToUpdatePlaceholders, failureReason, parentKey, folderPlaceholdersToKeep, @@ -1488,7 +1521,7 @@ private void UpdateOrDeleteFilePlaceholder( } else { - updatedPlaceholderList.TryAdd(placeholder.Path, placeholder); + addToUpdatePlaceholders(placeholder); this.AddParentFoldersToListToKeep(parentKey, folderPlaceholdersToKeep); } } @@ -1498,7 +1531,7 @@ private void ProcessGvUpdateDeletePlaceholderResult( PlaceholderListDatabase.PlaceholderData placeholder, string projectedSha, FileSystemResult result, - ConcurrentDictionary updatedPlaceholderList, + Action addToUpdatePlaceholders, UpdateFailureReason failureReason, string parentKey, ConcurrentHashSet folderPlaceholdersToKeep, @@ -1510,9 +1543,7 @@ private void ProcessGvUpdateDeletePlaceholderResult( case FSResult.Ok: if (!deleteOperation) { - updatedPlaceholderList.TryAdd( - placeholder.Path, - new PlaceholderListDatabase.PlaceholderData(placeholder.Path, projectedSha)); + addToUpdatePlaceholders(new PlaceholderListDatabase.PlaceholderData(placeholder.Path, projectedSha)); this.AddParentFoldersToListToKeep(parentKey, folderPlaceholdersToKeep); } From 276121130d3ba44019b98973c575988379c7c367 Mon Sep 17 00:00:00 2001 From: William Baker Date: Mon, 10 Sep 2018 14:15:03 -0700 Subject: [PATCH 108/272] Rename addToUpdatePlaceholders --- .../Projection/GitIndexProjection.cs | 30 +++++++++---------- 1 file changed, 15 insertions(+), 15 deletions(-) diff --git a/GVFS/GVFS.Virtualization/Projection/GitIndexProjection.cs b/GVFS/GVFS.Virtualization/Projection/GitIndexProjection.cs index 275092af7..5186bc38d 100644 --- a/GVFS/GVFS.Virtualization/Projection/GitIndexProjection.cs +++ b/GVFS/GVFS.Virtualization/Projection/GitIndexProjection.cs @@ -1108,7 +1108,7 @@ private void UpdatePlaceholders() // that supports fast searching (for ReExpandFolder) ConcurrentDictionary updatedPlaceholderDictionary; ConcurrentBag updatedPlaceholderBag; - Action addToUpdatePlaceholders; + Action addPlaceholderToUpdatedPlaceholders; if (GVFSPlatform.Instance.KernelDriver.EnumerationExpandsDirectories) { updatedPlaceholderDictionary = new ConcurrentDictionary( @@ -1116,13 +1116,13 @@ private void UpdatePlaceholders() capacity: placeholderFilesListCopy.Count + placeholderFoldersListCopy.Count, comparer: StringComparer.Ordinal); updatedPlaceholderBag = null; - addToUpdatePlaceholders = (data) => updatedPlaceholderDictionary.TryAdd(data.Path, data); + addPlaceholderToUpdatedPlaceholders = (data) => updatedPlaceholderDictionary.TryAdd(data.Path, data); } else { updatedPlaceholderDictionary = null; updatedPlaceholderBag = new ConcurrentBag(); - addToUpdatePlaceholders = (data) => updatedPlaceholderBag.Add(data); + addPlaceholderToUpdatedPlaceholders = (data) => updatedPlaceholderBag.Add(data); } this.ProcessListOnThreads( @@ -1131,7 +1131,7 @@ private void UpdatePlaceholders() (placeholderBatch, start, end, blobSizesConnection, availableSizes) => this.BatchPopulateMissingSizesFromRemote(blobSizesConnection, placeholderBatch, start, end, availableSizes), (placeholder, blobSizesConnection, availableSizes) => - this.UpdateOrDeleteFilePlaceholder(blobSizesConnection, placeholder, addToUpdatePlaceholders, folderPlaceholdersToKeep, availableSizes)); + this.UpdateOrDeleteFilePlaceholder(blobSizesConnection, placeholder, addPlaceholderToUpdatedPlaceholders, folderPlaceholdersToKeep, availableSizes)); this.blobSizes.Flush(); @@ -1147,7 +1147,7 @@ private void UpdatePlaceholders() { // Remove folder placeholders before re-expansion to ensure that projection changes that convert a folder to a file work // properly - if (!this.RemoveFolderPlaceholderIfEmpty(folderPlaceholder, addToUpdatePlaceholders, folderPlaceholdersToKeep)) + if (!this.RemoveFolderPlaceholderIfEmpty(folderPlaceholder, addPlaceholderToUpdatedPlaceholders, folderPlaceholdersToKeep)) { if (GVFSPlatform.Instance.KernelDriver.EnumerationExpandsDirectories && folderPlaceholder.IsExpandedFolder) { @@ -1391,12 +1391,12 @@ private void ReExpandFolder( /// private bool RemoveFolderPlaceholderIfEmpty( PlaceholderListDatabase.PlaceholderData placeholder, - Action addToUpdatePlaceholders, + Action addPlaceholderToUpdatedPlaceholders, ConcurrentHashSet folderPlaceholdersToKeep) { if (folderPlaceholdersToKeep.Contains(placeholder.Path)) { - addToUpdatePlaceholders(placeholder); + addPlaceholderToUpdatedPlaceholders(placeholder); return false; } @@ -1414,7 +1414,7 @@ private bool RemoveFolderPlaceholderIfEmpty( FolderData folderData; if (this.TryGetOrAddFolderDataFromCache(placeholder.Path, out folderData)) { - addToUpdatePlaceholders(placeholder); + addPlaceholderToUpdatedPlaceholders(placeholder); return false; } } @@ -1427,7 +1427,7 @@ private bool RemoveFolderPlaceholderIfEmpty( break; case FSResult.DirectoryNotEmpty: - addToUpdatePlaceholders(placeholder); + addPlaceholderToUpdatedPlaceholders(placeholder); break; case FSResult.FileOrPathNotFound: @@ -1453,7 +1453,7 @@ private bool RemoveFolderPlaceholderIfEmpty( private void UpdateOrDeleteFilePlaceholder( BlobSizes.BlobSizesConnection blobSizesConnection, PlaceholderListDatabase.PlaceholderData placeholder, - Action addToUpdatePlaceholders, + Action addPlaceholderToUpdatedPlaceholders, ConcurrentHashSet folderPlaceholdersToKeep, Dictionary availableSizes) { @@ -1470,7 +1470,7 @@ private void UpdateOrDeleteFilePlaceholder( placeholder, string.Empty, result, - addToUpdatePlaceholders, + addPlaceholderToUpdatedPlaceholders, failureReason, parentKey, folderPlaceholdersToKeep, @@ -1513,7 +1513,7 @@ private void UpdateOrDeleteFilePlaceholder( placeholder, projectedSha, result, - addToUpdatePlaceholders, + addPlaceholderToUpdatedPlaceholders, failureReason, parentKey, folderPlaceholdersToKeep, @@ -1521,7 +1521,7 @@ private void UpdateOrDeleteFilePlaceholder( } else { - addToUpdatePlaceholders(placeholder); + addPlaceholderToUpdatedPlaceholders(placeholder); this.AddParentFoldersToListToKeep(parentKey, folderPlaceholdersToKeep); } } @@ -1531,7 +1531,7 @@ private void ProcessGvUpdateDeletePlaceholderResult( PlaceholderListDatabase.PlaceholderData placeholder, string projectedSha, FileSystemResult result, - Action addToUpdatePlaceholders, + Action addPlaceholderToUpdatedPlaceholders, UpdateFailureReason failureReason, string parentKey, ConcurrentHashSet folderPlaceholdersToKeep, @@ -1543,7 +1543,7 @@ private void ProcessGvUpdateDeletePlaceholderResult( case FSResult.Ok: if (!deleteOperation) { - addToUpdatePlaceholders(new PlaceholderListDatabase.PlaceholderData(placeholder.Path, projectedSha)); + addPlaceholderToUpdatedPlaceholders(new PlaceholderListDatabase.PlaceholderData(placeholder.Path, projectedSha)); this.AddParentFoldersToListToKeep(parentKey, folderPlaceholdersToKeep); } From 8e86fb06d604ded7e8011a3a0d9d20a4ab57c367 Mon Sep 17 00:00:00 2001 From: William Baker Date: Mon, 10 Sep 2018 15:10:42 -0700 Subject: [PATCH 109/272] Mark PrefetchVerbTests.cs as NonParallelizable --- .../Tests/EnlistmentPerFixture/PrefetchVerbTests.cs | 1 + 1 file changed, 1 insertion(+) diff --git a/GVFS/GVFS.FunctionalTests/Tests/EnlistmentPerFixture/PrefetchVerbTests.cs b/GVFS/GVFS.FunctionalTests/Tests/EnlistmentPerFixture/PrefetchVerbTests.cs index 3500e0977..cb0a2a4a6 100644 --- a/GVFS/GVFS.FunctionalTests/Tests/EnlistmentPerFixture/PrefetchVerbTests.cs +++ b/GVFS/GVFS.FunctionalTests/Tests/EnlistmentPerFixture/PrefetchVerbTests.cs @@ -9,6 +9,7 @@ namespace GVFS.FunctionalTests.Tests.EnlistmentPerFixture { [TestFixture] + [NonParallelizable] public class PrefetchVerbTests : TestsWithEnlistmentPerFixture { private const string PrefetchCommitsAndTreesLock = "prefetch-commits-trees.lock"; From 28bf2c9439bc82c64c7b9082d226818c0a3c5aa4 Mon Sep 17 00:00:00 2001 From: William Baker Date: Mon, 10 Sep 2018 16:41:27 -0700 Subject: [PATCH 110/272] Changes for next round of PR feedback --- GVFS/GVFS.Common/FileBasedCollection.cs | 2 ++ GVFS/GVFS.Common/PlaceholderListDatabase.cs | 15 +++++---- .../Tools/TestConstants.cs | 2 +- .../Projection/GitIndexProjection.cs | 33 ++++++++++++++++--- 4 files changed, 39 insertions(+), 13 deletions(-) diff --git a/GVFS/GVFS.Common/FileBasedCollection.cs b/GVFS/GVFS.Common/FileBasedCollection.cs index 8fcf5a93a..325bf10d4 100644 --- a/GVFS/GVFS.Common/FileBasedCollection.cs +++ b/GVFS/GVFS.Common/FileBasedCollection.cs @@ -15,6 +15,8 @@ public abstract class FileBasedCollection : IDisposable private const string AddEntryPrefix = "A "; private const string RemoveEntryPrefix = "D "; + + // Use the same newline separator regardless of platform private const string NewLine = "\r\n"; private const int IoFailureRetryDelayMS = 50; private const int IoFailureLoggingThreshold = 500; diff --git a/GVFS/GVFS.Common/PlaceholderListDatabase.cs b/GVFS/GVFS.Common/PlaceholderListDatabase.cs index cf63018e5..5531fd0c0 100644 --- a/GVFS/GVFS.Common/PlaceholderListDatabase.cs +++ b/GVFS/GVFS.Common/PlaceholderListDatabase.cs @@ -8,12 +8,13 @@ namespace GVFS.Common { public class PlaceholderListDatabase : FileBasedCollection { - public const string PartialFolderValue = "@FPARTIAL_000000000000000000000000000000"; - public const string ExpandedFolderValue = "@FEXPANDED_00000000000000000000000000000"; + // Special folder values must: + // - Be 40 characters long + // - Not be a valid SHA-1 value (to avoid collisions with files) + public const string PartialFolderValue = " PARTIAL FOLDER"; + public const string ExpandedFolderValue = " EXPANDED FOLDER"; private const char PathTerminator = '\0'; - private const string FolderValuePrefix = "@F"; - private const string ExpandedFolderPrefix = "@FE"; // This list holds entries that would otherwise be lost because WriteAllEntriesAndFlush has not been called, but a file // snapshot has been taken using GetAllEntries. @@ -134,7 +135,7 @@ public void GetAllEntries(out List filePlaceholders, out List

{ - if (value.StartsWith(FolderValuePrefix, StringComparison.Ordinal)) + if (value == PartialFolderValue || value == ExpandedFolderValue) { folderPlaceholdersFromDisk.Add(new PlaceholderData(path: key, fileShaOrFolderValue: value)); } @@ -291,7 +292,7 @@ public bool IsFolder { get { - return this.Sha.StartsWith(FolderValuePrefix, StringComparison.Ordinal); + return this.Sha == PartialFolderValue || this.IsExpandedFolder; } } @@ -299,7 +300,7 @@ public bool IsExpandedFolder { get { - return this.Sha.StartsWith(ExpandedFolderPrefix, StringComparison.Ordinal); + return this.Sha == ExpandedFolderValue; } } } diff --git a/GVFS/GVFS.FunctionalTests/Tools/TestConstants.cs b/GVFS/GVFS.FunctionalTests/Tools/TestConstants.cs index ca6d8cc9f..ebc6ee28f 100644 --- a/GVFS/GVFS.FunctionalTests/Tools/TestConstants.cs +++ b/GVFS/GVFS.FunctionalTests/Tools/TestConstants.cs @@ -5,7 +5,7 @@ namespace GVFS.FunctionalTests.Tools public static class TestConstants { public const string AllZeroSha = "0000000000000000000000000000000000000000"; - public const string PartialFolderPlaceholderDatabaseValue = "@FPARTIAL_000000000000000000000000000000"; + public const string PartialFolderPlaceholderDatabaseValue = " PARTIAL FOLDER"; public static class DotGit { diff --git a/GVFS/GVFS.Virtualization/Projection/GitIndexProjection.cs b/GVFS/GVFS.Virtualization/Projection/GitIndexProjection.cs index 5186bc38d..ba40f28b6 100644 --- a/GVFS/GVFS.Virtualization/Projection/GitIndexProjection.cs +++ b/GVFS/GVFS.Virtualization/Projection/GitIndexProjection.cs @@ -1104,8 +1104,9 @@ private void UpdatePlaceholders() ConcurrentHashSet folderPlaceholdersToKeep = new ConcurrentHashSet(); - // On platforms that expand on enumeration we need to keep the updated placeholders in a collection - // that supports fast searching (for ReExpandFolder) + // updatedPlaceholderDictionary and updatedPlaceholderBag are mutually exclusive. + // - On platforms that expand on enumeration: updatedPlaceholderDictionary is used (required for ReExpandFolder) + // - On platforms that do not expand on enumeration: updatedPlaceholderBag is used (for speed) ConcurrentDictionary updatedPlaceholderDictionary; ConcurrentBag updatedPlaceholderBag; Action addPlaceholderToUpdatedPlaceholders; @@ -1140,9 +1141,11 @@ private void UpdatePlaceholders() // A hash of the folder placeholders is only required if the platform expands directories HashSet folderPlaceholders = GVFSPlatform.Instance.KernelDriver.EnumerationExpandsDirectories ? - new HashSet(placeholderFoldersListCopy.Select(x => x.Path), StringComparer.OrdinalIgnoreCase) : - null; + new HashSet(placeholderFoldersListCopy.Select(x => x.Path), StringComparer.OrdinalIgnoreCase) : + null; + // Order the folders in decscending order so that we walk the tree from bottom up (ensuring child folders are deleted before + // their parents) foreach (PlaceholderListDatabase.PlaceholderData folderPlaceholder in placeholderFoldersListCopy.OrderByDescending(x => x.Path)) { // Remove folder placeholders before re-expansion to ensure that projection changes that convert a folder to a file work @@ -1151,6 +1154,12 @@ private void UpdatePlaceholders() { if (GVFSPlatform.Instance.KernelDriver.EnumerationExpandsDirectories && folderPlaceholder.IsExpandedFolder) { + if (updatedPlaceholderDictionary == null) + { + throw new InvalidOperationException( + $"{nameof(updatedPlaceholderDictionary)} must be used when enumeration expands directories"); + } + this.ReExpandFolder(blobSizesConnection, folderPlaceholder.Path, updatedPlaceholderDictionary, folderPlaceholders); } } @@ -1159,10 +1168,22 @@ private void UpdatePlaceholders() if (GVFSPlatform.Instance.KernelDriver.EnumerationExpandsDirectories) { + if (updatedPlaceholderBag != null) + { + throw new InvalidOperationException( + $"{nameof(updatedPlaceholderBag)} should only be used when enumeration does not expand directories"); + } + this.placeholderList.WriteAllEntriesAndFlush(updatedPlaceholderDictionary.Values); } else { + if (updatedPlaceholderDictionary != null) + { + throw new InvalidOperationException( + $"{nameof(updatedPlaceholderDictionary)} should only be used when enumeration expands directories"); + } + this.placeholderList.WriteAllEntriesAndFlush(updatedPlaceholderBag); } @@ -1332,6 +1353,7 @@ private void ReExpandFolder( return; } + // TODO(Mac): Issue #255, batch file sizes up-front for the new placeholders written by ReExpandFolder folderData.PopulateSizes( this.context.Tracer, this.gitObjects, @@ -1403,7 +1425,8 @@ private bool RemoveFolderPlaceholderIfEmpty( if (GVFSPlatform.Instance.KernelDriver.EnumerationExpandsDirectories) { // If enumeration expands directories we should leave folder placeholders - // that are still in the projection on disk (even if they are empty). + // that are still in the projection on disk (they might still be physically empty + // on disk if they've not been expanded). // // If enumeration does not expand directories there is no harm in deleting empty // folder placeholders that are in the projection as they will be re-projected during From f7ee53fe69dfb6c9f62196d80330b5ca58d5d62d Mon Sep 17 00:00:00 2001 From: William Baker Date: Tue, 11 Sep 2018 10:20:14 -0700 Subject: [PATCH 111/272] PR Feedback: Cleanup DiskLayoutUpgradeTests --- .../Windows/Tests/DiskLayoutUpgradeTests.cs | 43 +++++++------------ 1 file changed, 16 insertions(+), 27 deletions(-) diff --git a/GVFS/GVFS.FunctionalTests.Windows/Windows/Tests/DiskLayoutUpgradeTests.cs b/GVFS/GVFS.FunctionalTests.Windows/Windows/Tests/DiskLayoutUpgradeTests.cs index 2d407de3b..1abbbaafb 100644 --- a/GVFS/GVFS.FunctionalTests.Windows/Windows/Tests/DiskLayoutUpgradeTests.cs +++ b/GVFS/GVFS.FunctionalTests.Windows/Windows/Tests/DiskLayoutUpgradeTests.cs @@ -367,22 +367,27 @@ private void PerformIOBeforePlaceholderDatabaseUpgradeTest() ProcessHelper.Run("CMD.exe", "/C mklink /D " + link + " " + target); } + private void PlaceholderDatabaseShouldIncludeCommonLinesForUpgradeTestIO(string[] placeholderLines) + { + placeholderLines.ShouldContain(x => x.Contains("A Readme.md")); + placeholderLines.ShouldContain(x => x.Contains("A Scripts\\RunUnitTests.bat")); + placeholderLines.ShouldContain(x => x.Contains("A GVFS\\GVFS.Common\\Git\\GitRefs.cs")); + placeholderLines.ShouldContain(x => x.Contains("A .gitignore")); + placeholderLines.ShouldContain(x => x == "A Scripts\0" + TestConstants.PartialFolderPlaceholderDatabaseValue); + placeholderLines.ShouldContain(x => x == "A GVFS\0" + TestConstants.PartialFolderPlaceholderDatabaseValue); + placeholderLines.ShouldContain(x => x == "A GVFS\\GVFS.Common\0" + TestConstants.PartialFolderPlaceholderDatabaseValue); + placeholderLines.ShouldContain(x => x == "A GVFS\\GVFS.Common\\Git\0" + TestConstants.PartialFolderPlaceholderDatabaseValue); + placeholderLines.ShouldContain(x => x == "A GVFS\\GVFS.Tests\0" + TestConstants.PartialFolderPlaceholderDatabaseValue); + } + private string[] GetPlaceholderDatabaseLinesBeforeUpgrade(string placeholderDatabasePath) { placeholderDatabasePath.ShouldBeAFile(this.fileSystem); string[] lines = this.fileSystem.ReadAllText(placeholderDatabasePath).Split(new char[] { '\r', '\n' }, StringSplitOptions.RemoveEmptyEntries); lines.Length.ShouldEqual(12); - lines.ShouldContain(x => x.Contains("Readme.md")); - lines.ShouldContain(x => x.Contains("Scripts\\RunUnitTests.bat")); - lines.ShouldContain(x => x.Contains("GVFS\\GVFS.Common\\Git\\GitRefs.cs")); + this.PlaceholderDatabaseShouldIncludeCommonLinesForUpgradeTestIO(lines); lines.ShouldContain(x => x.Contains("A GVFS\\GVFS.Tests\\Properties\\AssemblyInfo.cs")); - lines.ShouldContain(x => x.Contains("A .gitignore")); lines.ShouldContain(x => x == "D GVFS\\GVFS.Tests\\Properties\\AssemblyInfo.cs"); - lines.ShouldContain(x => x == "A Scripts\0" + TestConstants.PartialFolderPlaceholderDatabaseValue); - lines.ShouldContain(x => x == "A GVFS\0" + TestConstants.PartialFolderPlaceholderDatabaseValue); - lines.ShouldContain(x => x == "A GVFS\\GVFS.Common\0" + TestConstants.PartialFolderPlaceholderDatabaseValue); - lines.ShouldContain(x => x == "A GVFS\\GVFS.Common\\Git\0" + TestConstants.PartialFolderPlaceholderDatabaseValue); - lines.ShouldContain(x => x == "A GVFS\\GVFS.Tests\0" + TestConstants.PartialFolderPlaceholderDatabaseValue); lines.ShouldContain(x => x == "A GVFS\\GVFS.Tests\\Properties\0" + TestConstants.PartialFolderPlaceholderDatabaseValue); return lines; } @@ -392,15 +397,7 @@ private string[] GetPlaceholderDatabaseLinesAfterUpgradeFrom12_1(string placehol placeholderDatabasePath.ShouldBeAFile(this.fileSystem); string[] lines = this.fileSystem.ReadAllText(placeholderDatabasePath).Split(new char[] { '\r', '\n' }, StringSplitOptions.RemoveEmptyEntries); lines.Length.ShouldEqual(9); - lines.ShouldContain(x => x.Contains("Readme.md")); - lines.ShouldContain(x => x.Contains("Scripts\\RunUnitTests.bat")); - lines.ShouldContain(x => x.Contains("GVFS\\GVFS.Common\\Git\\GitRefs.cs")); - lines.ShouldContain(x => x.Contains("A .gitignore")); - lines.ShouldContain(x => x == "A Scripts\0" + TestConstants.PartialFolderPlaceholderDatabaseValue); - lines.ShouldContain(x => x == "A GVFS\0" + TestConstants.PartialFolderPlaceholderDatabaseValue); - lines.ShouldContain(x => x == "A GVFS\\GVFS.Common\0" + TestConstants.PartialFolderPlaceholderDatabaseValue); - lines.ShouldContain(x => x == "A GVFS\\GVFS.Common\\Git\0" + TestConstants.PartialFolderPlaceholderDatabaseValue); - lines.ShouldContain(x => x == "A GVFS\\GVFS.Tests\0" + TestConstants.PartialFolderPlaceholderDatabaseValue); + this.PlaceholderDatabaseShouldIncludeCommonLinesForUpgradeTestIO(lines); return lines; } @@ -409,15 +406,7 @@ private string[] GetPlaceholderDatabaseLinesAfterUpgradeFrom16(string placeholde placeholderDatabasePath.ShouldBeAFile(this.fileSystem); string[] lines = this.fileSystem.ReadAllText(placeholderDatabasePath).Split(new char[] { '\r', '\n' }, StringSplitOptions.RemoveEmptyEntries); lines.Length.ShouldEqual(10); - lines.ShouldContain(x => x.Contains("Readme.md")); - lines.ShouldContain(x => x.Contains("Scripts\\RunUnitTests.bat")); - lines.ShouldContain(x => x.Contains("GVFS\\GVFS.Common\\Git\\GitRefs.cs")); - lines.ShouldContain(x => x.Contains("A .gitignore")); - lines.ShouldContain(x => x == "A Scripts\0" + TestConstants.PartialFolderPlaceholderDatabaseValue); - lines.ShouldContain(x => x == "A GVFS\0" + TestConstants.PartialFolderPlaceholderDatabaseValue); - lines.ShouldContain(x => x == "A GVFS\\GVFS.Common\0" + TestConstants.PartialFolderPlaceholderDatabaseValue); - lines.ShouldContain(x => x == "A GVFS\\GVFS.Common\\Git\0" + TestConstants.PartialFolderPlaceholderDatabaseValue); - lines.ShouldContain(x => x == "A GVFS\\GVFS.Tests\0" + TestConstants.PartialFolderPlaceholderDatabaseValue); + this.PlaceholderDatabaseShouldIncludeCommonLinesForUpgradeTestIO(lines); lines.ShouldContain(x => x == "A GVFS\\GVFS.Tests\\Properties\0" + TestConstants.PartialFolderPlaceholderDatabaseValue); return lines; } From 5165e1ac74bd3e11149ed0afffea005957f0456e Mon Sep 17 00:00:00 2001 From: William Baker Date: Thu, 6 Sep 2018 13:34:29 -0700 Subject: [PATCH 112/272] Enable UpdatePlaceholderTests on Mac --- .../UpdatePlaceholderTests.cs | 128 ++++++++---------- 1 file changed, 55 insertions(+), 73 deletions(-) diff --git a/GVFS/GVFS.FunctionalTests/Tests/EnlistmentPerFixture/UpdatePlaceholderTests.cs b/GVFS/GVFS.FunctionalTests/Tests/EnlistmentPerFixture/UpdatePlaceholderTests.cs index c36877f58..cb938becb 100644 --- a/GVFS/GVFS.FunctionalTests/Tests/EnlistmentPerFixture/UpdatePlaceholderTests.cs +++ b/GVFS/GVFS.FunctionalTests/Tests/EnlistmentPerFixture/UpdatePlaceholderTests.cs @@ -8,12 +8,12 @@ using System.Collections.Generic; using System.Diagnostics; using System.IO; +using System.Runtime.InteropServices; namespace GVFS.FunctionalTests.Tests.EnlistmentPerFixture { [TestFixture] [Category(Categories.GitCommands)] - [Category(Categories.MacTODO.M3)] public class UpdatePlaceholderTests : TestsWithEnlistmentPerFixture { private const string TestParentFolderName = "Test_EPF_UpdatePlaceholderTests"; @@ -34,7 +34,9 @@ public virtual void SetupForTest() this.GitStatusShouldBeClean(NewFilesAndChangesCommitId); } + // WindowsOnly because test requires Windows specific file sharing behavior [TestCase, Order(1)] + [Category(Categories.WindowsOnly)] public void LockToPreventDelete_SingleFile() { string testFile1Contents = "TestContentsLockToPreventDelete \r\n"; @@ -42,10 +44,8 @@ public void LockToPreventDelete_SingleFile() string testFile1Path = this.Enlistment.GetVirtualPathTo(Path.Combine(TestParentFolderName, "LockToPreventDelete", testFile1Name)); testFile1Path.ShouldBeAFile(this.fileSystem).WithContents(testFile1Contents); - using (SafeFileHandle testFile1Handle = this.CreateFile(testFile1Path, FileShare.Read)) + using (FileStream testFile1 = File.Open(testFile1Path, FileMode.Open, FileAccess.Read, FileShare.Read)) { - testFile1Handle.IsInvalid.ShouldEqual(false); - ProcessResult result = this.InvokeGitAgainstGVFSRepo("checkout " + OldCommitId); result.Errors.ShouldContain( "GVFS was unable to delete the following files. To recover, close all handles to the files and run these commands:", @@ -70,7 +70,9 @@ public void LockToPreventDelete_SingleFile() testFile1Path.ShouldBeAFile(this.fileSystem).WithContents(testFile1Contents); } + // WindowsOnly because test requires Windows specific file sharing behavior [TestCase, Order(2)] + [Category(Categories.WindowsOnly)] public void LockToPreventDelete_MultipleFiles() { string testFile2Contents = "TestContentsLockToPreventDelete2 \r\n"; @@ -89,14 +91,10 @@ public void LockToPreventDelete_MultipleFiles() testFile3Path.ShouldBeAFile(this.fileSystem).WithContents(testFile3Contents); testFile4Path.ShouldBeAFile(this.fileSystem).WithContents(testFile4Contents); - using (SafeFileHandle testFile2Handle = this.CreateFile(testFile2Path, FileShare.Read)) - using (SafeFileHandle testFile3Handle = this.CreateFile(testFile3Path, FileShare.Read)) - using (SafeFileHandle testFile4Handle = this.CreateFile(testFile4Path, FileShare.Read)) + using (FileStream testFile2 = File.Open(testFile2Path, FileMode.Open, FileAccess.Read, FileShare.Read)) + using (FileStream testFile3 = File.Open(testFile3Path, FileMode.Open, FileAccess.Read, FileShare.Read)) + using (FileStream testFile4 = File.Open(testFile4Path, FileMode.Open, FileAccess.Read, FileShare.Read)) { - testFile2Handle.IsInvalid.ShouldEqual(false); - testFile3Handle.IsInvalid.ShouldEqual(false); - testFile4Handle.IsInvalid.ShouldEqual(false); - ProcessResult result = this.InvokeGitAgainstGVFSRepo("checkout " + OldCommitId); result.Errors.ShouldContain( "GVFS was unable to delete the following files. To recover, close all handles to the files and run these commands:", @@ -136,7 +134,9 @@ public void LockToPreventDelete_MultipleFiles() testFile4Path.ShouldBeAFile(this.fileSystem).WithContents(testFile4Contents); } + // WindowsOnly because test requires Windows specific file sharing behavior [TestCase, Order(3)] + [Category(Categories.WindowsOnly)] public void LockToPreventUpdate_SingleFile() { string testFile1Contents = "Commit2LockToPreventUpdate \r\n"; @@ -145,10 +145,8 @@ public void LockToPreventUpdate_SingleFile() string testFile1Path = this.Enlistment.GetVirtualPathTo(Path.Combine(TestParentFolderName, "LockToPreventUpdate", testFile1Name)); testFile1Path.ShouldBeAFile(this.fileSystem).WithContents(testFile1Contents); - using (SafeFileHandle testFile1Handle = this.CreateFile(testFile1Path, FileShare.Read)) + using (FileStream testFile1 = File.Open(testFile1Path, FileMode.Open, FileAccess.Read, FileShare.Read)) { - testFile1Handle.IsInvalid.ShouldEqual(false); - ProcessResult result = this.InvokeGitAgainstGVFSRepo("checkout " + OldCommitId); result.Errors.ShouldContain( "GVFS was unable to update the following files. To recover, close all handles to the files and run these commands:", @@ -173,7 +171,9 @@ public void LockToPreventUpdate_SingleFile() testFile1Path.ShouldBeAFile(this.fileSystem).WithContents(testFile1Contents); } + // WindowsOnly because test requires Windows specific file sharing behavior [TestCase, Order(4)] + [Category(Categories.WindowsOnly)] public void LockToPreventUpdate_MultipleFiles() { string testFile2Contents = "Commit2LockToPreventUpdate2 \r\n"; @@ -196,14 +196,10 @@ public void LockToPreventUpdate_MultipleFiles() testFile3Path.ShouldBeAFile(this.fileSystem).WithContents(testFile3Contents); testFile4Path.ShouldBeAFile(this.fileSystem).WithContents(testFile4Contents); - using (SafeFileHandle testFile2Handle = this.CreateFile(testFile2Path, FileShare.Read)) - using (SafeFileHandle testFile3Handle = this.CreateFile(testFile3Path, FileShare.Read)) - using (SafeFileHandle testFile4Handle = this.CreateFile(testFile4Path, FileShare.Read)) + using (FileStream testFile2 = File.Open(testFile2Path, FileMode.Open, FileAccess.Read, FileShare.Read)) + using (FileStream testFile3 = File.Open(testFile3Path, FileMode.Open, FileAccess.Read, FileShare.Read)) + using (FileStream testFile4 = File.Open(testFile4Path, FileMode.Open, FileAccess.Read, FileShare.Read)) { - testFile2Handle.IsInvalid.ShouldEqual(false); - testFile3Handle.IsInvalid.ShouldEqual(false); - testFile4Handle.IsInvalid.ShouldEqual(false); - ProcessResult result = this.InvokeGitAgainstGVFSRepo("checkout " + OldCommitId); result.Errors.ShouldContain( "GVFS was unable to update the following files. To recover, close all handles to the files and run these commands:", @@ -241,7 +237,9 @@ public void LockToPreventUpdate_MultipleFiles() testFile4Path.ShouldBeAFile(this.fileSystem).WithContents(testFile4Contents); } + // WindowsOnly because test requires Windows specific file sharing behavior [TestCase, Order(5)] + [Category(Categories.WindowsOnly)] public void LockToPreventUpdateAndDelete() { string testFileUpdate1Contents = "Commit2LockToPreventUpdateAndDelete \r\n"; @@ -276,20 +274,13 @@ public void LockToPreventUpdateAndDelete() testFileDelete2Path.ShouldBeAFile(this.fileSystem).WithContents(testFileDelete2Contents); testFileDelete3Path.ShouldBeAFile(this.fileSystem).WithContents(testFileDelete3Contents); - using (SafeFileHandle testFileUpdate1Handle = this.CreateFile(testFileUpdate1Path, FileShare.Read)) - using (SafeFileHandle testFileUpdate2Handle = this.CreateFile(testFileUpdate2Path, FileShare.Read)) - using (SafeFileHandle testFileUpdate3Handle = this.CreateFile(testFileUpdate3Path, FileShare.Read)) - using (SafeFileHandle testFileDelete1Handle = this.CreateFile(testFileDelete1Path, FileShare.Read)) - using (SafeFileHandle testFileDelete2Handle = this.CreateFile(testFileDelete2Path, FileShare.Read)) - using (SafeFileHandle testFileDelete3Handle = this.CreateFile(testFileDelete3Path, FileShare.Read)) + using (FileStream testFileUpdate1 = File.Open(testFileUpdate1Path, FileMode.Open, FileAccess.Read, FileShare.Read)) + using (FileStream testFileUpdate2 = File.Open(testFileUpdate2Path, FileMode.Open, FileAccess.Read, FileShare.Read)) + using (FileStream testFileUpdate3 = File.Open(testFileUpdate3Path, FileMode.Open, FileAccess.Read, FileShare.Read)) + using (FileStream testFileDelete1 = File.Open(testFileDelete1Path, FileMode.Open, FileAccess.Read, FileShare.Read)) + using (FileStream testFileDelete2 = File.Open(testFileDelete2Path, FileMode.Open, FileAccess.Read, FileShare.Read)) + using (FileStream testFileDelete3 = File.Open(testFileDelete3Path, FileMode.Open, FileAccess.Read, FileShare.Read)) { - testFileUpdate1Handle.IsInvalid.ShouldEqual(false); - testFileUpdate2Handle.IsInvalid.ShouldEqual(false); - testFileUpdate3Handle.IsInvalid.ShouldEqual(false); - testFileDelete1Handle.IsInvalid.ShouldEqual(false); - testFileDelete2Handle.IsInvalid.ShouldEqual(false); - testFileDelete3Handle.IsInvalid.ShouldEqual(false); - ProcessResult checkoutResult = GitProcess.InvokeProcess(this.Enlistment.RepoRoot, "checkout " + OldCommitId); checkoutResult.Errors.ShouldContain( "HEAD is now at " + OldCommitId, @@ -365,24 +356,18 @@ public void LockWithFullShareUpdateAndDelete() if (this.CanUpdateAndDeletePlaceholdersWithOpenHandles()) { - using (SafeFileHandle testFileUpdate4Handle = this.CreateFile(testFileUpdate4Path, FileShare.Read | FileShare.Delete)) - using (SafeFileHandle testFileDelete4Handle = this.CreateFile(testFileDelete4Path, FileShare.Read | FileShare.Delete)) + using (FileStream testFileUpdate4 = File.Open(testFileUpdate4Path, FileMode.Open, FileAccess.Read, FileShare.Read | FileShare.Delete)) + using (FileStream testFileDelete4 = File.Open(testFileDelete4Path, FileMode.Open, FileAccess.Read, FileShare.Read | FileShare.Delete)) { - testFileUpdate4Handle.IsInvalid.ShouldEqual(false); - testFileDelete4Handle.IsInvalid.ShouldEqual(false); - this.GitCheckoutCommitId(OldCommitId); this.GitStatusShouldBeClean(OldCommitId); } } else { - using (SafeFileHandle testFileUpdate4Handle = this.CreateFile(testFileUpdate4Path, FileShare.Read | FileShare.Delete)) - using (SafeFileHandle testFileDelete4Handle = this.CreateFile(testFileDelete4Path, FileShare.Read | FileShare.Delete)) + using (FileStream testFileUpdate4Handle = File.Open(testFileUpdate4Path, FileMode.Open, FileAccess.Read, FileShare.Read | FileShare.Delete)) + using (FileStream testFileDelete4Handle = File.Open(testFileDelete4Path, FileMode.Open, FileAccess.Read, FileShare.Read | FileShare.Delete)) { - testFileUpdate4Handle.IsInvalid.ShouldEqual(false); - testFileDelete4Handle.IsInvalid.ShouldEqual(false); - ProcessResult checkoutResult = GitProcess.InvokeProcess(this.Enlistment.RepoRoot, "checkout " + OldCommitId); checkoutResult.Errors.ShouldContain( "HEAD is now at " + OldCommitId, @@ -451,7 +436,9 @@ public void FileProjectedAfterPlaceholderDeleteFileAndCheckout() testFile3Path.ShouldBeAFile(this.fileSystem).WithContents(testFile3Contents); } + // WindowsOnly because test requires Windows specific file sharing behavior [TestCase, Order(8)] + [Category(Categories.WindowsOnly)] public void LockMoreThanMaxReportedFileNames() { string updateFilesFolder = "FilesToUpdate"; @@ -463,22 +450,24 @@ public void LockMoreThanMaxReportedFileNames() this.Enlistment.GetVirtualPathTo(Path.Combine(TestParentFolderName, "MaxFileListCount", deleteFilesFolder, i.ToString() + ".txt")).ShouldBeAFile(this.fileSystem); } - List openHandles = new List(); + List openFiles = new List(); try { for (int i = 1; i <= 51; ++i) { - SafeFileHandle handle = this.CreateFile( + FileStream file = File.Open( this.Enlistment.GetVirtualPathTo(Path.Combine(TestParentFolderName, "MaxFileListCount", updateFilesFolder, i.ToString() + ".txt")), + FileMode.Open, + FileAccess.Read, FileShare.Read); - openHandles.Add(handle); - handle.IsInvalid.ShouldEqual(false); + openFiles.Add(file); - handle = this.CreateFile( + file = File.Open( this.Enlistment.GetVirtualPathTo(Path.Combine(TestParentFolderName, "MaxFileListCount", deleteFilesFolder, i.ToString() + ".txt")), + FileMode.Open, + FileAccess.Read, FileShare.Read); - openHandles.Add(handle); - handle.IsInvalid.ShouldEqual(false); + openFiles.Add(file); } ProcessResult result = this.InvokeGitAgainstGVFSRepo("checkout " + OldCommitId); @@ -501,9 +490,9 @@ public void LockMoreThanMaxReportedFileNames() } finally { - foreach (SafeFileHandle handle in openHandles) + foreach (FileStream file in openFiles) { - handle.Dispose(); + file.Dispose(); } } @@ -569,30 +558,23 @@ private void GitCheckoutCommitId(string commitId) this.InvokeGitAgainstGVFSRepo("checkout " + commitId).Errors.ShouldContain("HEAD is now at " + commitId); } - private SafeFileHandle CreateFile(string path, FileShare shareMode) - { - return NativeMethods.CreateFile( - path, - (uint)FileAccess.Read, - shareMode, - IntPtr.Zero, - FileMode.Open, - (uint)FileAttributes.Normal, - IntPtr.Zero); - } - private bool CanUpdateAndDeletePlaceholdersWithOpenHandles() { - // https://msdn.microsoft.com/en-us/library/windows/desktop/ms724429(v=vs.85).aspx - FileVersionInfo kernel32Info = FileVersionInfo.GetVersionInfo(Path.Combine(Environment.SystemDirectory, "kernel32.dll")); - - // 16248 is first build with support - see 12658248 for details - if (kernel32Info.FileBuildPart >= 16248) - { - return true; + if (RuntimeInformation.IsOSPlatform(OSPlatform.Windows)) + { + // https://msdn.microsoft.com/en-us/library/windows/desktop/ms724429(v=vs.85).aspx + FileVersionInfo kernel32Info = FileVersionInfo.GetVersionInfo(Path.Combine(Environment.SystemDirectory, "kernel32.dll")); + + // 16248 is first build with support - see 12658248 for details + if (kernel32Info.FileBuildPart >= 16248) + { + return true; + } + + return false; } - return false; + return true; } } } From f6ac92e506a04d3a15746a437a8afd11d5fe6be3 Mon Sep 17 00:00:00 2001 From: William Baker Date: Tue, 11 Sep 2018 16:07:04 -0700 Subject: [PATCH 113/272] Don't update ModifiedPaths.dat for hardlinks created outside of repo --- ...iedPathsTests.cs => ModifiedPathsTests.cs} | 78 +++++++++++++++++-- .../FileSystem/FileSystemVirtualizer.cs | 19 +++-- 2 files changed, 83 insertions(+), 14 deletions(-) rename GVFS/GVFS.FunctionalTests/Tests/EnlistmentPerTestCase/{PersistedModifiedPathsTests.cs => ModifiedPathsTests.cs} (62%) diff --git a/GVFS/GVFS.FunctionalTests/Tests/EnlistmentPerTestCase/PersistedModifiedPathsTests.cs b/GVFS/GVFS.FunctionalTests/Tests/EnlistmentPerTestCase/ModifiedPathsTests.cs similarity index 62% rename from GVFS/GVFS.FunctionalTests/Tests/EnlistmentPerTestCase/PersistedModifiedPathsTests.cs rename to GVFS/GVFS.FunctionalTests/Tests/EnlistmentPerTestCase/ModifiedPathsTests.cs index ac5ade5e3..53a655a54 100644 --- a/GVFS/GVFS.FunctionalTests/Tests/EnlistmentPerTestCase/PersistedModifiedPathsTests.cs +++ b/GVFS/GVFS.FunctionalTests/Tests/EnlistmentPerTestCase/ModifiedPathsTests.cs @@ -6,12 +6,12 @@ using System; using System.IO; using System.Linq; +using System.Runtime.InteropServices; namespace GVFS.FunctionalTests.Tests.EnlistmentPerTestCase { [TestFixture] - [Category(Categories.MacTODO.M2)] - public class PersistedModifiedPathsTests : TestsWithEnlistmentPerTestCase + public class ModifiedPathsTests : TestsWithEnlistmentPerTestCase { private static readonly string FileToAdd = Path.Combine("GVFS", "TestAddFile.txt"); private static readonly string FileToUpdate = Path.Combine("GVFS", "GVFS", "Program.cs"); @@ -26,7 +26,7 @@ public class PersistedModifiedPathsTests : TestsWithEnlistmentPerTestCase private static readonly string FileToCreateOutsideRepo = "PersistedSparseExcludeTests_outsideRepo.txt"; private static readonly string FolderToCreateOutsideRepo = "PersistedSparseExcludeTests_outsideFolder"; private static readonly string FolderToDelete = "Scripts"; - private static readonly string ExpectedModifiedFilesContents = + private static readonly string ExpectedModifiedFilesContentsAfterRemount = @"A .gitattributes A GVFS/TestAddFile.txt A GVFS/GVFS/Program.cs @@ -47,8 +47,9 @@ A Scripts/RunUnitTests.bat A Scripts/ "; + [Category(Categories.MacTODO.M2)] [TestCaseSource(typeof(FileSystemRunner), FileSystemRunner.TestRunners)] - public void ExcludeSparseFileSavedAfterRemount(FileSystemRunner fileSystem) + public void ModifiedPathsSavedAfterRemount(FileSystemRunner fileSystem) { string fileToAdd = this.Enlistment.GetVirtualPathTo(FileToAdd); fileSystem.WriteAllText(fileToAdd, "Contents for the new file"); @@ -116,8 +117,73 @@ public void ExcludeSparseFileSavedAfterRemount(FileSystemRunner fileSystem) modifiedPathsDatabase.ShouldBeAFile(fileSystem); using (StreamReader reader = new StreamReader(File.Open(modifiedPathsDatabase, FileMode.Open, FileAccess.Read, FileShare.ReadWrite))) { - reader.ReadToEnd().ShouldEqual(ExpectedModifiedFilesContents); + reader.ReadToEnd().ShouldEqual(ExpectedModifiedFilesContentsAfterRemount); } - } + } + + [TestCaseSource(typeof(HardLinkRunners), HardLinkRunners.TestRunners)] + public void ModifiedPathsCorrectAfterHardLinking(FileSystemRunner fileSystem) + { + const string ExpectedModifiedFilesContentsAfterHardlinks = +@"A .gitattributes +A LinkToReadme.md +A LinkToFileOutsideSrc.txt +"; + + // Create a link from src\LinkToReadme.md to src\Readme.md + string existingFileInRepoPath = this.Enlistment.GetVirtualPathTo("Readme.md"); + string contents = existingFileInRepoPath.ShouldBeAFile(fileSystem).WithContents(); + string hardLinkToFileInRepoPath = this.Enlistment.GetVirtualPathTo("LinkToReadme.md"); + hardLinkToFileInRepoPath.ShouldNotExistOnDisk(fileSystem); + fileSystem.CreateHardLink(hardLinkToFileInRepoPath, existingFileInRepoPath); + hardLinkToFileInRepoPath.ShouldBeAFile(fileSystem).WithContents(contents); + + // Create a link from src\LinkToFileOutsideSrc.txt to FileOutsideRepo.txt + string fileOutsideOfRepoPath = Path.Combine(this.Enlistment.EnlistmentRoot, "FileOutsideRepo.txt"); + string fileOutsideOfRepoContents = "File outside of repo"; + fileOutsideOfRepoPath.ShouldNotExistOnDisk(fileSystem); + fileSystem.WriteAllText(fileOutsideOfRepoPath, fileOutsideOfRepoContents); + string hardLinkToFileOutsideRepoPath = this.Enlistment.GetVirtualPathTo("LinkToFileOutsideSrc.txt"); + hardLinkToFileOutsideRepoPath.ShouldNotExistOnDisk(fileSystem); + fileSystem.CreateHardLink(hardLinkToFileOutsideRepoPath, fileOutsideOfRepoPath); + hardLinkToFileOutsideRepoPath.ShouldBeAFile(fileSystem).WithContents(fileOutsideOfRepoContents); + + // Create a link from LinkOutsideSrcToInsideSrc.cs to src\GVFS\GVFS\Program.cs + string secondFileInRepoPath = this.Enlistment.GetVirtualPathTo("GVFS", "GVFS", "Program.cs"); + contents = secondFileInRepoPath.ShouldBeAFile(fileSystem).WithContents(); + string hardLinkOutsideRepoToFileInRepoPath = Path.Combine(this.Enlistment.EnlistmentRoot, "LinkOutsideSrcToInsideSrc.cs"); + hardLinkOutsideRepoToFileInRepoPath.ShouldNotExistOnDisk(fileSystem); + fileSystem.CreateHardLink(hardLinkOutsideRepoToFileInRepoPath, secondFileInRepoPath); + hardLinkOutsideRepoToFileInRepoPath.ShouldBeAFile(fileSystem).WithContents(contents); + + string modifiedPathsDatabase = Path.Combine(this.Enlistment.DotGVFSRoot, TestConstants.Databases.ModifiedPaths); + modifiedPathsDatabase.ShouldBeAFile(fileSystem); + using (StreamReader reader = new StreamReader(File.Open(modifiedPathsDatabase, FileMode.Open, FileAccess.Read, FileShare.ReadWrite))) + { + reader.ReadToEnd().ShouldEqual(ExpectedModifiedFilesContentsAfterHardlinks); + } + } + + private class HardLinkRunners + { + public const string TestRunners = "Runners"; + + public static object[] Runners + { + get + { + if (RuntimeInformation.IsOSPlatform(OSPlatform.Windows)) + { + return new[] + { + new object[] { new CmdRunner() }, + new object[] { new BashRunner() }, + }; + } + + return new[] { new object[] { new BashRunner() } }; + } + } + } } } diff --git a/GVFS/GVFS.Virtualization/FileSystem/FileSystemVirtualizer.cs b/GVFS/GVFS.Virtualization/FileSystem/FileSystemVirtualizer.cs index 4f811ce76..c77e94669 100644 --- a/GVFS/GVFS.Virtualization/FileSystem/FileSystemVirtualizer.cs +++ b/GVFS/GVFS.Virtualization/FileSystem/FileSystemVirtualizer.cs @@ -241,15 +241,18 @@ protected void OnHardLinkCreated(string relativeExistingFilePath, string relativ { try { - bool pathInDotGit = FileSystemCallbacks.IsPathInsideDotGit(relativeNewLinkPath); - - if (pathInDotGit) - { - this.OnDotGitFileOrFolderChanged(relativeNewLinkPath); - } - else + if (!string.IsNullOrEmpty(relativeNewLinkPath)) { - this.FileSystemCallbacks.OnFileHardLinkCreated(relativeNewLinkPath); + bool pathInDotGit = FileSystemCallbacks.IsPathInsideDotGit(relativeNewLinkPath); + + if (pathInDotGit) + { + this.OnDotGitFileOrFolderChanged(relativeNewLinkPath); + } + else + { + this.FileSystemCallbacks.OnFileHardLinkCreated(relativeNewLinkPath); + } } } catch (Exception e) From 5973857a9d0504b29a5b6f652762d0217391bee8 Mon Sep 17 00:00:00 2001 From: William Baker Date: Wed, 12 Sep 2018 10:24:39 -0700 Subject: [PATCH 114/272] PR Feedback: Move Windows specific UpdatePlaceholderTests tests into their own class --- .../GVFS.FunctionalTests.Windows.csproj | 1 + .../Tests/WindowsUpdatePlaceholderTests.cs.cs | 476 ++++++++++++++++++ .../UpdatePlaceholderTests.cs | 391 +------------- 3 files changed, 480 insertions(+), 388 deletions(-) create mode 100644 GVFS/GVFS.FunctionalTests.Windows/Windows/Tests/WindowsUpdatePlaceholderTests.cs.cs diff --git a/GVFS/GVFS.FunctionalTests.Windows/GVFS.FunctionalTests.Windows.csproj b/GVFS/GVFS.FunctionalTests.Windows/GVFS.FunctionalTests.Windows.csproj index 572078f97..4252e8522 100644 --- a/GVFS/GVFS.FunctionalTests.Windows/GVFS.FunctionalTests.Windows.csproj +++ b/GVFS/GVFS.FunctionalTests.Windows/GVFS.FunctionalTests.Windows.csproj @@ -101,6 +101,7 @@ + diff --git a/GVFS/GVFS.FunctionalTests.Windows/Windows/Tests/WindowsUpdatePlaceholderTests.cs.cs b/GVFS/GVFS.FunctionalTests.Windows/Windows/Tests/WindowsUpdatePlaceholderTests.cs.cs new file mode 100644 index 000000000..be59d5b21 --- /dev/null +++ b/GVFS/GVFS.FunctionalTests.Windows/Windows/Tests/WindowsUpdatePlaceholderTests.cs.cs @@ -0,0 +1,476 @@ +using GVFS.FunctionalTests.FileSystemRunners; +using GVFS.FunctionalTests.Should; +using GVFS.FunctionalTests.Tools; +using GVFS.Tests.Should; +using Microsoft.Win32.SafeHandles; +using NUnit.Framework; +using System; +using System.Collections.Generic; +using System.Diagnostics; +using System.IO; + +namespace GVFS.FunctionalTests.Tests.EnlistmentPerFixture +{ + // WindowsOnly because tests in this class depend on Windows specific file sharing behavior + [TestFixture] + [Category(Categories.WindowsOnly)] + [Category(Categories.GitCommands)] + public class WindowsUpdatePlaceholderTests : TestsWithEnlistmentPerFixture + { + private const string TestParentFolderName = "Test_EPF_UpdatePlaceholderTests"; + private const string OldCommitId = "5d7a7d4db1734fb468a4094469ec58d26301b59d"; + private const string NewFilesAndChangesCommitId = "fec239ea12de1eda6ae5329d4f345784d5b61ff9"; + private FileSystemRunner fileSystem; + + public WindowsUpdatePlaceholderTests() + { + this.fileSystem = new SystemIORunner(); + } + + [SetUp] + public virtual void SetupForTest() + { + // Start each test at NewFilesAndChangesCommitId + this.GitCheckoutCommitId(NewFilesAndChangesCommitId); + this.GitStatusShouldBeClean(NewFilesAndChangesCommitId); + } + + [TestCase, Order(1)] + public void LockToPreventDelete_SingleFile() + { + string testFile1Contents = "TestContentsLockToPreventDelete \r\n"; + string testFile1Name = "test.txt"; + string testFile1Path = this.Enlistment.GetVirtualPathTo(Path.Combine(TestParentFolderName, "LockToPreventDelete", testFile1Name)); + + testFile1Path.ShouldBeAFile(this.fileSystem).WithContents(testFile1Contents); + using (SafeFileHandle testFile1Handle = this.CreateFile(testFile1Path, FileShare.Read)) + { + testFile1Handle.IsInvalid.ShouldEqual(false); + + ProcessResult result = this.InvokeGitAgainstGVFSRepo("checkout " + OldCommitId); + result.Errors.ShouldContain( + "GVFS was unable to delete the following files. To recover, close all handles to the files and run these commands:", + "git clean -f " + TestParentFolderName + "/LockToPreventDelete/" + testFile1Name); + + GitHelpers.CheckGitCommandAgainstGVFSRepo( + this.Enlistment.RepoRoot, + "status -u", + "HEAD detached at " + OldCommitId, + "Untracked files:", + TestParentFolderName + "/LockToPreventDelete/" + testFile1Name); + } + + this.GitCleanFile(TestParentFolderName + "/LockToPreventDelete/" + testFile1Name); + this.GitStatusShouldBeClean(OldCommitId); + GVFSHelpers.ModifiedPathsShouldContain(this.fileSystem, this.Enlistment.DotGVFSRoot, TestParentFolderName + "/LockToPreventDelete/" + testFile1Name); + testFile1Path.ShouldNotExistOnDisk(this.fileSystem); + + this.GitCheckoutCommitId(NewFilesAndChangesCommitId); + + this.GitStatusShouldBeClean(NewFilesAndChangesCommitId); + testFile1Path.ShouldBeAFile(this.fileSystem).WithContents(testFile1Contents); + } + + [TestCase, Order(2)] + public void LockToPreventDelete_MultipleFiles() + { + string testFile2Contents = "TestContentsLockToPreventDelete2 \r\n"; + string testFile3Contents = "TestContentsLockToPreventDelete3 \r\n"; + string testFile4Contents = "TestContentsLockToPreventDelete4 \r\n"; + + string testFile2Name = "test2.txt"; + string testFile3Name = "test3.txt"; + string testFile4Name = "test4.txt"; + + string testFile2Path = this.Enlistment.GetVirtualPathTo(Path.Combine(TestParentFolderName, "LockToPreventDelete", testFile2Name)); + string testFile3Path = this.Enlistment.GetVirtualPathTo(Path.Combine(TestParentFolderName, "LockToPreventDelete", testFile3Name)); + string testFile4Path = this.Enlistment.GetVirtualPathTo(Path.Combine(TestParentFolderName, "LockToPreventDelete", testFile4Name)); + + testFile2Path.ShouldBeAFile(this.fileSystem).WithContents(testFile2Contents); + testFile3Path.ShouldBeAFile(this.fileSystem).WithContents(testFile3Contents); + testFile4Path.ShouldBeAFile(this.fileSystem).WithContents(testFile4Contents); + + using (SafeFileHandle testFile2Handle = this.CreateFile(testFile2Path, FileShare.Read)) + using (SafeFileHandle testFile3Handle = this.CreateFile(testFile3Path, FileShare.Read)) + using (SafeFileHandle testFile4Handle = this.CreateFile(testFile4Path, FileShare.Read)) + { + testFile2Handle.IsInvalid.ShouldEqual(false); + testFile3Handle.IsInvalid.ShouldEqual(false); + testFile4Handle.IsInvalid.ShouldEqual(false); + + ProcessResult result = this.InvokeGitAgainstGVFSRepo("checkout " + OldCommitId); + result.Errors.ShouldContain( + "GVFS was unable to delete the following files. To recover, close all handles to the files and run these commands:", + "git clean -f " + TestParentFolderName + "/LockToPreventDelete/" + testFile2Name, + "git clean -f " + TestParentFolderName + "/LockToPreventDelete/" + testFile3Name, + "git clean -f " + TestParentFolderName + "/LockToPreventDelete/" + testFile4Name); + + GitHelpers.CheckGitCommandAgainstGVFSRepo( + this.Enlistment.RepoRoot, + "status -u", + "HEAD detached at " + OldCommitId, + "Untracked files:", + TestParentFolderName + "/LockToPreventDelete/" + testFile2Name, + TestParentFolderName + "/LockToPreventDelete/" + testFile3Name, + TestParentFolderName + "/LockToPreventDelete/" + testFile4Name); + } + + this.GitCleanFile(TestParentFolderName + "/LockToPreventDelete/" + testFile2Name); + this.GitCleanFile(TestParentFolderName + "/LockToPreventDelete/" + testFile3Name); + this.GitCleanFile(TestParentFolderName + "/LockToPreventDelete/" + testFile4Name); + + this.GitStatusShouldBeClean(OldCommitId); + + GVFSHelpers.ModifiedPathsShouldContain(this.fileSystem, this.Enlistment.DotGVFSRoot, TestParentFolderName + "/LockToPreventDelete/" + testFile2Name); + GVFSHelpers.ModifiedPathsShouldContain(this.fileSystem, this.Enlistment.DotGVFSRoot, TestParentFolderName + "/LockToPreventDelete/" + testFile3Name); + GVFSHelpers.ModifiedPathsShouldContain(this.fileSystem, this.Enlistment.DotGVFSRoot, TestParentFolderName + "/LockToPreventDelete/" + testFile4Name); + + testFile2Path.ShouldNotExistOnDisk(this.fileSystem); + testFile3Path.ShouldNotExistOnDisk(this.fileSystem); + testFile4Path.ShouldNotExistOnDisk(this.fileSystem); + + this.GitCheckoutCommitId(NewFilesAndChangesCommitId); + + this.GitStatusShouldBeClean(NewFilesAndChangesCommitId); + testFile2Path.ShouldBeAFile(this.fileSystem).WithContents(testFile2Contents); + testFile3Path.ShouldBeAFile(this.fileSystem).WithContents(testFile3Contents); + testFile4Path.ShouldBeAFile(this.fileSystem).WithContents(testFile4Contents); + } + + [TestCase, Order(3)] + public void LockToPreventUpdate_SingleFile() + { + string testFile1Contents = "Commit2LockToPreventUpdate \r\n"; + string testFile1OldContents = "TestFileLockToPreventUpdate \r\n"; + string testFile1Name = "test.txt"; + string testFile1Path = this.Enlistment.GetVirtualPathTo(Path.Combine(TestParentFolderName, "LockToPreventUpdate", testFile1Name)); + + testFile1Path.ShouldBeAFile(this.fileSystem).WithContents(testFile1Contents); + using (SafeFileHandle testFile1Handle = this.CreateFile(testFile1Path, FileShare.Read)) + { + testFile1Handle.IsInvalid.ShouldEqual(false); + + ProcessResult result = this.InvokeGitAgainstGVFSRepo("checkout " + OldCommitId); + result.Errors.ShouldContain( + "GVFS was unable to update the following files. To recover, close all handles to the files and run these commands:", + "git checkout -- " + TestParentFolderName + "/LockToPreventUpdate/" + testFile1Name); + + GitHelpers.CheckGitCommandAgainstGVFSRepo( + this.Enlistment.RepoRoot, + "status", + "HEAD detached at " + OldCommitId, + "Changes not staged for commit:", + TestParentFolderName + "/LockToPreventUpdate/" + testFile1Name); + } + + this.GitCheckoutToDiscardChanges(TestParentFolderName + "/LockToPreventUpdate/" + testFile1Name); + this.GitStatusShouldBeClean(OldCommitId); + GVFSHelpers.ModifiedPathsShouldContain(this.fileSystem, this.Enlistment.DotGVFSRoot, TestParentFolderName + "/LockToPreventUpdate/" + testFile1Name); + testFile1Path.ShouldBeAFile(this.fileSystem).WithContents(testFile1OldContents); + + this.GitCheckoutCommitId(NewFilesAndChangesCommitId); + + this.GitStatusShouldBeClean(NewFilesAndChangesCommitId); + testFile1Path.ShouldBeAFile(this.fileSystem).WithContents(testFile1Contents); + } + + [TestCase, Order(4)] + public void LockToPreventUpdate_MultipleFiles() + { + string testFile2Contents = "Commit2LockToPreventUpdate2 \r\n"; + string testFile3Contents = "Commit2LockToPreventUpdate3 \r\n"; + string testFile4Contents = "Commit2LockToPreventUpdate4 \r\n"; + + string testFile2OldContents = "TestFileLockToPreventUpdate2 \r\n"; + string testFile3OldContents = "TestFileLockToPreventUpdate3 \r\n"; + string testFile4OldContents = "TestFileLockToPreventUpdate4 \r\n"; + + string testFile2Name = "test2.txt"; + string testFile3Name = "test3.txt"; + string testFile4Name = "test4.txt"; + + string testFile2Path = this.Enlistment.GetVirtualPathTo(Path.Combine(TestParentFolderName, "LockToPreventUpdate", testFile2Name)); + string testFile3Path = this.Enlistment.GetVirtualPathTo(Path.Combine(TestParentFolderName, "LockToPreventUpdate", testFile3Name)); + string testFile4Path = this.Enlistment.GetVirtualPathTo(Path.Combine(TestParentFolderName, "LockToPreventUpdate", testFile4Name)); + + testFile2Path.ShouldBeAFile(this.fileSystem).WithContents(testFile2Contents); + testFile3Path.ShouldBeAFile(this.fileSystem).WithContents(testFile3Contents); + testFile4Path.ShouldBeAFile(this.fileSystem).WithContents(testFile4Contents); + + using (SafeFileHandle testFile2Handle = this.CreateFile(testFile2Path, FileShare.Read)) + using (SafeFileHandle testFile3Handle = this.CreateFile(testFile3Path, FileShare.Read)) + using (SafeFileHandle testFile4Handle = this.CreateFile(testFile4Path, FileShare.Read)) + { + testFile2Handle.IsInvalid.ShouldEqual(false); + testFile3Handle.IsInvalid.ShouldEqual(false); + testFile4Handle.IsInvalid.ShouldEqual(false); + + ProcessResult result = this.InvokeGitAgainstGVFSRepo("checkout " + OldCommitId); + result.Errors.ShouldContain( + "GVFS was unable to update the following files. To recover, close all handles to the files and run these commands:", + "git checkout -- " + TestParentFolderName + "/LockToPreventUpdate/" + testFile2Name, + "git checkout -- " + TestParentFolderName + "/LockToPreventUpdate/" + testFile3Name, + "git checkout -- " + TestParentFolderName + "/LockToPreventUpdate/" + testFile4Name); + + GitHelpers.CheckGitCommandAgainstGVFSRepo( + this.Enlistment.RepoRoot, + "status", + "HEAD detached at " + OldCommitId, + "Changes not staged for commit:", + TestParentFolderName + "/LockToPreventUpdate/" + testFile2Name, + TestParentFolderName + "/LockToPreventUpdate/" + testFile3Name, + TestParentFolderName + "/LockToPreventUpdate/" + testFile4Name); + } + + this.GitCheckoutToDiscardChanges(TestParentFolderName + "/LockToPreventUpdate/" + testFile2Name); + this.GitCheckoutToDiscardChanges(TestParentFolderName + "/LockToPreventUpdate/" + testFile3Name); + this.GitCheckoutToDiscardChanges(TestParentFolderName + "/LockToPreventUpdate/" + testFile4Name); + + this.GitStatusShouldBeClean(OldCommitId); + GVFSHelpers.ModifiedPathsShouldContain(this.fileSystem, this.Enlistment.DotGVFSRoot, TestParentFolderName + "/LockToPreventUpdate/" + testFile2Name); + GVFSHelpers.ModifiedPathsShouldContain(this.fileSystem, this.Enlistment.DotGVFSRoot, TestParentFolderName + "/LockToPreventUpdate/" + testFile3Name); + GVFSHelpers.ModifiedPathsShouldContain(this.fileSystem, this.Enlistment.DotGVFSRoot, TestParentFolderName + "/LockToPreventUpdate/" + testFile4Name); + testFile2Path.ShouldBeAFile(this.fileSystem).WithContents(testFile2OldContents); + testFile3Path.ShouldBeAFile(this.fileSystem).WithContents(testFile3OldContents); + testFile4Path.ShouldBeAFile(this.fileSystem).WithContents(testFile4OldContents); + + this.GitCheckoutCommitId(NewFilesAndChangesCommitId); + + this.GitStatusShouldBeClean(NewFilesAndChangesCommitId); + testFile2Path.ShouldBeAFile(this.fileSystem).WithContents(testFile2Contents); + testFile3Path.ShouldBeAFile(this.fileSystem).WithContents(testFile3Contents); + testFile4Path.ShouldBeAFile(this.fileSystem).WithContents(testFile4Contents); + } + + [TestCase, Order(5)] + public void LockToPreventUpdateAndDelete() + { + string testFileUpdate1Contents = "Commit2LockToPreventUpdateAndDelete \r\n"; + string testFileUpdate2Contents = "Commit2LockToPreventUpdateAndDelete2 \r\n"; + string testFileUpdate3Contents = "Commit2LockToPreventUpdateAndDelete3 \r\n"; + string testFileDelete1Contents = "PreventDelete \r\n"; + string testFileDelete2Contents = "PreventDelete2 \r\n"; + string testFileDelete3Contents = "PreventDelete3 \r\n"; + + string testFileUpdate1OldContents = "TestFileLockToPreventUpdateAndDelete \r\n"; + string testFileUpdate2OldContents = "TestFileLockToPreventUpdateAndDelete2 \r\n"; + string testFileUpdate3OldContents = "TestFileLockToPreventUpdateAndDelete3 \r\n"; + + string testFileUpdate1Name = "test.txt"; + string testFileUpdate2Name = "test2.txt"; + string testFileUpdate3Name = "test3.txt"; + string testFileDelete1Name = "test_delete.txt"; + string testFileDelete2Name = "test_delete2.txt"; + string testFileDelete3Name = "test_delete3.txt"; + + string testFileUpdate1Path = this.Enlistment.GetVirtualPathTo(Path.Combine(TestParentFolderName, "LockToPreventUpdateAndDelete", testFileUpdate1Name)); + string testFileUpdate2Path = this.Enlistment.GetVirtualPathTo(Path.Combine(TestParentFolderName, "LockToPreventUpdateAndDelete", testFileUpdate2Name)); + string testFileUpdate3Path = this.Enlistment.GetVirtualPathTo(Path.Combine(TestParentFolderName, "LockToPreventUpdateAndDelete", testFileUpdate3Name)); + string testFileDelete1Path = this.Enlistment.GetVirtualPathTo(Path.Combine(TestParentFolderName, "LockToPreventUpdateAndDelete", testFileDelete1Name)); + string testFileDelete2Path = this.Enlistment.GetVirtualPathTo(Path.Combine(TestParentFolderName, "LockToPreventUpdateAndDelete", testFileDelete2Name)); + string testFileDelete3Path = this.Enlistment.GetVirtualPathTo(Path.Combine(TestParentFolderName, "LockToPreventUpdateAndDelete", testFileDelete3Name)); + + testFileUpdate1Path.ShouldBeAFile(this.fileSystem).WithContents(testFileUpdate1Contents); + testFileUpdate2Path.ShouldBeAFile(this.fileSystem).WithContents(testFileUpdate2Contents); + testFileUpdate3Path.ShouldBeAFile(this.fileSystem).WithContents(testFileUpdate3Contents); + testFileDelete1Path.ShouldBeAFile(this.fileSystem).WithContents(testFileDelete1Contents); + testFileDelete2Path.ShouldBeAFile(this.fileSystem).WithContents(testFileDelete2Contents); + testFileDelete3Path.ShouldBeAFile(this.fileSystem).WithContents(testFileDelete3Contents); + + using (SafeFileHandle testFileUpdate1Handle = this.CreateFile(testFileUpdate1Path, FileShare.Read)) + using (SafeFileHandle testFileUpdate2Handle = this.CreateFile(testFileUpdate2Path, FileShare.Read)) + using (SafeFileHandle testFileUpdate3Handle = this.CreateFile(testFileUpdate3Path, FileShare.Read)) + using (SafeFileHandle testFileDelete1Handle = this.CreateFile(testFileDelete1Path, FileShare.Read)) + using (SafeFileHandle testFileDelete2Handle = this.CreateFile(testFileDelete2Path, FileShare.Read)) + using (SafeFileHandle testFileDelete3Handle = this.CreateFile(testFileDelete3Path, FileShare.Read)) + { + testFileUpdate1Handle.IsInvalid.ShouldEqual(false); + testFileUpdate2Handle.IsInvalid.ShouldEqual(false); + testFileUpdate3Handle.IsInvalid.ShouldEqual(false); + testFileDelete1Handle.IsInvalid.ShouldEqual(false); + testFileDelete2Handle.IsInvalid.ShouldEqual(false); + testFileDelete3Handle.IsInvalid.ShouldEqual(false); + + ProcessResult checkoutResult = GitProcess.InvokeProcess(this.Enlistment.RepoRoot, "checkout " + OldCommitId); + checkoutResult.Errors.ShouldContain( + "HEAD is now at " + OldCommitId, + "GVFS was unable to delete the following files. To recover, close all handles to the files and run these commands:", + "git clean -f " + TestParentFolderName + "/LockToPreventUpdateAndDelete/" + testFileDelete1Name, + "git clean -f " + TestParentFolderName + "/LockToPreventUpdateAndDelete/" + testFileDelete2Name, + "git clean -f " + TestParentFolderName + "/LockToPreventUpdateAndDelete/" + testFileDelete3Name, + "GVFS was unable to update the following files. To recover, close all handles to the files and run these commands:", + "git checkout -- " + TestParentFolderName + "/LockToPreventUpdateAndDelete/" + testFileUpdate1Name, + "git checkout -- " + TestParentFolderName + "/LockToPreventUpdateAndDelete/" + testFileUpdate2Name, + "git checkout -- " + TestParentFolderName + "/LockToPreventUpdateAndDelete/" + testFileUpdate3Name); + + GitHelpers.CheckGitCommandAgainstGVFSRepo( + this.Enlistment.RepoRoot, + "status", + "HEAD detached at " + OldCommitId, + "modified: Test_EPF_UpdatePlaceholderTests/LockToPreventUpdateAndDelete/test.txt", + "modified: Test_EPF_UpdatePlaceholderTests/LockToPreventUpdateAndDelete/test2.txt", + "modified: Test_EPF_UpdatePlaceholderTests/LockToPreventUpdateAndDelete/test3.txt", + "Untracked files:\n (use \"git add ...\" to include in what will be committed)\n\n\tTest_EPF_UpdatePlaceholderTests/LockToPreventUpdateAndDelete/test_delete.txt\n\tTest_EPF_UpdatePlaceholderTests/LockToPreventUpdateAndDelete/test_delete2.txt\n\tTest_EPF_UpdatePlaceholderTests/LockToPreventUpdateAndDelete/test_delete3.txt", + "no changes added to commit (use \"git add\" and/or \"git commit -a\")\n"); + } + + this.GitCheckoutToDiscardChanges(TestParentFolderName + "/LockToPreventUpdateAndDelete/" + testFileUpdate1Name); + this.GitCheckoutToDiscardChanges(TestParentFolderName + "/LockToPreventUpdateAndDelete/" + testFileUpdate2Name); + this.GitCheckoutToDiscardChanges(TestParentFolderName + "/LockToPreventUpdateAndDelete/" + testFileUpdate3Name); + this.GitCleanFile(TestParentFolderName + "/LockToPreventUpdateAndDelete/" + testFileDelete1Name); + this.GitCleanFile(TestParentFolderName + "/LockToPreventUpdateAndDelete/" + testFileDelete2Name); + this.GitCleanFile(TestParentFolderName + "/LockToPreventUpdateAndDelete/" + testFileDelete3Name); + + this.GitStatusShouldBeClean(OldCommitId); + + GVFSHelpers.ModifiedPathsShouldContain(this.fileSystem, this.Enlistment.DotGVFSRoot, TestParentFolderName + "/LockToPreventUpdateAndDelete/" + testFileUpdate1Name); + GVFSHelpers.ModifiedPathsShouldContain(this.fileSystem, this.Enlistment.DotGVFSRoot, TestParentFolderName + "/LockToPreventUpdateAndDelete/" + testFileUpdate2Name); + GVFSHelpers.ModifiedPathsShouldContain(this.fileSystem, this.Enlistment.DotGVFSRoot, TestParentFolderName + "/LockToPreventUpdateAndDelete/" + testFileUpdate3Name); + GVFSHelpers.ModifiedPathsShouldContain(this.fileSystem, this.Enlistment.DotGVFSRoot, TestParentFolderName + "/LockToPreventUpdateAndDelete/" + testFileDelete1Name); + GVFSHelpers.ModifiedPathsShouldContain(this.fileSystem, this.Enlistment.DotGVFSRoot, TestParentFolderName + "/LockToPreventUpdateAndDelete/" + testFileDelete2Name); + GVFSHelpers.ModifiedPathsShouldContain(this.fileSystem, this.Enlistment.DotGVFSRoot, TestParentFolderName + "/LockToPreventUpdateAndDelete/" + testFileDelete3Name); + + testFileUpdate1Path.ShouldBeAFile(this.fileSystem).WithContents(testFileUpdate1OldContents); + testFileUpdate2Path.ShouldBeAFile(this.fileSystem).WithContents(testFileUpdate2OldContents); + testFileUpdate3Path.ShouldBeAFile(this.fileSystem).WithContents(testFileUpdate3OldContents); + testFileDelete1Path.ShouldNotExistOnDisk(this.fileSystem); + testFileDelete2Path.ShouldNotExistOnDisk(this.fileSystem); + testFileDelete3Path.ShouldNotExistOnDisk(this.fileSystem); + + this.GitCheckoutCommitId(NewFilesAndChangesCommitId); + + this.GitStatusShouldBeClean(NewFilesAndChangesCommitId); + testFileUpdate1Path.ShouldBeAFile(this.fileSystem).WithContents(testFileUpdate1Contents); + testFileUpdate2Path.ShouldBeAFile(this.fileSystem).WithContents(testFileUpdate2Contents); + testFileUpdate3Path.ShouldBeAFile(this.fileSystem).WithContents(testFileUpdate3Contents); + testFileDelete1Path.ShouldBeAFile(this.fileSystem).WithContents(testFileDelete1Contents); + testFileDelete2Path.ShouldBeAFile(this.fileSystem).WithContents(testFileDelete2Contents); + testFileDelete3Path.ShouldBeAFile(this.fileSystem).WithContents(testFileDelete3Contents); + } + + [TestCase, Order(6)] + public void LockMoreThanMaxReportedFileNames() + { + string updateFilesFolder = "FilesToUpdate"; + string deleteFilesFolder = "FilesToDelete"; + + for (int i = 1; i <= 51; ++i) + { + this.Enlistment.GetVirtualPathTo(Path.Combine(TestParentFolderName, "MaxFileListCount", updateFilesFolder, i.ToString() + ".txt")).ShouldBeAFile(this.fileSystem); + this.Enlistment.GetVirtualPathTo(Path.Combine(TestParentFolderName, "MaxFileListCount", deleteFilesFolder, i.ToString() + ".txt")).ShouldBeAFile(this.fileSystem); + } + + List openHandles = new List(); + try + { + for (int i = 1; i <= 51; ++i) + { + SafeFileHandle handle = this.CreateFile( + this.Enlistment.GetVirtualPathTo(Path.Combine(TestParentFolderName, "MaxFileListCount", updateFilesFolder, i.ToString() + ".txt")), + FileShare.Read); + openHandles.Add(handle); + handle.IsInvalid.ShouldEqual(false); + + handle = this.CreateFile( + this.Enlistment.GetVirtualPathTo(Path.Combine(TestParentFolderName, "MaxFileListCount", deleteFilesFolder, i.ToString() + ".txt")), + FileShare.Read); + openHandles.Add(handle); + handle.IsInvalid.ShouldEqual(false); + } + + ProcessResult result = this.InvokeGitAgainstGVFSRepo("checkout " + OldCommitId); + result.Errors.ShouldContain( + "GVFS failed to update 102 files, run 'git status' to check the status of files in the repo"); + + List expectedOutputStrings = new List() + { + "HEAD detached at " + OldCommitId, + "no changes added to commit (use \"git add\" and/or \"git commit -a\")\n" + }; + + for (int expectedFilePrefix = 1; expectedFilePrefix <= 51; ++expectedFilePrefix) + { + expectedOutputStrings.Add("modified: Test_EPF_UpdatePlaceholderTests/MaxFileListCount/" + updateFilesFolder + "/" + expectedFilePrefix.ToString() + ".txt"); + expectedOutputStrings.Add("Test_EPF_UpdatePlaceholderTests/MaxFileListCount/" + deleteFilesFolder + "/" + expectedFilePrefix.ToString() + ".txt"); + } + + GitHelpers.CheckGitCommandAgainstGVFSRepo(this.Enlistment.RepoRoot, "status -u", expectedOutputStrings.ToArray()); + } + finally + { + foreach (SafeFileHandle handle in openHandles) + { + handle.Dispose(); + } + } + + for (int i = 1; i <= 51; ++i) + { + this.GitCheckoutToDiscardChanges(TestParentFolderName + "/MaxFileListCount/" + updateFilesFolder + "/" + i.ToString() + ".txt"); + this.GitCleanFile(TestParentFolderName + "/MaxFileListCount/" + deleteFilesFolder + "/" + i.ToString() + ".txt"); + } + + this.GitStatusShouldBeClean(OldCommitId); + this.GitCheckoutCommitId(NewFilesAndChangesCommitId); + this.GitStatusShouldBeClean(NewFilesAndChangesCommitId); + } + + private ProcessResult InvokeGitAgainstGVFSRepo(string command) + { + return GitHelpers.InvokeGitAgainstGVFSRepo(this.Enlistment.RepoRoot, command); + } + + private void GitStatusShouldBeClean(string commitId) + { + GitHelpers.CheckGitCommandAgainstGVFSRepo( + this.Enlistment.RepoRoot, + "status", + "HEAD detached at " + commitId, + "nothing to commit, working tree clean"); + } + + private void GitCleanFile(string gitPath) + { + GitHelpers.CheckGitCommandAgainstGVFSRepo( + this.Enlistment.RepoRoot, + "clean -f " + gitPath, + "Removing " + gitPath); + } + + private void GitCheckoutToDiscardChanges(string gitPath) + { + GitHelpers.CheckGitCommandAgainstGVFSRepo(this.Enlistment.RepoRoot, "checkout -- " + gitPath); + } + + private void GitCheckoutCommitId(string commitId) + { + this.InvokeGitAgainstGVFSRepo("checkout " + commitId).Errors.ShouldContain("HEAD is now at " + commitId); + } + + private SafeFileHandle CreateFile(string path, FileShare shareMode) + { + return NativeMethods.CreateFile( + path, + (uint)FileAccess.Read, + shareMode, + IntPtr.Zero, + FileMode.Open, + (uint)FileAttributes.Normal, + IntPtr.Zero); + } + + private bool CanUpdateAndDeletePlaceholdersWithOpenHandles() + { + // https://msdn.microsoft.com/en-us/library/windows/desktop/ms724429(v=vs.85).aspx + FileVersionInfo kernel32Info = FileVersionInfo.GetVersionInfo(Path.Combine(Environment.SystemDirectory, "kernel32.dll")); + + // 16248 is first build with support - see 12658248 for details + if (kernel32Info.FileBuildPart >= 16248) + { + return true; + } + + return false; + } + } +} \ No newline at end of file diff --git a/GVFS/GVFS.FunctionalTests/Tests/EnlistmentPerFixture/UpdatePlaceholderTests.cs b/GVFS/GVFS.FunctionalTests/Tests/EnlistmentPerFixture/UpdatePlaceholderTests.cs index cb938becb..e451911d6 100644 --- a/GVFS/GVFS.FunctionalTests/Tests/EnlistmentPerFixture/UpdatePlaceholderTests.cs +++ b/GVFS/GVFS.FunctionalTests/Tests/EnlistmentPerFixture/UpdatePlaceholderTests.cs @@ -2,10 +2,8 @@ using GVFS.FunctionalTests.Should; using GVFS.FunctionalTests.Tools; using GVFS.Tests.Should; -using Microsoft.Win32.SafeHandles; using NUnit.Framework; using System; -using System.Collections.Generic; using System.Diagnostics; using System.IO; using System.Runtime.InteropServices; @@ -32,313 +30,9 @@ public virtual void SetupForTest() // Start each test at NewFilesAndChangesCommitId this.GitCheckoutCommitId(NewFilesAndChangesCommitId); this.GitStatusShouldBeClean(NewFilesAndChangesCommitId); - } + } - // WindowsOnly because test requires Windows specific file sharing behavior [TestCase, Order(1)] - [Category(Categories.WindowsOnly)] - public void LockToPreventDelete_SingleFile() - { - string testFile1Contents = "TestContentsLockToPreventDelete \r\n"; - string testFile1Name = "test.txt"; - string testFile1Path = this.Enlistment.GetVirtualPathTo(Path.Combine(TestParentFolderName, "LockToPreventDelete", testFile1Name)); - - testFile1Path.ShouldBeAFile(this.fileSystem).WithContents(testFile1Contents); - using (FileStream testFile1 = File.Open(testFile1Path, FileMode.Open, FileAccess.Read, FileShare.Read)) - { - ProcessResult result = this.InvokeGitAgainstGVFSRepo("checkout " + OldCommitId); - result.Errors.ShouldContain( - "GVFS was unable to delete the following files. To recover, close all handles to the files and run these commands:", - "git clean -f " + TestParentFolderName + "/LockToPreventDelete/" + testFile1Name); - - GitHelpers.CheckGitCommandAgainstGVFSRepo( - this.Enlistment.RepoRoot, - "status -u", - "HEAD detached at " + OldCommitId, - "Untracked files:", - TestParentFolderName + "/LockToPreventDelete/" + testFile1Name); - } - - this.GitCleanFile(TestParentFolderName + "/LockToPreventDelete/" + testFile1Name); - this.GitStatusShouldBeClean(OldCommitId); - GVFSHelpers.ModifiedPathsShouldContain(this.fileSystem, this.Enlistment.DotGVFSRoot, TestParentFolderName + "/LockToPreventDelete/" + testFile1Name); - testFile1Path.ShouldNotExistOnDisk(this.fileSystem); - - this.GitCheckoutCommitId(NewFilesAndChangesCommitId); - - this.GitStatusShouldBeClean(NewFilesAndChangesCommitId); - testFile1Path.ShouldBeAFile(this.fileSystem).WithContents(testFile1Contents); - } - - // WindowsOnly because test requires Windows specific file sharing behavior - [TestCase, Order(2)] - [Category(Categories.WindowsOnly)] - public void LockToPreventDelete_MultipleFiles() - { - string testFile2Contents = "TestContentsLockToPreventDelete2 \r\n"; - string testFile3Contents = "TestContentsLockToPreventDelete3 \r\n"; - string testFile4Contents = "TestContentsLockToPreventDelete4 \r\n"; - - string testFile2Name = "test2.txt"; - string testFile3Name = "test3.txt"; - string testFile4Name = "test4.txt"; - - string testFile2Path = this.Enlistment.GetVirtualPathTo(Path.Combine(TestParentFolderName, "LockToPreventDelete", testFile2Name)); - string testFile3Path = this.Enlistment.GetVirtualPathTo(Path.Combine(TestParentFolderName, "LockToPreventDelete", testFile3Name)); - string testFile4Path = this.Enlistment.GetVirtualPathTo(Path.Combine(TestParentFolderName, "LockToPreventDelete", testFile4Name)); - - testFile2Path.ShouldBeAFile(this.fileSystem).WithContents(testFile2Contents); - testFile3Path.ShouldBeAFile(this.fileSystem).WithContents(testFile3Contents); - testFile4Path.ShouldBeAFile(this.fileSystem).WithContents(testFile4Contents); - - using (FileStream testFile2 = File.Open(testFile2Path, FileMode.Open, FileAccess.Read, FileShare.Read)) - using (FileStream testFile3 = File.Open(testFile3Path, FileMode.Open, FileAccess.Read, FileShare.Read)) - using (FileStream testFile4 = File.Open(testFile4Path, FileMode.Open, FileAccess.Read, FileShare.Read)) - { - ProcessResult result = this.InvokeGitAgainstGVFSRepo("checkout " + OldCommitId); - result.Errors.ShouldContain( - "GVFS was unable to delete the following files. To recover, close all handles to the files and run these commands:", - "git clean -f " + TestParentFolderName + "/LockToPreventDelete/" + testFile2Name, - "git clean -f " + TestParentFolderName + "/LockToPreventDelete/" + testFile3Name, - "git clean -f " + TestParentFolderName + "/LockToPreventDelete/" + testFile4Name); - - GitHelpers.CheckGitCommandAgainstGVFSRepo( - this.Enlistment.RepoRoot, - "status -u", - "HEAD detached at " + OldCommitId, - "Untracked files:", - TestParentFolderName + "/LockToPreventDelete/" + testFile2Name, - TestParentFolderName + "/LockToPreventDelete/" + testFile3Name, - TestParentFolderName + "/LockToPreventDelete/" + testFile4Name); - } - - this.GitCleanFile(TestParentFolderName + "/LockToPreventDelete/" + testFile2Name); - this.GitCleanFile(TestParentFolderName + "/LockToPreventDelete/" + testFile3Name); - this.GitCleanFile(TestParentFolderName + "/LockToPreventDelete/" + testFile4Name); - - this.GitStatusShouldBeClean(OldCommitId); - - GVFSHelpers.ModifiedPathsShouldContain(this.fileSystem, this.Enlistment.DotGVFSRoot, TestParentFolderName + "/LockToPreventDelete/" + testFile2Name); - GVFSHelpers.ModifiedPathsShouldContain(this.fileSystem, this.Enlistment.DotGVFSRoot, TestParentFolderName + "/LockToPreventDelete/" + testFile3Name); - GVFSHelpers.ModifiedPathsShouldContain(this.fileSystem, this.Enlistment.DotGVFSRoot, TestParentFolderName + "/LockToPreventDelete/" + testFile4Name); - - testFile2Path.ShouldNotExistOnDisk(this.fileSystem); - testFile3Path.ShouldNotExistOnDisk(this.fileSystem); - testFile4Path.ShouldNotExistOnDisk(this.fileSystem); - - this.GitCheckoutCommitId(NewFilesAndChangesCommitId); - - this.GitStatusShouldBeClean(NewFilesAndChangesCommitId); - testFile2Path.ShouldBeAFile(this.fileSystem).WithContents(testFile2Contents); - testFile3Path.ShouldBeAFile(this.fileSystem).WithContents(testFile3Contents); - testFile4Path.ShouldBeAFile(this.fileSystem).WithContents(testFile4Contents); - } - - // WindowsOnly because test requires Windows specific file sharing behavior - [TestCase, Order(3)] - [Category(Categories.WindowsOnly)] - public void LockToPreventUpdate_SingleFile() - { - string testFile1Contents = "Commit2LockToPreventUpdate \r\n"; - string testFile1OldContents = "TestFileLockToPreventUpdate \r\n"; - string testFile1Name = "test.txt"; - string testFile1Path = this.Enlistment.GetVirtualPathTo(Path.Combine(TestParentFolderName, "LockToPreventUpdate", testFile1Name)); - - testFile1Path.ShouldBeAFile(this.fileSystem).WithContents(testFile1Contents); - using (FileStream testFile1 = File.Open(testFile1Path, FileMode.Open, FileAccess.Read, FileShare.Read)) - { - ProcessResult result = this.InvokeGitAgainstGVFSRepo("checkout " + OldCommitId); - result.Errors.ShouldContain( - "GVFS was unable to update the following files. To recover, close all handles to the files and run these commands:", - "git checkout -- " + TestParentFolderName + "/LockToPreventUpdate/" + testFile1Name); - - GitHelpers.CheckGitCommandAgainstGVFSRepo( - this.Enlistment.RepoRoot, - "status", - "HEAD detached at " + OldCommitId, - "Changes not staged for commit:", - TestParentFolderName + "/LockToPreventUpdate/" + testFile1Name); - } - - this.GitCheckoutToDiscardChanges(TestParentFolderName + "/LockToPreventUpdate/" + testFile1Name); - this.GitStatusShouldBeClean(OldCommitId); - GVFSHelpers.ModifiedPathsShouldContain(this.fileSystem, this.Enlistment.DotGVFSRoot, TestParentFolderName + "/LockToPreventUpdate/" + testFile1Name); - testFile1Path.ShouldBeAFile(this.fileSystem).WithContents(testFile1OldContents); - - this.GitCheckoutCommitId(NewFilesAndChangesCommitId); - - this.GitStatusShouldBeClean(NewFilesAndChangesCommitId); - testFile1Path.ShouldBeAFile(this.fileSystem).WithContents(testFile1Contents); - } - - // WindowsOnly because test requires Windows specific file sharing behavior - [TestCase, Order(4)] - [Category(Categories.WindowsOnly)] - public void LockToPreventUpdate_MultipleFiles() - { - string testFile2Contents = "Commit2LockToPreventUpdate2 \r\n"; - string testFile3Contents = "Commit2LockToPreventUpdate3 \r\n"; - string testFile4Contents = "Commit2LockToPreventUpdate4 \r\n"; - - string testFile2OldContents = "TestFileLockToPreventUpdate2 \r\n"; - string testFile3OldContents = "TestFileLockToPreventUpdate3 \r\n"; - string testFile4OldContents = "TestFileLockToPreventUpdate4 \r\n"; - - string testFile2Name = "test2.txt"; - string testFile3Name = "test3.txt"; - string testFile4Name = "test4.txt"; - - string testFile2Path = this.Enlistment.GetVirtualPathTo(Path.Combine(TestParentFolderName, "LockToPreventUpdate", testFile2Name)); - string testFile3Path = this.Enlistment.GetVirtualPathTo(Path.Combine(TestParentFolderName, "LockToPreventUpdate", testFile3Name)); - string testFile4Path = this.Enlistment.GetVirtualPathTo(Path.Combine(TestParentFolderName, "LockToPreventUpdate", testFile4Name)); - - testFile2Path.ShouldBeAFile(this.fileSystem).WithContents(testFile2Contents); - testFile3Path.ShouldBeAFile(this.fileSystem).WithContents(testFile3Contents); - testFile4Path.ShouldBeAFile(this.fileSystem).WithContents(testFile4Contents); - - using (FileStream testFile2 = File.Open(testFile2Path, FileMode.Open, FileAccess.Read, FileShare.Read)) - using (FileStream testFile3 = File.Open(testFile3Path, FileMode.Open, FileAccess.Read, FileShare.Read)) - using (FileStream testFile4 = File.Open(testFile4Path, FileMode.Open, FileAccess.Read, FileShare.Read)) - { - ProcessResult result = this.InvokeGitAgainstGVFSRepo("checkout " + OldCommitId); - result.Errors.ShouldContain( - "GVFS was unable to update the following files. To recover, close all handles to the files and run these commands:", - "git checkout -- " + TestParentFolderName + "/LockToPreventUpdate/" + testFile2Name, - "git checkout -- " + TestParentFolderName + "/LockToPreventUpdate/" + testFile3Name, - "git checkout -- " + TestParentFolderName + "/LockToPreventUpdate/" + testFile4Name); - - GitHelpers.CheckGitCommandAgainstGVFSRepo( - this.Enlistment.RepoRoot, - "status", - "HEAD detached at " + OldCommitId, - "Changes not staged for commit:", - TestParentFolderName + "/LockToPreventUpdate/" + testFile2Name, - TestParentFolderName + "/LockToPreventUpdate/" + testFile3Name, - TestParentFolderName + "/LockToPreventUpdate/" + testFile4Name); - } - - this.GitCheckoutToDiscardChanges(TestParentFolderName + "/LockToPreventUpdate/" + testFile2Name); - this.GitCheckoutToDiscardChanges(TestParentFolderName + "/LockToPreventUpdate/" + testFile3Name); - this.GitCheckoutToDiscardChanges(TestParentFolderName + "/LockToPreventUpdate/" + testFile4Name); - - this.GitStatusShouldBeClean(OldCommitId); - GVFSHelpers.ModifiedPathsShouldContain(this.fileSystem, this.Enlistment.DotGVFSRoot, TestParentFolderName + "/LockToPreventUpdate/" + testFile2Name); - GVFSHelpers.ModifiedPathsShouldContain(this.fileSystem, this.Enlistment.DotGVFSRoot, TestParentFolderName + "/LockToPreventUpdate/" + testFile3Name); - GVFSHelpers.ModifiedPathsShouldContain(this.fileSystem, this.Enlistment.DotGVFSRoot, TestParentFolderName + "/LockToPreventUpdate/" + testFile4Name); - testFile2Path.ShouldBeAFile(this.fileSystem).WithContents(testFile2OldContents); - testFile3Path.ShouldBeAFile(this.fileSystem).WithContents(testFile3OldContents); - testFile4Path.ShouldBeAFile(this.fileSystem).WithContents(testFile4OldContents); - - this.GitCheckoutCommitId(NewFilesAndChangesCommitId); - - this.GitStatusShouldBeClean(NewFilesAndChangesCommitId); - testFile2Path.ShouldBeAFile(this.fileSystem).WithContents(testFile2Contents); - testFile3Path.ShouldBeAFile(this.fileSystem).WithContents(testFile3Contents); - testFile4Path.ShouldBeAFile(this.fileSystem).WithContents(testFile4Contents); - } - - // WindowsOnly because test requires Windows specific file sharing behavior - [TestCase, Order(5)] - [Category(Categories.WindowsOnly)] - public void LockToPreventUpdateAndDelete() - { - string testFileUpdate1Contents = "Commit2LockToPreventUpdateAndDelete \r\n"; - string testFileUpdate2Contents = "Commit2LockToPreventUpdateAndDelete2 \r\n"; - string testFileUpdate3Contents = "Commit2LockToPreventUpdateAndDelete3 \r\n"; - string testFileDelete1Contents = "PreventDelete \r\n"; - string testFileDelete2Contents = "PreventDelete2 \r\n"; - string testFileDelete3Contents = "PreventDelete3 \r\n"; - - string testFileUpdate1OldContents = "TestFileLockToPreventUpdateAndDelete \r\n"; - string testFileUpdate2OldContents = "TestFileLockToPreventUpdateAndDelete2 \r\n"; - string testFileUpdate3OldContents = "TestFileLockToPreventUpdateAndDelete3 \r\n"; - - string testFileUpdate1Name = "test.txt"; - string testFileUpdate2Name = "test2.txt"; - string testFileUpdate3Name = "test3.txt"; - string testFileDelete1Name = "test_delete.txt"; - string testFileDelete2Name = "test_delete2.txt"; - string testFileDelete3Name = "test_delete3.txt"; - - string testFileUpdate1Path = this.Enlistment.GetVirtualPathTo(Path.Combine(TestParentFolderName, "LockToPreventUpdateAndDelete", testFileUpdate1Name)); - string testFileUpdate2Path = this.Enlistment.GetVirtualPathTo(Path.Combine(TestParentFolderName, "LockToPreventUpdateAndDelete", testFileUpdate2Name)); - string testFileUpdate3Path = this.Enlistment.GetVirtualPathTo(Path.Combine(TestParentFolderName, "LockToPreventUpdateAndDelete", testFileUpdate3Name)); - string testFileDelete1Path = this.Enlistment.GetVirtualPathTo(Path.Combine(TestParentFolderName, "LockToPreventUpdateAndDelete", testFileDelete1Name)); - string testFileDelete2Path = this.Enlistment.GetVirtualPathTo(Path.Combine(TestParentFolderName, "LockToPreventUpdateAndDelete", testFileDelete2Name)); - string testFileDelete3Path = this.Enlistment.GetVirtualPathTo(Path.Combine(TestParentFolderName, "LockToPreventUpdateAndDelete", testFileDelete3Name)); - - testFileUpdate1Path.ShouldBeAFile(this.fileSystem).WithContents(testFileUpdate1Contents); - testFileUpdate2Path.ShouldBeAFile(this.fileSystem).WithContents(testFileUpdate2Contents); - testFileUpdate3Path.ShouldBeAFile(this.fileSystem).WithContents(testFileUpdate3Contents); - testFileDelete1Path.ShouldBeAFile(this.fileSystem).WithContents(testFileDelete1Contents); - testFileDelete2Path.ShouldBeAFile(this.fileSystem).WithContents(testFileDelete2Contents); - testFileDelete3Path.ShouldBeAFile(this.fileSystem).WithContents(testFileDelete3Contents); - - using (FileStream testFileUpdate1 = File.Open(testFileUpdate1Path, FileMode.Open, FileAccess.Read, FileShare.Read)) - using (FileStream testFileUpdate2 = File.Open(testFileUpdate2Path, FileMode.Open, FileAccess.Read, FileShare.Read)) - using (FileStream testFileUpdate3 = File.Open(testFileUpdate3Path, FileMode.Open, FileAccess.Read, FileShare.Read)) - using (FileStream testFileDelete1 = File.Open(testFileDelete1Path, FileMode.Open, FileAccess.Read, FileShare.Read)) - using (FileStream testFileDelete2 = File.Open(testFileDelete2Path, FileMode.Open, FileAccess.Read, FileShare.Read)) - using (FileStream testFileDelete3 = File.Open(testFileDelete3Path, FileMode.Open, FileAccess.Read, FileShare.Read)) - { - ProcessResult checkoutResult = GitProcess.InvokeProcess(this.Enlistment.RepoRoot, "checkout " + OldCommitId); - checkoutResult.Errors.ShouldContain( - "HEAD is now at " + OldCommitId, - "GVFS was unable to delete the following files. To recover, close all handles to the files and run these commands:", - "git clean -f " + TestParentFolderName + "/LockToPreventUpdateAndDelete/" + testFileDelete1Name, - "git clean -f " + TestParentFolderName + "/LockToPreventUpdateAndDelete/" + testFileDelete2Name, - "git clean -f " + TestParentFolderName + "/LockToPreventUpdateAndDelete/" + testFileDelete3Name, - "GVFS was unable to update the following files. To recover, close all handles to the files and run these commands:", - "git checkout -- " + TestParentFolderName + "/LockToPreventUpdateAndDelete/" + testFileUpdate1Name, - "git checkout -- " + TestParentFolderName + "/LockToPreventUpdateAndDelete/" + testFileUpdate2Name, - "git checkout -- " + TestParentFolderName + "/LockToPreventUpdateAndDelete/" + testFileUpdate3Name); - - GitHelpers.CheckGitCommandAgainstGVFSRepo( - this.Enlistment.RepoRoot, - "status", - "HEAD detached at " + OldCommitId, - "modified: Test_EPF_UpdatePlaceholderTests/LockToPreventUpdateAndDelete/test.txt", - "modified: Test_EPF_UpdatePlaceholderTests/LockToPreventUpdateAndDelete/test2.txt", - "modified: Test_EPF_UpdatePlaceholderTests/LockToPreventUpdateAndDelete/test3.txt", - "Untracked files:\n (use \"git add ...\" to include in what will be committed)\n\n\tTest_EPF_UpdatePlaceholderTests/LockToPreventUpdateAndDelete/test_delete.txt\n\tTest_EPF_UpdatePlaceholderTests/LockToPreventUpdateAndDelete/test_delete2.txt\n\tTest_EPF_UpdatePlaceholderTests/LockToPreventUpdateAndDelete/test_delete3.txt", - "no changes added to commit (use \"git add\" and/or \"git commit -a\")\n"); - } - - this.GitCheckoutToDiscardChanges(TestParentFolderName + "/LockToPreventUpdateAndDelete/" + testFileUpdate1Name); - this.GitCheckoutToDiscardChanges(TestParentFolderName + "/LockToPreventUpdateAndDelete/" + testFileUpdate2Name); - this.GitCheckoutToDiscardChanges(TestParentFolderName + "/LockToPreventUpdateAndDelete/" + testFileUpdate3Name); - this.GitCleanFile(TestParentFolderName + "/LockToPreventUpdateAndDelete/" + testFileDelete1Name); - this.GitCleanFile(TestParentFolderName + "/LockToPreventUpdateAndDelete/" + testFileDelete2Name); - this.GitCleanFile(TestParentFolderName + "/LockToPreventUpdateAndDelete/" + testFileDelete3Name); - - this.GitStatusShouldBeClean(OldCommitId); - - GVFSHelpers.ModifiedPathsShouldContain(this.fileSystem, this.Enlistment.DotGVFSRoot, TestParentFolderName + "/LockToPreventUpdateAndDelete/" + testFileUpdate1Name); - GVFSHelpers.ModifiedPathsShouldContain(this.fileSystem, this.Enlistment.DotGVFSRoot, TestParentFolderName + "/LockToPreventUpdateAndDelete/" + testFileUpdate2Name); - GVFSHelpers.ModifiedPathsShouldContain(this.fileSystem, this.Enlistment.DotGVFSRoot, TestParentFolderName + "/LockToPreventUpdateAndDelete/" + testFileUpdate3Name); - GVFSHelpers.ModifiedPathsShouldContain(this.fileSystem, this.Enlistment.DotGVFSRoot, TestParentFolderName + "/LockToPreventUpdateAndDelete/" + testFileDelete1Name); - GVFSHelpers.ModifiedPathsShouldContain(this.fileSystem, this.Enlistment.DotGVFSRoot, TestParentFolderName + "/LockToPreventUpdateAndDelete/" + testFileDelete2Name); - GVFSHelpers.ModifiedPathsShouldContain(this.fileSystem, this.Enlistment.DotGVFSRoot, TestParentFolderName + "/LockToPreventUpdateAndDelete/" + testFileDelete3Name); - - testFileUpdate1Path.ShouldBeAFile(this.fileSystem).WithContents(testFileUpdate1OldContents); - testFileUpdate2Path.ShouldBeAFile(this.fileSystem).WithContents(testFileUpdate2OldContents); - testFileUpdate3Path.ShouldBeAFile(this.fileSystem).WithContents(testFileUpdate3OldContents); - testFileDelete1Path.ShouldNotExistOnDisk(this.fileSystem); - testFileDelete2Path.ShouldNotExistOnDisk(this.fileSystem); - testFileDelete3Path.ShouldNotExistOnDisk(this.fileSystem); - - this.GitCheckoutCommitId(NewFilesAndChangesCommitId); - - this.GitStatusShouldBeClean(NewFilesAndChangesCommitId); - testFileUpdate1Path.ShouldBeAFile(this.fileSystem).WithContents(testFileUpdate1Contents); - testFileUpdate2Path.ShouldBeAFile(this.fileSystem).WithContents(testFileUpdate2Contents); - testFileUpdate3Path.ShouldBeAFile(this.fileSystem).WithContents(testFileUpdate3Contents); - testFileDelete1Path.ShouldBeAFile(this.fileSystem).WithContents(testFileDelete1Contents); - testFileDelete2Path.ShouldBeAFile(this.fileSystem).WithContents(testFileDelete2Contents); - testFileDelete3Path.ShouldBeAFile(this.fileSystem).WithContents(testFileDelete3Contents); - } - - [TestCase, Order(6)] public void LockWithFullShareUpdateAndDelete() { string testFileUpdate4Contents = "Commit2LockToPreventUpdateAndDelete4 \r\n"; @@ -398,7 +92,7 @@ public void LockWithFullShareUpdateAndDelete() testFileDelete4Path.ShouldBeAFile(this.fileSystem).WithContents(testFileDelete4Contents); } - [TestCase, Order(7)] + [TestCase, Order(2)] public void FileProjectedAfterPlaceholderDeleteFileAndCheckout() { string testFile1Contents = "ProjectAfterDeleteAndCheckout \r\n"; @@ -436,78 +130,7 @@ public void FileProjectedAfterPlaceholderDeleteFileAndCheckout() testFile3Path.ShouldBeAFile(this.fileSystem).WithContents(testFile3Contents); } - // WindowsOnly because test requires Windows specific file sharing behavior - [TestCase, Order(8)] - [Category(Categories.WindowsOnly)] - public void LockMoreThanMaxReportedFileNames() - { - string updateFilesFolder = "FilesToUpdate"; - string deleteFilesFolder = "FilesToDelete"; - - for (int i = 1; i <= 51; ++i) - { - this.Enlistment.GetVirtualPathTo(Path.Combine(TestParentFolderName, "MaxFileListCount", updateFilesFolder, i.ToString() + ".txt")).ShouldBeAFile(this.fileSystem); - this.Enlistment.GetVirtualPathTo(Path.Combine(TestParentFolderName, "MaxFileListCount", deleteFilesFolder, i.ToString() + ".txt")).ShouldBeAFile(this.fileSystem); - } - - List openFiles = new List(); - try - { - for (int i = 1; i <= 51; ++i) - { - FileStream file = File.Open( - this.Enlistment.GetVirtualPathTo(Path.Combine(TestParentFolderName, "MaxFileListCount", updateFilesFolder, i.ToString() + ".txt")), - FileMode.Open, - FileAccess.Read, - FileShare.Read); - openFiles.Add(file); - - file = File.Open( - this.Enlistment.GetVirtualPathTo(Path.Combine(TestParentFolderName, "MaxFileListCount", deleteFilesFolder, i.ToString() + ".txt")), - FileMode.Open, - FileAccess.Read, - FileShare.Read); - openFiles.Add(file); - } - - ProcessResult result = this.InvokeGitAgainstGVFSRepo("checkout " + OldCommitId); - result.Errors.ShouldContain( - "GVFS failed to update 102 files, run 'git status' to check the status of files in the repo"); - - List expectedOutputStrings = new List() - { - "HEAD detached at " + OldCommitId, - "no changes added to commit (use \"git add\" and/or \"git commit -a\")\n" - }; - - for (int expectedFilePrefix = 1; expectedFilePrefix <= 51; ++expectedFilePrefix) - { - expectedOutputStrings.Add("modified: Test_EPF_UpdatePlaceholderTests/MaxFileListCount/" + updateFilesFolder + "/" + expectedFilePrefix.ToString() + ".txt"); - expectedOutputStrings.Add("Test_EPF_UpdatePlaceholderTests/MaxFileListCount/" + deleteFilesFolder + "/" + expectedFilePrefix.ToString() + ".txt"); - } - - GitHelpers.CheckGitCommandAgainstGVFSRepo(this.Enlistment.RepoRoot, "status -u", expectedOutputStrings.ToArray()); - } - finally - { - foreach (FileStream file in openFiles) - { - file.Dispose(); - } - } - - for (int i = 1; i <= 51; ++i) - { - this.GitCheckoutToDiscardChanges(TestParentFolderName + "/MaxFileListCount/" + updateFilesFolder + "/" + i.ToString() + ".txt"); - this.GitCleanFile(TestParentFolderName + "/MaxFileListCount/" + deleteFilesFolder + "/" + i.ToString() + ".txt"); - } - - this.GitStatusShouldBeClean(OldCommitId); - this.GitCheckoutCommitId(NewFilesAndChangesCommitId); - this.GitStatusShouldBeClean(NewFilesAndChangesCommitId); - } - - [TestCase, Order(9)] + [TestCase, Order(3)] public void FullFilesDontAffectThePlaceholderDatabase() { string testFile = Path.Combine(this.Enlistment.RepoRoot, "FullFilesDontAffectThePlaceholderDatabase"); @@ -540,14 +163,6 @@ private void GitStatusShouldBeClean(string commitId) "nothing to commit, working tree clean"); } - private void GitCleanFile(string gitPath) - { - GitHelpers.CheckGitCommandAgainstGVFSRepo( - this.Enlistment.RepoRoot, - "clean -f " + gitPath, - "Removing " + gitPath); - } - private void GitCheckoutToDiscardChanges(string gitPath) { GitHelpers.CheckGitCommandAgainstGVFSRepo(this.Enlistment.RepoRoot, "checkout -- " + gitPath); From 2701f18ee8680e68efa0444fca0522d6058ff22a Mon Sep 17 00:00:00 2001 From: William Baker Date: Wed, 12 Sep 2018 10:26:07 -0700 Subject: [PATCH 115/272] Revert whitespace change --- .../Tests/EnlistmentPerFixture/UpdatePlaceholderTests.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/GVFS/GVFS.FunctionalTests/Tests/EnlistmentPerFixture/UpdatePlaceholderTests.cs b/GVFS/GVFS.FunctionalTests/Tests/EnlistmentPerFixture/UpdatePlaceholderTests.cs index e451911d6..b279288a7 100644 --- a/GVFS/GVFS.FunctionalTests/Tests/EnlistmentPerFixture/UpdatePlaceholderTests.cs +++ b/GVFS/GVFS.FunctionalTests/Tests/EnlistmentPerFixture/UpdatePlaceholderTests.cs @@ -30,7 +30,7 @@ public virtual void SetupForTest() // Start each test at NewFilesAndChangesCommitId this.GitCheckoutCommitId(NewFilesAndChangesCommitId); this.GitStatusShouldBeClean(NewFilesAndChangesCommitId); - } + } [TestCase, Order(1)] public void LockWithFullShareUpdateAndDelete() From 1741cacf77230fa77701f90743fbdf0a601ead2c Mon Sep 17 00:00:00 2001 From: Kevin Willford Date: Thu, 6 Sep 2018 13:21:28 -0600 Subject: [PATCH 116/272] Update readme to include additional build badges --- Readme.md | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/Readme.md b/Readme.md index ffbf01864..e08a37903 100644 --- a/Readme.md +++ b/Readme.md @@ -2,15 +2,16 @@ ## Windows -|Branch|Unit Tests|Functional Tests| -|:--:|:--:|:--:| -|**master**|[![Build status](https://gvfs.visualstudio.com/ci/_apis/build/status/CI%20-%20Windows%20-%20master?branchName=master)](https://gvfs.visualstudio.com/ci/_build/latest?definitionId=7)|[![Build status](https://gvfs.visualstudio.com/ci/_apis/build/status/CI%20-%20Windows%20-%20Full%20Functional%20Tests?branchName=master)](https://gvfs.visualstudio.com/ci/_build/latest?definitionId=6)| - +|Branch|Unit Tests|Functional Tests|Large Repo Perf| +|:--:|:--:|:--:|:--:| +|**master**|[![Build status](https://dev.azure.com/gvfs/ci/_apis/build/status/CI%20-%20Windows?branchName=master)](https://dev.azure.com/gvfs/ci/_build/latest?definitionId=7)|[![Build status](https://dev.azure.com/gvfs/ci/_apis/build/status/CI%20-%20Windows%20-%20Full%20Functional%20Tests?branchName=master)](https://dev.azure.com/gvfs/ci/_build/latest?definitionId=6)|[![Build status](https://dev.azure.com/mseng/VSOnline/_apis/build/status/GVFS/GitHub%20VFSForGit%20Large%20Repo%20Perf%20Tests?branchName=master)](https://dev.azure.com/mseng/VSOnline/_build/latest?definitionId=7179)| +|**shipped**|[![Build status](https://dev.azure.com/gvfs/ci/_apis/build/status/CI%20-%20Windows?branchName=releases%2Fshipped)](https://dev.azure.com/gvfs/ci/_build/latest?definitionId=7)|[![Build status](https://dev.azure.com/gvfs/ci/_apis/build/status/CI%20-%20Windows%20-%20Full%20Functional%20Tests?branchName=releases%2Fshipped)](https://dev.azure.com/gvfs/ci/_build/latest?definitionId=6)|[![Build status](https://dev.azure.com/mseng/VSOnline/_apis/build/status/GVFS/GitHub%20VFSForGit%20Large%20Repo%20Perf%20Tests?branchName=releases%2Fshipped)](https://dev.azure.com/mseng/VSOnline/_build/latest?definitionId=7179)| ## Mac |Branch|Unit Tests|Functional Tests| |:--:|:--:|:--:| -|**master**|[![Build status](https://gvfs.visualstudio.com/ci/_apis/build/status/CI%20-%20Mac%20-%20master?branchName=master)](https://gvfs.visualstudio.com/ci/_build/latest?definitionId=15)|| +|**master**|[![Build status](https://dev.azure.com/gvfs/ci/_apis/build/status/CI%20-%20Mac?branchName=master)](https://dev.azure.com/gvfs/ci/_build/latest?definitionId=15)|[![Build status](https://dev.azure.com/mseng/VSOnline/_apis/build/status/GVFS/GitHub%20VFSForGit%20Mac%20Functional%20Tests?branchName=master)](https://dev.azure.com/mseng/VSOnline/_build/latest?definitionId=7350)| +|**shipped**|[![Build status](https://dev.azure.com/gvfs/ci/_apis/build/status/CI%20-%20Mac?branchName=releases%2Fshipped)](https://dev.azure.com/gvfs/ci/_build/latest?definitionId=15)|| ## What is VFS for Git? From a7c4f7b570a2843e158de98cce2dd148e631ce65 Mon Sep 17 00:00:00 2001 From: William Baker Date: Wed, 12 Sep 2018 11:04:21 -0700 Subject: [PATCH 117/272] Enable RebaseTests --- GVFS/GVFS.FunctionalTests/Tests/GitCommands/RebaseTests.cs | 1 - 1 file changed, 1 deletion(-) diff --git a/GVFS/GVFS.FunctionalTests/Tests/GitCommands/RebaseTests.cs b/GVFS/GVFS.FunctionalTests/Tests/GitCommands/RebaseTests.cs index 72b1726d6..4f7a62926 100644 --- a/GVFS/GVFS.FunctionalTests/Tests/GitCommands/RebaseTests.cs +++ b/GVFS/GVFS.FunctionalTests/Tests/GitCommands/RebaseTests.cs @@ -4,7 +4,6 @@ namespace GVFS.FunctionalTests.Tests.GitCommands { [TestFixture] [Category(Categories.GitCommands)] - [Category(Categories.MacTODO.M3)] public class RebaseTests : GitRepoTests { public RebaseTests() : base(enlistmentPerTest: true) From dc40d25baa58c3cef5fbf73a094f5272a97919dc Mon Sep 17 00:00:00 2001 From: William Baker Date: Wed, 12 Sep 2018 11:30:08 -0700 Subject: [PATCH 118/272] Enable CheckoutTests.CheckoutBranchThatHasFolderShouldGetDeleted --- GVFS/GVFS.FunctionalTests/Tests/GitCommands/CheckoutTests.cs | 1 - 1 file changed, 1 deletion(-) diff --git a/GVFS/GVFS.FunctionalTests/Tests/GitCommands/CheckoutTests.cs b/GVFS/GVFS.FunctionalTests/Tests/GitCommands/CheckoutTests.cs index 0fb311fc6..59debe779 100644 --- a/GVFS/GVFS.FunctionalTests/Tests/GitCommands/CheckoutTests.cs +++ b/GVFS/GVFS.FunctionalTests/Tests/GitCommands/CheckoutTests.cs @@ -251,7 +251,6 @@ public void CheckoutBranchAfterReadingAllFilesAndVerifyContentsCorrect() } [TestCase] - [Category(Categories.MacTODO.M3)] public void CheckoutBranchThatHasFolderShouldGetDeleted() { // this.ControlGitRepo.Commitish should not have the folder Test_ConflictTests\AddedFiles From 1376a8802b61e0c40989e147b7af27f426b4b5a1 Mon Sep 17 00:00:00 2001 From: William Baker Date: Wed, 12 Sep 2018 14:40:35 -0700 Subject: [PATCH 119/272] Update PowerShellRunner to support creating hardlinks --- .../FileSystemRunners/PowerShellRunner.cs | 10 ++++++++++ .../ModifiedPathsTests.cs | 20 ++++++++++++------- 2 files changed, 23 insertions(+), 7 deletions(-) diff --git a/GVFS/GVFS.FunctionalTests/FileSystemRunners/PowerShellRunner.cs b/GVFS/GVFS.FunctionalTests/FileSystemRunners/PowerShellRunner.cs index 9d6a1da97..46b86aa40 100644 --- a/GVFS/GVFS.FunctionalTests/FileSystemRunners/PowerShellRunner.cs +++ b/GVFS/GVFS.FunctionalTests/FileSystemRunners/PowerShellRunner.cs @@ -32,6 +32,11 @@ public class PowerShellRunner : ShellRunner "PermissionDenied" }; + public override bool SupportsHardlinkCreation + { + get { return true; } + } + protected override string FileName { get @@ -105,6 +110,11 @@ public override void CreateEmptyFile(string path) this.RunProcess(string.Format("-Command \"&{{ New-Item -ItemType file {0}}}\"", path)); } + public override void CreateHardLink(string newLinkFilePath, string existingFilePath) + { + this.RunProcess(string.Format("-Command \"&{{ New-Item -ItemType HardLink -Path {0} -Value {1}}}\"", newLinkFilePath, existingFilePath)); + } + public override void WriteAllText(string path, string contents) { this.RunProcess(string.Format("-Command \"&{{ Out-File -FilePath {0} -InputObject '{1}' -Encoding ascii -NoNewline}}\"", path, contents)); diff --git a/GVFS/GVFS.FunctionalTests/Tests/EnlistmentPerTestCase/ModifiedPathsTests.cs b/GVFS/GVFS.FunctionalTests/Tests/EnlistmentPerTestCase/ModifiedPathsTests.cs index 53a655a54..ba72a6a09 100644 --- a/GVFS/GVFS.FunctionalTests/Tests/EnlistmentPerTestCase/ModifiedPathsTests.cs +++ b/GVFS/GVFS.FunctionalTests/Tests/EnlistmentPerTestCase/ModifiedPathsTests.cs @@ -3,10 +3,9 @@ using GVFS.FunctionalTests.Tools; using GVFS.Tests.Should; using NUnit.Framework; -using System; +using System.Collections.Generic; using System.IO; using System.Linq; -using System.Runtime.InteropServices; namespace GVFS.FunctionalTests.Tests.EnlistmentPerTestCase { @@ -172,15 +171,22 @@ public static object[] Runners { get { - if (RuntimeInformation.IsOSPlatform(OSPlatform.Windows)) + List hardLinkRunners = new List(); + foreach (object[] runner in FileSystemRunner.Runners.ToList()) { - return new[] + FileSystemRunner fileSystem = runner.ToList().First() as FileSystemRunner; + if (fileSystem.SupportsHardlinkCreation) { - new object[] { new CmdRunner() }, - new object[] { new BashRunner() }, - }; + hardLinkRunners.Add(new object[] { fileSystem }); + } } + if (hardLinkRunners.Count > 0) + { + return hardLinkRunners.ToArray(); + } + + // Always return at least one runner that supports creating hard links return new[] { new object[] { new BashRunner() } }; } } From d5f4519fd496b40b9cf2f5e1c9cefef36181b410 Mon Sep 17 00:00:00 2001 From: William Baker Date: Wed, 12 Sep 2018 15:34:11 -0700 Subject: [PATCH 120/272] Add comment to DeleteFileThenCheckout --- GVFS/GVFS.FunctionalTests/Tests/GitCommands/CheckoutTests.cs | 1 + 1 file changed, 1 insertion(+) diff --git a/GVFS/GVFS.FunctionalTests/Tests/GitCommands/CheckoutTests.cs b/GVFS/GVFS.FunctionalTests/Tests/GitCommands/CheckoutTests.cs index 59debe779..ec5db6744 100644 --- a/GVFS/GVFS.FunctionalTests/Tests/GitCommands/CheckoutTests.cs +++ b/GVFS/GVFS.FunctionalTests/Tests/GitCommands/CheckoutTests.cs @@ -802,6 +802,7 @@ public void SuccessfullyChecksOutDirectoryToFileToDirectory() this.ShouldNotExistOnDisk("d", "c"); } + // TODO(Mac): This test needs the fix for issue #264 [TestCase] [Category(Categories.MacTODO.M3)] public void DeleteFileThenCheckout() From 4f2f359c82b940f3b335c5efcb72dc4e1a5ce984 Mon Sep 17 00:00:00 2001 From: William Baker Date: Wed, 12 Sep 2018 16:35:38 -0700 Subject: [PATCH 121/272] Mac: Prevent deletes of the index by non-git processes and enable related functional tests --- .../FileSystemRunners/BashRunner.cs | 19 +++++++++++-- .../BasicFileSystemTests.cs | 1 - .../EnlistmentPerFixture/GVFSLockTests.cs | 28 +++++++++++++------ .../MacFileSystemVirtualizer.cs | 15 ++++++++++ 4 files changed, 52 insertions(+), 11 deletions(-) diff --git a/GVFS/GVFS.FunctionalTests/FileSystemRunners/BashRunner.cs b/GVFS/GVFS.FunctionalTests/FileSystemRunners/BashRunner.cs index 40f2fbc12..b848981a3 100644 --- a/GVFS/GVFS.FunctionalTests/FileSystemRunners/BashRunner.cs +++ b/GVFS/GVFS.FunctionalTests/FileSystemRunners/BashRunner.cs @@ -4,6 +4,7 @@ using System; using System.Diagnostics; using System.IO; +using System.Runtime.InteropServices; using System.Threading; namespace GVFS.FunctionalTests.FileSystemRunners @@ -28,11 +29,18 @@ public class BashRunner : ShellRunner "Function not implemented" }; - private static string[] permissionDeniedMessage = new string[] + private static string[] permissionDeniedWindowsMessage = new string[] { "Permission denied" }; + // TODO(Mac): Update this message when the kext returns a more accurate + // error code + private static string[] permissionDeniedMacMessage = new string[] + { + "Resource temporarily unavailable" + }; + private readonly string pathToBash; public BashRunner() @@ -246,7 +254,14 @@ public override void DeleteFile_FileShouldNotBeFound(string path) public override void DeleteFile_AccessShouldBeDenied(string path) { - this.DeleteFile(path).ShouldContain(permissionDeniedMessage); + if (RuntimeInformation.IsOSPlatform(OSPlatform.Windows)) + { + this.DeleteFile(path).ShouldContain(permissionDeniedWindowsMessage); + } + else + { + this.DeleteFile(path).ShouldContain(permissionDeniedMacMessage); + } } public override void ReadAllText_FileShouldNotBeFound(string path) diff --git a/GVFS/GVFS.FunctionalTests/Tests/EnlistmentPerFixture/BasicFileSystemTests.cs b/GVFS/GVFS.FunctionalTests/Tests/EnlistmentPerFixture/BasicFileSystemTests.cs index 214f12697..3c05f8151 100644 --- a/GVFS/GVFS.FunctionalTests/Tests/EnlistmentPerFixture/BasicFileSystemTests.cs +++ b/GVFS/GVFS.FunctionalTests/Tests/EnlistmentPerFixture/BasicFileSystemTests.cs @@ -785,7 +785,6 @@ public void MoveDotGitFullFolderTreeToDotGitFullFolder(FileSystemRunner fileSyst } [TestCaseSource(typeof(FileSystemRunner), FileSystemRunner.TestRunners)] - [Category(Categories.MacTODO.M4)] public void DeleteIndexFileFails(FileSystemRunner fileSystem) { string indexFilePath = this.Enlistment.GetVirtualPathTo(Path.Combine(".git", "index")); diff --git a/GVFS/GVFS.FunctionalTests/Tests/EnlistmentPerFixture/GVFSLockTests.cs b/GVFS/GVFS.FunctionalTests/Tests/EnlistmentPerFixture/GVFSLockTests.cs index c7047c266..084e49111 100644 --- a/GVFS/GVFS.FunctionalTests/Tests/EnlistmentPerFixture/GVFSLockTests.cs +++ b/GVFS/GVFS.FunctionalTests/Tests/EnlistmentPerFixture/GVFSLockTests.cs @@ -32,7 +32,6 @@ private enum MoveFileFlags : uint } [TestCase] - [Category(Categories.MacTODO.M2)] public void GitCheckoutFailsOutsideLock() { const string BackupPrefix = "BACKUP_"; @@ -65,21 +64,18 @@ public void GitCheckoutFailsOutsideLock() } [TestCase] - [Category(Categories.MacTODO.M4)] public void LockPreventsRenameFromOutsideRootOnTopOfIndex() { this.OverwritingIndexShouldFail(Path.Combine(this.Enlistment.EnlistmentRoot, "LockPreventsRenameFromOutsideRootOnTopOfIndex.txt")); } [TestCase] - [Category(Categories.MacTODO.M4)] public void LockPreventsRenameFromInsideWorkingTreeOnTopOfIndex() { this.OverwritingIndexShouldFail(this.Enlistment.GetVirtualPathTo("LockPreventsRenameFromInsideWorkingTreeOnTopOfIndex.txt")); } [TestCase] - [Category(Categories.MacTODO.M4)] public void LockPreventsRenameOfIndexLockOnTopOfIndex() { this.OverwritingIndexShouldFail(this.Enlistment.GetVirtualPathTo(".git", "index.lock")); @@ -91,6 +87,9 @@ private static extern bool MoveFileEx( string newFileName, uint flags); + [DllImport("libc", EntryPoint = "rename", SetLastError = true)] + private static extern int Rename(string oldPath, string newPath); + private void OverwritingIndexShouldFail(string testFilePath) { string indexPath = this.Enlistment.GetVirtualPathTo(".git", "index"); @@ -102,15 +101,28 @@ private void OverwritingIndexShouldFail(string testFilePath) this.fileSystem.WriteAllText(testFilePath, testFileContents); this.Enlistment.WaitForBackgroundOperations().ShouldEqual(true, "Background operations failed to complete."); - MoveFileEx( - testFilePath, - indexPath, - (uint)(MoveFileFlags.MoveFileReplaceExisting | MoveFileFlags.MoveFileCopyAllowed)).ShouldBeFalse("GVFS should prevent renaming on top of index when GVFSLock is not held"); + + this.RenameAndOverwrite(testFilePath, indexPath).ShouldBeFalse("GVFS should prevent renaming on top of index when GVFSLock is not held"); byte[] newIndexContents = File.ReadAllBytes(indexPath); indexContents.SequenceEqual(newIndexContents).ShouldBeTrue("Index contenst should not have changed"); this.fileSystem.DeleteFile(testFilePath); } + + private bool RenameAndOverwrite(string oldPath, string newPath) + { + if (RuntimeInformation.IsOSPlatform(OSPlatform.Windows)) + { + return MoveFileEx( + oldPath, + newPath, + (uint)(MoveFileFlags.MoveFileReplaceExisting | MoveFileFlags.MoveFileCopyAllowed)); + } + else + { + return Rename(oldPath, newPath) == 0; + } + } } } diff --git a/GVFS/GVFS.Platform.Mac/MacFileSystemVirtualizer.cs b/GVFS/GVFS.Platform.Mac/MacFileSystemVirtualizer.cs index bc13595cd..6881adedc 100644 --- a/GVFS/GVFS.Platform.Mac/MacFileSystemVirtualizer.cs +++ b/GVFS/GVFS.Platform.Mac/MacFileSystemVirtualizer.cs @@ -330,6 +330,21 @@ private Result OnPreDelete(string relativePath, bool isDirectory) bool pathInsideDotGit = Virtualization.FileSystemCallbacks.IsPathInsideDotGit(relativePath); if (pathInsideDotGit) { + if (relativePath.Equals(GVFSConstants.DotGit.Index, StringComparison.OrdinalIgnoreCase)) + { + string lockedGitCommand = this.Context.Repository.GVFSLock.GetLockedGitCommand(); + if (string.IsNullOrEmpty(lockedGitCommand)) + { + EventMetadata metadata = new EventMetadata(); + metadata.Add("Area", EtwArea); + metadata.Add(TracingConstants.MessageKey.WarningMessage, "Blocked index delete outside the lock"); + this.Context.Tracer.RelatedEvent(EventLevel.Warning, $"{nameof(OnPreDelete)}_BlockedIndexDelete", metadata); + + // TODO(Mac): Is this the correct Result to return? + return Result.EAccessDenied; + } + } + this.OnDotGitFileOrFolderDeleted(relativePath); } else From a1fb40fb6871cd5a079ae78f980bbed1367811a5 Mon Sep 17 00:00:00 2001 From: Kevin Willford Date: Wed, 12 Sep 2018 13:09:09 -0600 Subject: [PATCH 122/272] Clean up code that was referring to sparse-check our always_exclude --- .../Prefetch/Git/GitIndexGenerator.cs | 11 ++-- .../EnlistmentPerFixture/DehydrateTests.cs | 5 +- .../EnlistmentPerFixture/GitFilesTests.cs | 16 +++--- .../WorkingDirectoryTests.cs | 2 +- .../ModifiedPathsTests.cs | 54 +++++++++---------- .../Tests/GitCommands/CheckoutTests.cs | 2 +- .../Tests/GitCommands/UpdateIndexTests.cs | 2 +- .../GVFS.FunctionalTests/Tools/GVFSHelpers.cs | 7 ++- GVFS/GVFS.FunctionalTests/Tools/GitHelpers.cs | 2 - .../Tools/TestConstants.cs | 1 + .../FileSystemCallbacks.cs | 2 +- .../Projection/GitIndexProjection.cs | 2 +- GVFS/GVFS/CommandLine/CloneVerb.cs | 4 -- GVFS/GVFS/CommandLine/DehydrateVerb.cs | 41 +------------- GVFS/GVFS/CommandLine/GVFSVerb.cs | 1 - GVFS/GVFS/RepairJobs/GitIndexRepairJob.cs | 2 - 16 files changed, 52 insertions(+), 102 deletions(-) diff --git a/GVFS/GVFS.Common/Prefetch/Git/GitIndexGenerator.cs b/GVFS/GVFS.Common/Prefetch/Git/GitIndexGenerator.cs index 3bc66f07c..98f0ac577 100644 --- a/GVFS/GVFS.Common/Prefetch/Git/GitIndexGenerator.cs +++ b/GVFS/GVFS.Common/Prefetch/Git/GitIndexGenerator.cs @@ -60,11 +60,11 @@ public GitIndexGenerator(ITracer tracer, Enlistment enlistment, bool shouldHashI public bool HasFailures { get; private set; } - public void CreateFromHeadTree(uint indexVersion, HashSet sparseCheckoutEntries = null) + public void CreateFromHeadTree(uint indexVersion) { using (ITracer updateIndexActivity = this.tracer.StartActivity("CreateFromHeadTree", EventLevel.Informational)) { - Thread entryWritingThread = new Thread(() => this.WriteAllEntries(indexVersion, sparseCheckoutEntries)); + Thread entryWritingThread = new Thread(() => this.WriteAllEntries(indexVersion)); entryWritingThread.Start(); GitProcess git = new GitProcess(this.enlistment); @@ -94,7 +94,7 @@ private void EnqueueEntriesFromLsTree(string line) } } - private void WriteAllEntries(uint version, HashSet sparseCheckoutEntries) + private void WriteAllEntries(uint version) { try { @@ -109,10 +109,7 @@ private void WriteAllEntries(uint version, HashSet sparseCheckoutEntries LsTreeEntry entry; while (this.entryQueue.TryTake(out entry, Timeout.Infinite)) { - bool skipWorkTree = - sparseCheckoutEntries != null && - !sparseCheckoutEntries.Contains(entry.Filename) && - !sparseCheckoutEntries.Contains(this.GetDirectoryNameForGitPath(entry.Filename)); + bool skipWorkTree = false; this.WriteEntry(writer, version, entry.Sha, entry.Filename, skipWorkTree, ref lastStringLength); } diff --git a/GVFS/GVFS.FunctionalTests/Tests/EnlistmentPerFixture/DehydrateTests.cs b/GVFS/GVFS.FunctionalTests/Tests/EnlistmentPerFixture/DehydrateTests.cs index 1fc03382b..9c51fefcc 100644 --- a/GVFS/GVFS.FunctionalTests/Tests/EnlistmentPerFixture/DehydrateTests.cs +++ b/GVFS/GVFS.FunctionalTests/Tests/EnlistmentPerFixture/DehydrateTests.cs @@ -65,10 +65,7 @@ public void DehydrateShouldBackupFiles() // .git folder items string gitFolder = Path.Combine(backupFolderItems[0], ".git"); - this.DirectoryShouldContain(gitFolder, "index", "info"); - - string gitInfoFolder = Path.Combine(gitFolder, "info"); - this.DirectoryShouldContain(gitInfoFolder, "sparse-checkout"); + this.DirectoryShouldContain(gitFolder, "index"); // .gvfs folder items string gvfsFolder = Path.Combine(backupFolderItems[0], ".gvfs"); diff --git a/GVFS/GVFS.FunctionalTests/Tests/EnlistmentPerFixture/GitFilesTests.cs b/GVFS/GVFS.FunctionalTests/Tests/EnlistmentPerFixture/GitFilesTests.cs index d1b56fc48..3100e7ad0 100644 --- a/GVFS/GVFS.FunctionalTests/Tests/EnlistmentPerFixture/GitFilesTests.cs +++ b/GVFS/GVFS.FunctionalTests/Tests/EnlistmentPerFixture/GitFilesTests.cs @@ -141,7 +141,7 @@ public void RenameFolderTest() [TestCase, Order(6)] [Category(Categories.MacTODO.M2)] - public void CaseOnlyRenameOfNewFolderKeepsExcludeEntries() + public void CaseOnlyRenameOfNewFolderKeepsModifiedPathsEntries() { string[] expectedModifiedPathsEntries = { @@ -162,7 +162,7 @@ public void CaseOnlyRenameOfNewFolderKeepsExcludeEntries() } [TestCase, Order(7)] - public void ReadingFileDoesNotUpdateIndexOrSparseCheckout() + public void ReadingFileDoesNotUpdateIndexOrModifiedPaths() { string gitFileToCheck = "GVFS/GVFS.FunctionalTests/Category/CategoryConstants.cs"; string virtualFile = this.Enlistment.GetVirtualPathTo(gitFileToCheck); @@ -296,13 +296,13 @@ public void DeletedFolderAndChildrenAddedToToModifiedPathsFile() } [TestCase, Order(13)] - public void FileRenamedOutOfRepoAddedToModifiedPathsFile() + public void FileRenamedOutOfRepoAddedToModifiedPathsAndSkipWorktreeBitCleared() { string fileToRenameEntry = "GVFlt_MoveFileTest/PartialToOutside/from/lessInFrom.txt"; string fileToRenameVirtualPath = this.Enlistment.GetVirtualPathTo(fileToRenameEntry); this.VerifyWorktreeBit(fileToRenameEntry, LsFilesStatus.SkipWorktree); - string fileOutsideRepoPath = Path.Combine(this.Enlistment.EnlistmentRoot, "FileRenamedOutOfRepoAddedToSparseCheckoutAndSkipWorktreeBitCleared.txt"); + string fileOutsideRepoPath = Path.Combine(this.Enlistment.EnlistmentRoot, $"{nameof(this.FileRenamedOutOfRepoAddedToModifiedPathsAndSkipWorktreeBitCleared)}.txt"); this.fileSystem.MoveFile(fileToRenameVirtualPath, fileOutsideRepoPath); fileOutsideRepoPath.ShouldBeAFile(this.fileSystem).WithContents("lessData"); @@ -315,13 +315,13 @@ public void FileRenamedOutOfRepoAddedToModifiedPathsFile() } [TestCase, Order(14)] - public void OverwrittenFileAddedToSparseCheckoutAndSkipWorktreeBitCleared() + public void OverwrittenFileAddedToModifiedPathsAndSkipWorktreeBitCleared() { string fileToOverwriteEntry = "Test_EPF_WorkingDirectoryTests/1/2/3/4/ReadDeepProjectedFile.cpp"; string fileToOverwriteVirtualPath = this.Enlistment.GetVirtualPathTo(fileToOverwriteEntry); this.VerifyWorktreeBit(fileToOverwriteEntry, LsFilesStatus.SkipWorktree); - string testContents = "Test contents for FileRenamedOutOfRepoWillBeAddedToSparseCheckoutAndHaveSkipWorktreeBitCleared"; + string testContents = $"Test contents for {nameof(this.OverwrittenFileAddedToModifiedPathsAndSkipWorktreeBitCleared)}"; this.fileSystem.WriteAllText(fileToOverwriteVirtualPath, testContents); this.Enlistment.WaitForBackgroundOperations().ShouldEqual(true, "Background operations failed to complete."); @@ -336,13 +336,13 @@ public void OverwrittenFileAddedToSparseCheckoutAndSkipWorktreeBitCleared() [TestCase, Order(15)] [Category(Categories.MacTODO.M2)] - public void SupersededFileAddedToSparseCheckoutAndSkipWorktreeBitCleared() + public void SupersededFileAddedToModifiedPathsAndSkipWorktreeBitCleared() { string fileToSupersedeEntry = "GVFlt_FileOperationTest/WriteAndVerify.txt"; string fileToSupersedePath = this.Enlistment.GetVirtualPathTo("GVFlt_FileOperationTest\\WriteAndVerify.txt"); this.VerifyWorktreeBit(fileToSupersedeEntry, LsFilesStatus.SkipWorktree); - string newContent = "SupersededFileWillBeAddedToSparseCheckoutAndHaveSkipWorktreeBitCleared test new contents"; + string newContent = $"{nameof(this.SupersededFileAddedToModifiedPathsAndSkipWorktreeBitCleared)} test new contents"; SupersedeFile(fileToSupersedePath, newContent).ShouldEqual(true); this.Enlistment.WaitForBackgroundOperations().ShouldEqual(true, "Background operations failed to complete."); diff --git a/GVFS/GVFS.FunctionalTests/Tests/EnlistmentPerFixture/WorkingDirectoryTests.cs b/GVFS/GVFS.FunctionalTests/Tests/EnlistmentPerFixture/WorkingDirectoryTests.cs index d5088205e..7dd6dd41b 100644 --- a/GVFS/GVFS.FunctionalTests/Tests/EnlistmentPerFixture/WorkingDirectoryTests.cs +++ b/GVFS/GVFS.FunctionalTests/Tests/EnlistmentPerFixture/WorkingDirectoryTests.cs @@ -430,7 +430,7 @@ public void FolderContentsCorrectAfterCreateNewFolderRenameAndCheckoutCommitWith folder.ShouldNotExistOnDisk(this.fileSystem); GVFSHelpers.ModifiedPathsShouldNotContain(this.fileSystem, this.Enlistment.DotGVFSRoot, folderName); - // Confirm sparse-checkout picks up renamed folder + // Confirm modified paths picks up renamed folder string newFolder = this.Enlistment.GetVirtualPathTo("newFolder"); this.fileSystem.CreateDirectory(newFolder); this.fileSystem.MoveDirectory(newFolder, folder); diff --git a/GVFS/GVFS.FunctionalTests/Tests/EnlistmentPerTestCase/ModifiedPathsTests.cs b/GVFS/GVFS.FunctionalTests/Tests/EnlistmentPerTestCase/ModifiedPathsTests.cs index 53a655a54..3e9a65187 100644 --- a/GVFS/GVFS.FunctionalTests/Tests/EnlistmentPerTestCase/ModifiedPathsTests.cs +++ b/GVFS/GVFS.FunctionalTests/Tests/EnlistmentPerTestCase/ModifiedPathsTests.cs @@ -3,9 +3,7 @@ using GVFS.FunctionalTests.Tools; using GVFS.Tests.Should; using NUnit.Framework; -using System; using System.IO; -using System.Linq; using System.Runtime.InteropServices; namespace GVFS.FunctionalTests.Tests.EnlistmentPerTestCase @@ -18,33 +16,33 @@ public class ModifiedPathsTests : TestsWithEnlistmentPerTestCase private static readonly string FileToDelete = "Readme.md"; private static readonly string FileToRename = Path.Combine("GVFS", "GVFS.Mount", "MountVerb.cs"); private static readonly string RenameFileTarget = Path.Combine("GVFS", "GVFS.Mount", "MountVerb2.cs"); - private static readonly string FolderToCreate = "PersistedSparseExcludeTests_NewFolder"; - private static readonly string FolderToRename = "PersistedSparseExcludeTests_NewFolderForRename"; - private static readonly string RenameFolderTarget = "PersistedSparseExcludeTests_NewFolderForRename2"; + private static readonly string FolderToCreate = $"{nameof(ModifiedPathsTests)}_NewFolder"; + private static readonly string FolderToRename = $"{nameof(ModifiedPathsTests)}_NewFolderForRename"; + private static readonly string RenameFolderTarget = $"{nameof(ModifiedPathsTests)}_NewFolderForRename2"; private static readonly string DotGitFileToCreate = Path.Combine(".git", "TestFileFromDotGit.txt"); private static readonly string RenameNewDotGitFileTarget = "TestFileFromDotGit.txt"; - private static readonly string FileToCreateOutsideRepo = "PersistedSparseExcludeTests_outsideRepo.txt"; - private static readonly string FolderToCreateOutsideRepo = "PersistedSparseExcludeTests_outsideFolder"; + private static readonly string FileToCreateOutsideRepo = $"{nameof(ModifiedPathsTests)}_outsideRepo.txt"; + private static readonly string FolderToCreateOutsideRepo = $"{nameof(ModifiedPathsTests)}_outsideFolder"; private static readonly string FolderToDelete = "Scripts"; - private static readonly string ExpectedModifiedFilesContentsAfterRemount = -@"A .gitattributes -A GVFS/TestAddFile.txt -A GVFS/GVFS/Program.cs -A Readme.md -A GVFS/GVFS.Mount/MountVerb.cs -A GVFS/GVFS.Mount/MountVerb2.cs -A PersistedSparseExcludeTests_NewFolder/ -A PersistedSparseExcludeTests_NewFolderForRename/ -A PersistedSparseExcludeTests_NewFolderForRename2/ -A TestFileFromDotGit.txt -A PersistedSparseExcludeTests_outsideRepo.txt -A PersistedSparseExcludeTests_outsideFolder/ -A Scripts/CreateCommonAssemblyVersion.bat -A Scripts/CreateCommonCliAssemblyVersion.bat -A Scripts/CreateCommonVersionHeader.bat -A Scripts/RunFunctionalTests.bat -A Scripts/RunUnitTests.bat -A Scripts/ + private static readonly string ExpectedModifiedFilesContentsAfterRemount = +$@"A .gitattributes +A {GVFSHelpers.ConvertPathToGitFormat(FileToAdd)} +A {GVFSHelpers.ConvertPathToGitFormat(FileToUpdate)} +A {FileToDelete} +A {GVFSHelpers.ConvertPathToGitFormat(FileToRename)} +A {GVFSHelpers.ConvertPathToGitFormat(RenameFileTarget)} +A {FolderToCreate}/ +A {FolderToRename}/ +A {RenameFolderTarget}/ +A {RenameNewDotGitFileTarget} +A {FileToCreateOutsideRepo} +A {FolderToCreateOutsideRepo}/ +A {FolderToDelete}/CreateCommonAssemblyVersion.bat +A {FolderToDelete}/CreateCommonCliAssemblyVersion.bat +A {FolderToDelete}/CreateCommonVersionHeader.bat +A {FolderToDelete}/RunFunctionalTests.bat +A {FolderToDelete}/RunUnitTests.bat +A {FolderToDelete}/ "; [Category(Categories.MacTODO.M2)] @@ -72,14 +70,14 @@ public void ModifiedPathsSavedAfterRemount(FileSystemRunner fileSystem) string folderToRenameTarget = this.Enlistment.GetVirtualPathTo(RenameFolderTarget); fileSystem.MoveDirectory(folderToRename, folderToRenameTarget); - // Moving the new folder out of the repo should not change the always_exclude file + // Moving the new folder out of the repo should not change the modified paths string folderTargetOutsideSrc = Path.Combine(this.Enlistment.EnlistmentRoot, RenameFolderTarget); folderTargetOutsideSrc.ShouldNotExistOnDisk(fileSystem); fileSystem.MoveDirectory(folderToRenameTarget, folderTargetOutsideSrc); folderTargetOutsideSrc.ShouldBeADirectory(fileSystem); folderToRenameTarget.ShouldNotExistOnDisk(fileSystem); - // Moving a file from the .git folder to the working directory should add the file to the sparse-checkout + // Moving a file from the .git folder to the working directory should add the file to the modified paths string dotGitfileToAdd = this.Enlistment.GetVirtualPathTo(DotGitFileToCreate); fileSystem.WriteAllText(dotGitfileToAdd, "Contents for the new file in dot git"); fileSystem.MoveFile(dotGitfileToAdd, this.Enlistment.GetVirtualPathTo(RenameNewDotGitFileTarget)); diff --git a/GVFS/GVFS.FunctionalTests/Tests/GitCommands/CheckoutTests.cs b/GVFS/GVFS.FunctionalTests/Tests/GitCommands/CheckoutTests.cs index 0fb311fc6..30f0f02bc 100644 --- a/GVFS/GVFS.FunctionalTests/Tests/GitCommands/CheckoutTests.cs +++ b/GVFS/GVFS.FunctionalTests/Tests/GitCommands/CheckoutTests.cs @@ -823,7 +823,7 @@ public void DeleteFileThenCheckout() [Category(Categories.MacTODO.M3)] public void CheckoutEditCheckoutWithoutFolderThenCheckoutWithMultipleFiles() { - // Edit the file to get the entry in the sparse-checkout file + // Edit the file to get the entry in the modified paths database this.EditFile("Changing the content of one file", "DeleteFileWithNameAheadOfDotAndSwitchCommits", "1"); this.RunGitCommand("reset --hard -q HEAD"); diff --git a/GVFS/GVFS.FunctionalTests/Tests/GitCommands/UpdateIndexTests.cs b/GVFS/GVFS.FunctionalTests/Tests/GitCommands/UpdateIndexTests.cs index 943e54abb..5ca28eaa4 100644 --- a/GVFS/GVFS.FunctionalTests/Tests/GitCommands/UpdateIndexTests.cs +++ b/GVFS/GVFS.FunctionalTests/Tests/GitCommands/UpdateIndexTests.cs @@ -52,7 +52,7 @@ public void UpdateIndexRemoveAddFileOpenForWrite() GitHelpers.InvokeGitAgainstGVFSRepo(this.Enlistment.RepoRoot, "update-index --remove Test_ConflictTests/AddedFiles/AddedByBothDifferentContent.txt"); this.FilesShouldMatchCheckoutOfTargetBranch(); - // Open Test_ConflictTests/AddedFiles/AddedByBothDifferentContent.txt for write so that it's added to the sparse-checkout + // Open Test_ConflictTests/AddedFiles/AddedByBothDifferentContent.txt for write so that it's added to the modified paths database using (FileStream stream = File.Open(Path.Combine(this.Enlistment.RepoRoot, @"Test_ConflictTests\AddedFiles\AddedByBothDifferentContent.txt"), FileMode.Open, FileAccess.Write)) { // TODO 940287: Remove this File.Open once update-index --add\--remove are working as expected diff --git a/GVFS/GVFS.FunctionalTests/Tools/GVFSHelpers.cs b/GVFS/GVFS.FunctionalTests/Tools/GVFSHelpers.cs index 139fb0a06..95e988c18 100644 --- a/GVFS/GVFS.FunctionalTests/Tools/GVFSHelpers.cs +++ b/GVFS/GVFS.FunctionalTests/Tools/GVFSHelpers.cs @@ -23,7 +23,12 @@ public static class GVFSHelpers private const string DiskLayoutMinorVersionKey = "DiskLayoutMinorVersion"; private const string LocalCacheRootKey = "LocalCacheRoot"; private const string GitObjectsRootKey = "GitObjectsRoot"; - private const string BlobSizesRootKey = "BlobSizesRoot"; + private const string BlobSizesRootKey = "BlobSizesRoot"; + + public static string ConvertPathToGitFormat(string path) + { + return path.Replace(Path.DirectorySeparatorChar, TestConstants.GitPathSeparator); + } public static void SaveDiskLayoutVersion(string dotGVFSRoot, string majorVersion, string minorVersion) { diff --git a/GVFS/GVFS.FunctionalTests/Tools/GitHelpers.cs b/GVFS/GVFS.FunctionalTests/Tools/GitHelpers.cs index 729d1d700..333499ba0 100644 --- a/GVFS/GVFS.FunctionalTests/Tools/GitHelpers.cs +++ b/GVFS/GVFS.FunctionalTests/Tools/GitHelpers.cs @@ -12,8 +12,6 @@ namespace GVFS.FunctionalTests.Tools { public static class GitHelpers { - public const string AlwaysExcludeFilePath = @".git\info\always_exclude"; - ///

/// This string must match the command name provided in the /// GVFS.FunctionalTests.LockHolder program. diff --git a/GVFS/GVFS.FunctionalTests/Tools/TestConstants.cs b/GVFS/GVFS.FunctionalTests/Tools/TestConstants.cs index ebc6ee28f..0bf84a1b1 100644 --- a/GVFS/GVFS.FunctionalTests/Tools/TestConstants.cs +++ b/GVFS/GVFS.FunctionalTests/Tools/TestConstants.cs @@ -6,6 +6,7 @@ public static class TestConstants { public const string AllZeroSha = "0000000000000000000000000000000000000000"; public const string PartialFolderPlaceholderDatabaseValue = " PARTIAL FOLDER"; + public const char GitPathSeparator = '/'; public static class DotGit { diff --git a/GVFS/GVFS.Virtualization/FileSystemCallbacks.cs b/GVFS/GVFS.Virtualization/FileSystemCallbacks.cs index ce1b8afe4..a9ea5200e 100644 --- a/GVFS/GVFS.Virtualization/FileSystemCallbacks.cs +++ b/GVFS/GVFS.Virtualization/FileSystemCallbacks.cs @@ -705,7 +705,7 @@ private FileSystemTaskResult ExecuteBackgroundOperation(FileSystemTask gitUpdate exceptionMetadata.Add("virtualPath", gitUpdate.VirtualPath); exceptionMetadata.Add(TracingConstants.MessageKey.InfoMessage, "DirectoryNotFoundException while traversing folder path"); exceptionMetadata.Add("folderPath", folderPath); - this.context.Tracer.RelatedEvent(EventLevel.Informational, "DirectoryNotFoundWhileUpdatingAlwaysExclude", exceptionMetadata); + this.context.Tracer.RelatedEvent(EventLevel.Informational, "DirectoryNotFoundWhileUpdatingModifiedPaths", exceptionMetadata); } catch (IOException e) { diff --git a/GVFS/GVFS.Virtualization/Projection/GitIndexProjection.cs b/GVFS/GVFS.Virtualization/Projection/GitIndexProjection.cs index ba40f28b6..66b139615 100644 --- a/GVFS/GVFS.Virtualization/Projection/GitIndexProjection.cs +++ b/GVFS/GVFS.Virtualization/Projection/GitIndexProjection.cs @@ -1607,7 +1607,7 @@ private void ProcessGvUpdateDeletePlaceholderResult( // update in the working directory will be full files but we will have a placeholder entry for them as well. // There have been reports of FileSystemVirtualizationInvalidOperation getting hit without a corresponding background - // task having been scheduled (to add the file to the sparse-checkout and clear the skip-worktree bit). + // task having been scheduled (to add the file to the modified paths). // Schedule OnFailedPlaceholderUpdate\OnFailedPlaceholderDelete to be sure that Git starts managing this // file. Currently the only known way that this can happen is deleting a partial file and putting a full // file in its place while GVFS is unmounted. diff --git a/GVFS/GVFS/CommandLine/CloneVerb.cs b/GVFS/GVFS/CommandLine/CloneVerb.cs index 376e0f2a0..00a429c07 100644 --- a/GVFS/GVFS/CommandLine/CloneVerb.cs +++ b/GVFS/GVFS/CommandLine/CloneVerb.cs @@ -552,10 +552,6 @@ private Result CreateClone( Path.Combine(enlistment.WorkingDirectoryRoot, GVFSConstants.DotGit.Head), "ref: refs/heads/" + branch); - File.AppendAllText( - Path.Combine(enlistment.WorkingDirectoryRoot, GVFSConstants.DotGit.Info.SparseCheckoutPath), - GVFSConstants.GitPathSeparatorString + GVFSConstants.SpecialGitFiles.GitAttributes + "\n"); - if (!this.TryDownloadRootGitAttributes(enlistment, gitObjects, gitRepo, out errorMessage)) { return new Result(errorMessage); diff --git a/GVFS/GVFS/CommandLine/DehydrateVerb.cs b/GVFS/GVFS/CommandLine/DehydrateVerb.cs index 39100557a..71342283a 100644 --- a/GVFS/GVFS/CommandLine/DehydrateVerb.cs +++ b/GVFS/GVFS/CommandLine/DehydrateVerb.cs @@ -232,7 +232,6 @@ private bool TryBackupFiles(ITracer tracer, GVFSEnlistment enlistment, string ba { string backupSrc = Path.Combine(backupRoot, "src"); string backupGit = Path.Combine(backupRoot, ".git"); - string backupInfo = Path.Combine(backupGit, GVFSConstants.DotGit.Info.Name); string backupGvfs = Path.Combine(backupRoot, ".gvfs"); string backupDatabases = Path.Combine(backupGvfs, GVFSConstants.DotGVFS.Databases.Name); @@ -243,7 +242,6 @@ private bool TryBackupFiles(ITracer tracer, GVFSEnlistment enlistment, string ba string ioError; if (!this.TryIO(tracer, () => Directory.CreateDirectory(backupRoot), "Create backup directory", out ioError) || !this.TryIO(tracer, () => Directory.CreateDirectory(backupGit), "Create backup .git directory", out ioError) || - !this.TryIO(tracer, () => Directory.CreateDirectory(backupInfo), "Create backup .git\\info directory", out ioError) || !this.TryIO(tracer, () => Directory.CreateDirectory(backupGvfs), "Create backup .gvfs directory", out ioError) || !this.TryIO(tracer, () => Directory.CreateDirectory(backupDatabases), "Create backup .gvfs databases directory", out ioError)) { @@ -266,43 +264,6 @@ private bool TryBackupFiles(ITracer tracer, GVFSEnlistment enlistment, string ba return false; } - // ... but then move the hydration-related files back to the backup... - if (!this.TryIO( - tracer, - () => File.Move( - Path.Combine(enlistment.WorkingDirectoryRoot, GVFSConstants.DotGit.Info.SparseCheckoutPath), - Path.Combine(backupInfo, GVFSConstants.DotGit.Info.SparseCheckoutName)), - "Backup the sparse-checkout file", - out errorMessage) || - !this.TryIO( - tracer, - () => - { - if (File.Exists(Path.Combine(enlistment.WorkingDirectoryRoot, GVFSConstants.DotGit.Info.AlwaysExcludePath))) - { - File.Move( - Path.Combine(enlistment.WorkingDirectoryRoot, GVFSConstants.DotGit.Info.AlwaysExcludePath), - Path.Combine(backupInfo, GVFSConstants.DotGit.Info.AlwaysExcludeName)); - } - }, - "Backup the always_exclude file", - out errorMessage)) - { - return false; - } - - // ... and recreate empty ones in the new .git folder... - if (!this.TryIO( - tracer, - () => File.AppendAllText( - Path.Combine(enlistment.WorkingDirectoryRoot, GVFSConstants.DotGit.Info.SparseCheckoutPath), - GVFSConstants.GitPathSeparatorString + GVFSConstants.SpecialGitFiles.GitAttributes + "\n"), - "Recreate a new sparse-checkout file", - out errorMessage)) - { - return false; - } - // ... backup the .gvfs hydration-related data structures... string databasesFolder = Path.Combine(enlistment.DotGVFSRoot, GVFSConstants.DotGVFS.Databases.Name); if (!TryBackupFilesInFolder(tracer, databasesFolder, backupDatabases, searchPattern: "*", filenamesToSkip: "RepoMetadata.dat")) @@ -419,7 +380,7 @@ private bool TryRecreateIndex(ITracer tracer, GVFSEnlistment enlistment) if (!this.ShowStatusWhileRunning( () => { - // Create a new index based on the new minimal sparse-checkout + // Create a new index based on the new minimal modified paths using (NamedPipeServer pipeServer = AllowAllLocksNamedPipeServer.Create(tracer, enlistment)) { GitProcess git = new GitProcess(enlistment); diff --git a/GVFS/GVFS/CommandLine/GVFSVerb.cs b/GVFS/GVFS/CommandLine/GVFSVerb.cs index dbb9440d6..325e2dcee 100644 --- a/GVFS/GVFS/CommandLine/GVFSVerb.cs +++ b/GVFS/GVFS/CommandLine/GVFSVerb.cs @@ -84,7 +84,6 @@ public static bool TrySetRequiredGitConfigSettings(Enlistment enlistment) { "core.midx", "true" }, { "core.preloadIndex", "true" }, { "core.safecrlf", "false" }, - { "core.sparseCheckout", "true" }, { "core.untrackedCache", "false" }, { "core.repositoryformatversion", "0" }, { "core.filemode", "false" }, diff --git a/GVFS/GVFS/RepairJobs/GitIndexRepairJob.cs b/GVFS/GVFS/RepairJobs/GitIndexRepairJob.cs index f6cb9f547..ce41ed125 100644 --- a/GVFS/GVFS/RepairJobs/GitIndexRepairJob.cs +++ b/GVFS/GVFS/RepairJobs/GitIndexRepairJob.cs @@ -9,13 +9,11 @@ namespace GVFS.RepairJobs public class GitIndexRepairJob : RepairJob { private readonly string indexPath; - private readonly string sparseCheckoutPath; public GitIndexRepairJob(ITracer tracer, TextWriter output, GVFSEnlistment enlistment) : base(tracer, output, enlistment) { this.indexPath = Path.Combine(this.Enlistment.DotGitRoot, GVFSConstants.DotGit.IndexName); - this.sparseCheckoutPath = Path.Combine(this.Enlistment.WorkingDirectoryRoot, GVFSConstants.DotGit.Info.SparseCheckoutPath); } public override string Name From abd1cae3f0d7ef9a2715dd1378e428edd46d975f Mon Sep 17 00:00:00 2001 From: William Baker Date: Thu, 13 Sep 2018 08:51:46 -0700 Subject: [PATCH 123/272] Fix StyleCop issues --- GVFS/GVFS.Platform.Mac/MacFileSystemVirtualizer.cs | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/GVFS/GVFS.Platform.Mac/MacFileSystemVirtualizer.cs b/GVFS/GVFS.Platform.Mac/MacFileSystemVirtualizer.cs index 6881adedc..ca7f823f8 100644 --- a/GVFS/GVFS.Platform.Mac/MacFileSystemVirtualizer.cs +++ b/GVFS/GVFS.Platform.Mac/MacFileSystemVirtualizer.cs @@ -16,6 +16,8 @@ public class MacFileSystemVirtualizer : FileSystemVirtualizer { public static readonly byte[] PlaceholderVersionId = ToVersionIdByteArray(new byte[] { PlaceholderVersion }); + private const string ClassName = nameof(MacFileSystemVirtualizer); + private VirtualizationInstance virtualizationInstance; public MacFileSystemVirtualizer(GVFSContext context, GVFSGitObjects gitObjects) @@ -32,6 +34,14 @@ public MacFileSystemVirtualizer( this.virtualizationInstance = virtualizationInstance ?? new VirtualizationInstance(); } + protected override string EtwArea + { + get + { + return ClassName; + } + } + public static FSResult ResultToFSResult(Result result) { switch (result) @@ -336,7 +346,7 @@ private Result OnPreDelete(string relativePath, bool isDirectory) if (string.IsNullOrEmpty(lockedGitCommand)) { EventMetadata metadata = new EventMetadata(); - metadata.Add("Area", EtwArea); + metadata.Add("Area", this.EtwArea); metadata.Add(TracingConstants.MessageKey.WarningMessage, "Blocked index delete outside the lock"); this.Context.Tracer.RelatedEvent(EventLevel.Warning, $"{nameof(OnPreDelete)}_BlockedIndexDelete", metadata); From 0e177b4730be46109ee0333af04a5ab35d2f7846 Mon Sep 17 00:00:00 2001 From: Kevin Willford Date: Thu, 13 Sep 2018 09:49:24 -0600 Subject: [PATCH 124/272] Add build badges for Large Repo Build and branch specific links --- Readme.md | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/Readme.md b/Readme.md index e08a37903..e049af181 100644 --- a/Readme.md +++ b/Readme.md @@ -2,16 +2,16 @@ ## Windows -|Branch|Unit Tests|Functional Tests|Large Repo Perf| -|:--:|:--:|:--:|:--:| -|**master**|[![Build status](https://dev.azure.com/gvfs/ci/_apis/build/status/CI%20-%20Windows?branchName=master)](https://dev.azure.com/gvfs/ci/_build/latest?definitionId=7)|[![Build status](https://dev.azure.com/gvfs/ci/_apis/build/status/CI%20-%20Windows%20-%20Full%20Functional%20Tests?branchName=master)](https://dev.azure.com/gvfs/ci/_build/latest?definitionId=6)|[![Build status](https://dev.azure.com/mseng/VSOnline/_apis/build/status/GVFS/GitHub%20VFSForGit%20Large%20Repo%20Perf%20Tests?branchName=master)](https://dev.azure.com/mseng/VSOnline/_build/latest?definitionId=7179)| -|**shipped**|[![Build status](https://dev.azure.com/gvfs/ci/_apis/build/status/CI%20-%20Windows?branchName=releases%2Fshipped)](https://dev.azure.com/gvfs/ci/_build/latest?definitionId=7)|[![Build status](https://dev.azure.com/gvfs/ci/_apis/build/status/CI%20-%20Windows%20-%20Full%20Functional%20Tests?branchName=releases%2Fshipped)](https://dev.azure.com/gvfs/ci/_build/latest?definitionId=6)|[![Build status](https://dev.azure.com/mseng/VSOnline/_apis/build/status/GVFS/GitHub%20VFSForGit%20Large%20Repo%20Perf%20Tests?branchName=releases%2Fshipped)](https://dev.azure.com/mseng/VSOnline/_build/latest?definitionId=7179)| +|Branch|Unit Tests|Functional Tests|Large Repo Perf|Large Repo Build| +|:--:|:--:|:--:|:--:|:--:| +|**master**|[![Build status](https://dev.azure.com/gvfs/ci/_apis/build/status/CI%20-%20Windows?branchName=master)](https://dev.azure.com/gvfs/ci/_build/latest?definitionId=7&branchName=master)|[![Build status](https://dev.azure.com/gvfs/ci/_apis/build/status/CI%20-%20Windows%20-%20Full%20Functional%20Tests?branchName=master)](https://dev.azure.com/gvfs/ci/_build/latest?definitionId=6&branchName=master)|[![Build status](https://dev.azure.com/mseng/VSOnline/_apis/build/status/GVFS/GitHub%20VFSForGit%20Large%20Repo%20Perf%20Tests?branchName=master)](https://dev.azure.com/mseng/VSOnline/_build/latest?definitionId=7179&branchName=master)|[![Build status](https://dev.azure.com/mseng/VSOnline/_apis/build/status/GVFS/GitHub%20VFSForGit%20Large%20Repo%20Build?branchName=master)](https://dev.azure.com/mseng/VSOnline/_build/latest?definitionId=7180&branchName=master)| +|**shipped**|[![Build status](https://dev.azure.com/gvfs/ci/_apis/build/status/CI%20-%20Windows?branchName=releases%2Fshipped)](https://dev.azure.com/gvfs/ci/_build/latest?definitionId=7&branchName=releases%2Fshipped)|[![Build status](https://dev.azure.com/gvfs/ci/_apis/build/status/CI%20-%20Windows%20-%20Full%20Functional%20Tests?branchName=releases%2Fshipped)](https://dev.azure.com/gvfs/ci/_build/latest?definitionId=6&branchName=releases%2Fshipped)|[![Build status](https://dev.azure.com/mseng/VSOnline/_apis/build/status/GVFS/GitHub%20VFSForGit%20Large%20Repo%20Perf%20Tests?branchName=releases%2Fshipped)](https://dev.azure.com/mseng/VSOnline/_build/latest?definitionId=7179&branchName=releases%2Fshipped)|[![Build status](https://dev.azure.com/mseng/VSOnline/_apis/build/status/GVFS/GitHub%20VFSForGit%20Large%20Repo%20Build?branchName=releases%2Fshipped)](https://dev.azure.com/mseng/VSOnline/_build/latest?definitionId=7180&branchName=releases%2Fshipped)| ## Mac |Branch|Unit Tests|Functional Tests| |:--:|:--:|:--:| -|**master**|[![Build status](https://dev.azure.com/gvfs/ci/_apis/build/status/CI%20-%20Mac?branchName=master)](https://dev.azure.com/gvfs/ci/_build/latest?definitionId=15)|[![Build status](https://dev.azure.com/mseng/VSOnline/_apis/build/status/GVFS/GitHub%20VFSForGit%20Mac%20Functional%20Tests?branchName=master)](https://dev.azure.com/mseng/VSOnline/_build/latest?definitionId=7350)| -|**shipped**|[![Build status](https://dev.azure.com/gvfs/ci/_apis/build/status/CI%20-%20Mac?branchName=releases%2Fshipped)](https://dev.azure.com/gvfs/ci/_build/latest?definitionId=15)|| +|**master**|[![Build status](https://dev.azure.com/gvfs/ci/_apis/build/status/CI%20-%20Mac?branchName=master)](https://dev.azure.com/gvfs/ci/_build/latest?definitionId=15&branchName=master)|[![Build status](https://dev.azure.com/mseng/VSOnline/_apis/build/status/GVFS/CI%20-%20Mac%20-%20Functional%20Tests?branchName=master)](https://dev.azure.com/mseng/VSOnline/_build/latest?definitionId=7376&branchName=master)| +|**shipped**|[![Build status](https://dev.azure.com/gvfs/ci/_apis/build/status/CI%20-%20Mac?branchName=releases%2Fshipped)](https://dev.azure.com/gvfs/ci/_build/latest?definitionId=15&branchName=releases%2Fshipped)|[![Build status](https://dev.azure.com/mseng/VSOnline/_apis/build/status/GVFS/CI%20-%20Mac%20-%20Functional%20Tests?branchName=releases%2Fshipped)](https://dev.azure.com/mseng/VSOnline/_build/latest?definitionId=7376&branchName=releases%2Fshipped)| ## What is VFS for Git? From 6f17e13c5e1e867ec1e101eed3946c4a6e089a29 Mon Sep 17 00:00:00 2001 From: Saeed Noursalehi Date: Thu, 13 Sep 2018 10:20:50 -0700 Subject: [PATCH 125/272] Re-enable all tests that are failing due to auth cache issues --- GVFS/GVFS.FunctionalTests/Categories.cs | 1 - GVFS/GVFS.FunctionalTests/Program.cs | 1 - .../Tests/EnlistmentPerFixture/MultithreadedReadWriteTests.cs | 3 +-- GVFS/GVFS.FunctionalTests/Tests/GitCommands/CheckoutTests.cs | 4 ---- 4 files changed, 1 insertion(+), 8 deletions(-) diff --git a/GVFS/GVFS.FunctionalTests/Categories.cs b/GVFS/GVFS.FunctionalTests/Categories.cs index 181d671ad..3d65d7d6b 100644 --- a/GVFS/GVFS.FunctionalTests/Categories.cs +++ b/GVFS/GVFS.FunctionalTests/Categories.cs @@ -15,7 +15,6 @@ public static class MacTODO // machines but not on the build agents public const string FailsOnBuildAgent = "FailsOnBuildAgent"; public const string NeedsLockHolder = "NeedsDotCoreLockHolder"; - public const string NeedsCachePoisonFix = "NeedsCachePoisonFix"; public const string M2 = "M2_StaticViewGitCommands"; public const string M3 = "M3_AllGitCommands"; public const string M4 = "M4_All"; diff --git a/GVFS/GVFS.FunctionalTests/Program.cs b/GVFS/GVFS.FunctionalTests/Program.cs index a9cf972c0..b467b28d5 100644 --- a/GVFS/GVFS.FunctionalTests/Program.cs +++ b/GVFS/GVFS.FunctionalTests/Program.cs @@ -67,7 +67,6 @@ public static void Main(string[] args) { excludeCategories.Add(Categories.MacTODO.NeedsLockHolder); excludeCategories.Add(Categories.MacTODO.FailsOnBuildAgent); - excludeCategories.Add(Categories.MacTODO.NeedsCachePoisonFix); excludeCategories.Add(Categories.MacTODO.M2); excludeCategories.Add(Categories.MacTODO.M3); excludeCategories.Add(Categories.MacTODO.M4); diff --git a/GVFS/GVFS.FunctionalTests/Tests/EnlistmentPerFixture/MultithreadedReadWriteTests.cs b/GVFS/GVFS.FunctionalTests/Tests/EnlistmentPerFixture/MultithreadedReadWriteTests.cs index 9846642af..df916dc3c 100644 --- a/GVFS/GVFS.FunctionalTests/Tests/EnlistmentPerFixture/MultithreadedReadWriteTests.cs +++ b/GVFS/GVFS.FunctionalTests/Tests/EnlistmentPerFixture/MultithreadedReadWriteTests.cs @@ -14,7 +14,6 @@ namespace GVFS.FunctionalTests.Tests.EnlistmentPerFixture public class MultithreadedReadWriteTests : TestsWithEnlistmentPerFixture { [TestCase, Order(1)] - [Category(Categories.MacTODO.NeedsCachePoisonFix)] public void CanReadVirtualFileInParallel() { // Note: This test MUST go first, or else it needs to ensure that it is reading a unique path compared to the @@ -27,7 +26,7 @@ public void CanReadVirtualFileInParallel() Exception readException = null; - Thread[] threads = new Thread[32]; + Thread[] threads = new Thread[128]; for (int i = 0; i < threads.Length; ++i) { threads[i] = new Thread(() => diff --git a/GVFS/GVFS.FunctionalTests/Tests/GitCommands/CheckoutTests.cs b/GVFS/GVFS.FunctionalTests/Tests/GitCommands/CheckoutTests.cs index 0fb311fc6..a0d801a83 100644 --- a/GVFS/GVFS.FunctionalTests/Tests/GitCommands/CheckoutTests.cs +++ b/GVFS/GVFS.FunctionalTests/Tests/GitCommands/CheckoutTests.cs @@ -88,7 +88,6 @@ private enum NativeFileAccess : uint } [TestCase] - [Category(Categories.MacTODO.NeedsCachePoisonFix)] public void ReadDeepFilesAfterCheckout() { // In commit 8df701986dea0a5e78b742d2eaf9348825b14d35 the CheckoutNewBranchFromStartingPointTest files were not present @@ -104,7 +103,6 @@ public void ReadDeepFilesAfterCheckout() } [TestCase] - [Category(Categories.MacTODO.NeedsCachePoisonFix)] public void CheckoutNewBranchFromStartingPointTest() { // In commit 8df701986dea0a5e78b742d2eaf9348825b14d35 the CheckoutNewBranchFromStartingPointTest files were not present @@ -121,7 +119,6 @@ public void CheckoutNewBranchFromStartingPointTest() } [TestCase] - [Category(Categories.MacTODO.NeedsCachePoisonFix)] public void CheckoutOrhpanBranchFromStartingPointTest() { // In commit 8df701986dea0a5e78b742d2eaf9348825b14d35 the CheckoutOrhpanBranchFromStartingPointTest files were not present @@ -734,7 +731,6 @@ public void DeleteFolderAndChangeBranchToFolderWithDifferentCase() } [TestCase] - [Category(Categories.MacTODO.NeedsCachePoisonFix)] public void SuccessfullyChecksOutDirectoryToFileToDirectory() { // This test switches between two branches and verifies specific transitions occured From 32e2f37318a3a66a707707d70e9e2afece197b64 Mon Sep 17 00:00:00 2001 From: William Baker Date: Thu, 13 Sep 2018 12:18:54 -0700 Subject: [PATCH 126/272] Changes for PR Feedback --- .../FileSystemRunners/BashRunner.cs | 28 ++++++++++--------- .../FileSystemRunners/PowerShellRunner.cs | 1 + .../FileSystemRunners/SystemIORunner.cs | 4 ++- .../EnlistmentPerFixture/GVFSLockTests.cs | 10 +++---- .../MacFileSystemVirtualizer.cs | 9 +----- .../WindowsFileSystemVirtualizer.cs | 8 +----- 6 files changed, 26 insertions(+), 34 deletions(-) diff --git a/GVFS/GVFS.FunctionalTests/FileSystemRunners/BashRunner.cs b/GVFS/GVFS.FunctionalTests/FileSystemRunners/BashRunner.cs index b848981a3..63dd6824b 100644 --- a/GVFS/GVFS.FunctionalTests/FileSystemRunners/BashRunner.cs +++ b/GVFS/GVFS.FunctionalTests/FileSystemRunners/BashRunner.cs @@ -29,14 +29,12 @@ public class BashRunner : ShellRunner "Function not implemented" }; - private static string[] permissionDeniedWindowsMessage = new string[] + private static string[] windowsPermissionDeniedMessage = new string[] { "Permission denied" }; - // TODO(Mac): Update this message when the kext returns a more accurate - // error code - private static string[] permissionDeniedMacMessage = new string[] + private static string[] macPermissionDeniedMessage = new string[] { "Resource temporarily unavailable" }; @@ -116,7 +114,7 @@ public override void MoveFileShouldFail(string sourcePath, string targetPath) { // BashRunner does nothing special when a failure is expected, so just confirm source file is still present this.MoveFile(sourcePath, targetPath); - this.FileExists(sourcePath).ShouldEqual(true); + this.FileExists(sourcePath).ShouldBeTrue(); } public override void MoveFile_FileShouldNotBeFound(string sourcePath, string targetPath) @@ -254,14 +252,8 @@ public override void DeleteFile_FileShouldNotBeFound(string path) public override void DeleteFile_AccessShouldBeDenied(string path) { - if (RuntimeInformation.IsOSPlatform(OSPlatform.Windows)) - { - this.DeleteFile(path).ShouldContain(permissionDeniedWindowsMessage); - } - else - { - this.DeleteFile(path).ShouldContain(permissionDeniedMacMessage); - } + this.DeleteFile(path).ShouldContain(this.GetPermissionDeniedError()); + this.FileExists(path).ShouldBeTrue($"{path} does not exist when it should"); } public override void ReadAllText_FileShouldNotBeFound(string path) @@ -287,5 +279,15 @@ private string ConvertWinPathToBashPath(string winPath) bashPath = bashPath.Replace('\\', '/'); return bashPath; } + + private string[] GetPermissionDeniedError() + { + if (RuntimeInformation.IsOSPlatform(OSPlatform.Windows)) + { + return windowsPermissionDeniedMessage; + } + + return macPermissionDeniedMessage; + } } } diff --git a/GVFS/GVFS.FunctionalTests/FileSystemRunners/PowerShellRunner.cs b/GVFS/GVFS.FunctionalTests/FileSystemRunners/PowerShellRunner.cs index 9d6a1da97..ce38b6861 100644 --- a/GVFS/GVFS.FunctionalTests/FileSystemRunners/PowerShellRunner.cs +++ b/GVFS/GVFS.FunctionalTests/FileSystemRunners/PowerShellRunner.cs @@ -177,6 +177,7 @@ public override void DeleteFile_FileShouldNotBeFound(string path) public override void DeleteFile_AccessShouldBeDenied(string path) { this.DeleteFile(path).ShouldContain(permissionDeniedMessage); + this.FileExists(path).ShouldBeTrue($"{path} does not exist when it should"); } public override void ReadAllText_FileShouldNotBeFound(string path) diff --git a/GVFS/GVFS.FunctionalTests/FileSystemRunners/SystemIORunner.cs b/GVFS/GVFS.FunctionalTests/FileSystemRunners/SystemIORunner.cs index 8ad8bdffe..096d4ab3f 100644 --- a/GVFS/GVFS.FunctionalTests/FileSystemRunners/SystemIORunner.cs +++ b/GVFS/GVFS.FunctionalTests/FileSystemRunners/SystemIORunner.cs @@ -1,4 +1,5 @@ -using NUnit.Framework; +using GVFS.Tests.Should; +using NUnit.Framework; using System; using System.Diagnostics; using System.IO; @@ -61,6 +62,7 @@ public override void DeleteFile_FileShouldNotBeFound(string path) public override void DeleteFile_AccessShouldBeDenied(string path) { this.ShouldFail(() => { this.DeleteFile(path); }); + this.FileExists(path).ShouldBeTrue($"{path} does not exist when it should"); } public override string ReadAllText(string path) diff --git a/GVFS/GVFS.FunctionalTests/Tests/EnlistmentPerFixture/GVFSLockTests.cs b/GVFS/GVFS.FunctionalTests/Tests/EnlistmentPerFixture/GVFSLockTests.cs index 084e49111..abe17b495 100644 --- a/GVFS/GVFS.FunctionalTests/Tests/EnlistmentPerFixture/GVFSLockTests.cs +++ b/GVFS/GVFS.FunctionalTests/Tests/EnlistmentPerFixture/GVFSLockTests.cs @@ -81,14 +81,14 @@ public void LockPreventsRenameOfIndexLockOnTopOfIndex() this.OverwritingIndexShouldFail(this.Enlistment.GetVirtualPathTo(".git", "index.lock")); } - [DllImport("kernel32.dll", SetLastError = true, CharSet = CharSet.Unicode)] - private static extern bool MoveFileEx( + [DllImport("kernel32.dll", EntryPoint = "MoveFileEx", SetLastError = true, CharSet = CharSet.Unicode)] + private static extern bool WindowsMoveFileEx( string existingFileName, string newFileName, uint flags); [DllImport("libc", EntryPoint = "rename", SetLastError = true)] - private static extern int Rename(string oldPath, string newPath); + private static extern int MacRename(string oldPath, string newPath); private void OverwritingIndexShouldFail(string testFilePath) { @@ -114,14 +114,14 @@ private bool RenameAndOverwrite(string oldPath, string newPath) { if (RuntimeInformation.IsOSPlatform(OSPlatform.Windows)) { - return MoveFileEx( + return WindowsMoveFileEx( oldPath, newPath, (uint)(MoveFileFlags.MoveFileReplaceExisting | MoveFileFlags.MoveFileCopyAllowed)); } else { - return Rename(oldPath, newPath) == 0; + return MacRename(oldPath, newPath) == 0; } } } diff --git a/GVFS/GVFS.Platform.Mac/MacFileSystemVirtualizer.cs b/GVFS/GVFS.Platform.Mac/MacFileSystemVirtualizer.cs index ca7f823f8..3d22f7f34 100644 --- a/GVFS/GVFS.Platform.Mac/MacFileSystemVirtualizer.cs +++ b/GVFS/GVFS.Platform.Mac/MacFileSystemVirtualizer.cs @@ -34,13 +34,7 @@ public MacFileSystemVirtualizer( this.virtualizationInstance = virtualizationInstance ?? new VirtualizationInstance(); } - protected override string EtwArea - { - get - { - return ClassName; - } - } + protected override string EtwArea => ClassName; public static FSResult ResultToFSResult(Result result) { @@ -350,7 +344,6 @@ private Result OnPreDelete(string relativePath, bool isDirectory) metadata.Add(TracingConstants.MessageKey.WarningMessage, "Blocked index delete outside the lock"); this.Context.Tracer.RelatedEvent(EventLevel.Warning, $"{nameof(OnPreDelete)}_BlockedIndexDelete", metadata); - // TODO(Mac): Is this the correct Result to return? return Result.EAccessDenied; } } diff --git a/GVFS/GVFS.Platform.Windows/WindowsFileSystemVirtualizer.cs b/GVFS/GVFS.Platform.Windows/WindowsFileSystemVirtualizer.cs index 6418fda14..42c1f109a 100644 --- a/GVFS/GVFS.Platform.Windows/WindowsFileSystemVirtualizer.cs +++ b/GVFS/GVFS.Platform.Windows/WindowsFileSystemVirtualizer.cs @@ -55,13 +55,7 @@ public WindowsFileSystemVirtualizer( this.activeCommands = new ConcurrentDictionary(); } - protected override string EtwArea - { - get - { - return ClassName; - } - } + protected override string EtwArea => ClassName; /// /// Public for unit testing From a52c411708a0202486acd51bb74041762a692cd5 Mon Sep 17 00:00:00 2001 From: William Baker Date: Thu, 13 Sep 2018 12:20:24 -0700 Subject: [PATCH 127/272] Add message to check in MoveFileShouldFail --- GVFS/GVFS.FunctionalTests/FileSystemRunners/BashRunner.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/GVFS/GVFS.FunctionalTests/FileSystemRunners/BashRunner.cs b/GVFS/GVFS.FunctionalTests/FileSystemRunners/BashRunner.cs index 63dd6824b..95c325876 100644 --- a/GVFS/GVFS.FunctionalTests/FileSystemRunners/BashRunner.cs +++ b/GVFS/GVFS.FunctionalTests/FileSystemRunners/BashRunner.cs @@ -114,7 +114,7 @@ public override void MoveFileShouldFail(string sourcePath, string targetPath) { // BashRunner does nothing special when a failure is expected, so just confirm source file is still present this.MoveFile(sourcePath, targetPath); - this.FileExists(sourcePath).ShouldBeTrue(); + this.FileExists(sourcePath).ShouldBeTrue($"{sourcePath} does not exist when it should"); } public override void MoveFile_FileShouldNotBeFound(string sourcePath, string targetPath) From c05bb8cfe25a1a24622175906e85173f04699263 Mon Sep 17 00:00:00 2001 From: William Baker Date: Thu, 13 Sep 2018 14:25:33 -0700 Subject: [PATCH 128/272] Remove SupportsHardlinkCreation and update SystemIORunner to PInvoke to native hard link API --- .../FileSystemRunners/BashRunner.cs | 5 --- .../FileSystemRunners/CmdRunner.cs | 5 --- .../FileSystemRunners/FileSystemRunner.cs | 11 +------ .../FileSystemRunners/PowerShellRunner.cs | 5 --- .../FileSystemRunners/SystemIORunner.cs | 24 +++++++++++++- .../EnlistmentPerFixture/GitFilesTests.cs | 5 --- .../ModifiedPathsTests.cs | 31 +------------------ .../Tests/GitCommands/AddStageTests.cs | 5 --- .../Tests/GitCommands/GitRepoTests.cs | 10 ++---- 9 files changed, 27 insertions(+), 74 deletions(-) diff --git a/GVFS/GVFS.FunctionalTests/FileSystemRunners/BashRunner.cs b/GVFS/GVFS.FunctionalTests/FileSystemRunners/BashRunner.cs index 40f2fbc12..7da4e8583 100644 --- a/GVFS/GVFS.FunctionalTests/FileSystemRunners/BashRunner.cs +++ b/GVFS/GVFS.FunctionalTests/FileSystemRunners/BashRunner.cs @@ -47,11 +47,6 @@ public BashRunner() } } - public override bool SupportsHardlinkCreation - { - get { return true; } - } - protected override string FileName { get diff --git a/GVFS/GVFS.FunctionalTests/FileSystemRunners/CmdRunner.cs b/GVFS/GVFS.FunctionalTests/FileSystemRunners/CmdRunner.cs index cb5184c06..fd40bef8a 100644 --- a/GVFS/GVFS.FunctionalTests/FileSystemRunners/CmdRunner.cs +++ b/GVFS/GVFS.FunctionalTests/FileSystemRunners/CmdRunner.cs @@ -27,11 +27,6 @@ public class CmdRunner : ShellRunner "The process cannot access the file because it is being used by another process" }; - public override bool SupportsHardlinkCreation - { - get { return true; } - } - protected override string FileName { get diff --git a/GVFS/GVFS.FunctionalTests/FileSystemRunners/FileSystemRunner.cs b/GVFS/GVFS.FunctionalTests/FileSystemRunners/FileSystemRunner.cs index b8a2f9cc6..e064be50a 100644 --- a/GVFS/GVFS.FunctionalTests/FileSystemRunners/FileSystemRunner.cs +++ b/GVFS/GVFS.FunctionalTests/FileSystemRunners/FileSystemRunner.cs @@ -47,11 +47,6 @@ public static FileSystemRunner DefaultRunner get { return defaultRunner; } } - public virtual bool SupportsHardlinkCreation - { - get { return false; } - } - // File methods public abstract bool FileExists(string path); public abstract string MoveFile(string sourcePath, string targetPath); @@ -74,11 +69,7 @@ public virtual bool SupportsHardlinkCreation public abstract void ReadAllText_FileShouldNotBeFound(string path); public abstract void CreateEmptyFile(string path); - - public virtual void CreateHardLink(string newLinkFilePath, string existingFilePath) - { - Assert.Fail($"This runner does not support {nameof(this.CreateHardLink)}"); - } + public abstract void CreateHardLink(string newLinkFilePath, string existingFilePath); /// /// Write the specified contents to the specified file. By calling this method the caller is diff --git a/GVFS/GVFS.FunctionalTests/FileSystemRunners/PowerShellRunner.cs b/GVFS/GVFS.FunctionalTests/FileSystemRunners/PowerShellRunner.cs index 46b86aa40..6c3208b46 100644 --- a/GVFS/GVFS.FunctionalTests/FileSystemRunners/PowerShellRunner.cs +++ b/GVFS/GVFS.FunctionalTests/FileSystemRunners/PowerShellRunner.cs @@ -32,11 +32,6 @@ public class PowerShellRunner : ShellRunner "PermissionDenied" }; - public override bool SupportsHardlinkCreation - { - get { return true; } - } - protected override string FileName { get diff --git a/GVFS/GVFS.FunctionalTests/FileSystemRunners/SystemIORunner.cs b/GVFS/GVFS.FunctionalTests/FileSystemRunners/SystemIORunner.cs index 8ad8bdffe..6d947b510 100644 --- a/GVFS/GVFS.FunctionalTests/FileSystemRunners/SystemIORunner.cs +++ b/GVFS/GVFS.FunctionalTests/FileSystemRunners/SystemIORunner.cs @@ -1,4 +1,5 @@ -using NUnit.Framework; +using GVFS.Tests.Should; +using NUnit.Framework; using System; using System.Diagnostics; using System.IO; @@ -75,6 +76,18 @@ public override void CreateEmptyFile(string path) } } + public override void CreateHardLink(string newLinkFilePath, string existingFilePath) + { + if (RuntimeInformation.IsOSPlatform(OSPlatform.Windows)) + { + WindowsCreateHardLink(newLinkFilePath, existingFilePath, IntPtr.Zero).ShouldBeTrue($"Failed to create hard link: {Marshal.GetLastWin32Error()}"); + } + else + { + MacCreateHardLink(existingFilePath, newLinkFilePath).ShouldEqual(0, $"Failed to create hard link: {Marshal.GetLastWin32Error()}"); + } + } + public override void WriteAllText(string path, string contents) { File.WriteAllText(path, contents); @@ -178,6 +191,15 @@ public override void ReadAllText_FileShouldNotBeFound(string path) [DllImport("kernel32", SetLastError = true)] private static extern bool MoveFileEx(string existingFileName, string newFileName, int flags); + [DllImport("libc", EntryPoint = "link", SetLastError = true)] + private static extern int MacCreateHardLink(string oldPath, string newPath); + + [DllImport("kernel32.dll", EntryPoint = "CreateHardLink", SetLastError = true, CharSet = CharSet.Unicode)] + private static extern bool WindowsCreateHardLink( + string newLinkFileName, + string existingFileName, + IntPtr securityAttributes); + private static void RetryOnException(Action action) { for (int i = 0; i < 10; i++) diff --git a/GVFS/GVFS.FunctionalTests/Tests/EnlistmentPerFixture/GitFilesTests.cs b/GVFS/GVFS.FunctionalTests/Tests/EnlistmentPerFixture/GitFilesTests.cs index d1b56fc48..55961a00c 100644 --- a/GVFS/GVFS.FunctionalTests/Tests/EnlistmentPerFixture/GitFilesTests.cs +++ b/GVFS/GVFS.FunctionalTests/Tests/EnlistmentPerFixture/GitFilesTests.cs @@ -42,11 +42,6 @@ public void CreateFileTest() [TestCase, Order(2)] public void CreateHardLinkTest() { - if (!this.fileSystem.SupportsHardlinkCreation) - { - return; - } - string existingFileName = "fileToLinkTo.txt"; string existingFilePath = this.Enlistment.GetVirtualPathTo(existingFileName); GVFSHelpers.ModifiedPathsShouldNotContain(this.fileSystem, this.Enlistment.DotGVFSRoot, existingFileName); diff --git a/GVFS/GVFS.FunctionalTests/Tests/EnlistmentPerTestCase/ModifiedPathsTests.cs b/GVFS/GVFS.FunctionalTests/Tests/EnlistmentPerTestCase/ModifiedPathsTests.cs index ba72a6a09..0dc57253c 100644 --- a/GVFS/GVFS.FunctionalTests/Tests/EnlistmentPerTestCase/ModifiedPathsTests.cs +++ b/GVFS/GVFS.FunctionalTests/Tests/EnlistmentPerTestCase/ModifiedPathsTests.cs @@ -120,7 +120,7 @@ public void ModifiedPathsSavedAfterRemount(FileSystemRunner fileSystem) } } - [TestCaseSource(typeof(HardLinkRunners), HardLinkRunners.TestRunners)] + [TestCaseSource(typeof(FileSystemRunner), FileSystemRunner.TestRunners)] public void ModifiedPathsCorrectAfterHardLinking(FileSystemRunner fileSystem) { const string ExpectedModifiedFilesContentsAfterHardlinks = @@ -162,34 +162,5 @@ A LinkToFileOutsideSrc.txt reader.ReadToEnd().ShouldEqual(ExpectedModifiedFilesContentsAfterHardlinks); } } - - private class HardLinkRunners - { - public const string TestRunners = "Runners"; - - public static object[] Runners - { - get - { - List hardLinkRunners = new List(); - foreach (object[] runner in FileSystemRunner.Runners.ToList()) - { - FileSystemRunner fileSystem = runner.ToList().First() as FileSystemRunner; - if (fileSystem.SupportsHardlinkCreation) - { - hardLinkRunners.Add(new object[] { fileSystem }); - } - } - - if (hardLinkRunners.Count > 0) - { - return hardLinkRunners.ToArray(); - } - - // Always return at least one runner that supports creating hard links - return new[] { new object[] { new BashRunner() } }; - } - } - } } } diff --git a/GVFS/GVFS.FunctionalTests/Tests/GitCommands/AddStageTests.cs b/GVFS/GVFS.FunctionalTests/Tests/GitCommands/AddStageTests.cs index e8a36a23c..f0d4305f6 100644 --- a/GVFS/GVFS.FunctionalTests/Tests/GitCommands/AddStageTests.cs +++ b/GVFS/GVFS.FunctionalTests/Tests/GitCommands/AddStageTests.cs @@ -32,11 +32,6 @@ public void StageBasicTest() [TestCase, Order(3)] public void AddAndStageHardLinksTest() { - if (!this.FileSystem.SupportsHardlinkCreation) - { - return; - } - this.CreateHardLink("ReadmeLink.md", "Readme.md"); this.ValidateGitCommand("add ReadmeLink.md"); this.RunGitCommand("commit -m \"Created ReadmeLink.md\""); diff --git a/GVFS/GVFS.FunctionalTests/Tests/GitCommands/GitRepoTests.cs b/GVFS/GVFS.FunctionalTests/Tests/GitCommands/GitRepoTests.cs index 932ee1eaa..9200be7c5 100644 --- a/GVFS/GVFS.FunctionalTests/Tests/GitCommands/GitRepoTests.cs +++ b/GVFS/GVFS.FunctionalTests/Tests/GitCommands/GitRepoTests.cs @@ -224,14 +224,8 @@ protected void CreateHardLink(string newLinkFileName, string existingFileName) string virtualNewLinkFile = Path.Combine(this.Enlistment.RepoRoot, newLinkFileName); string controlNewLinkFile = Path.Combine(this.ControlGitRepo.RootPath, newLinkFileName); - // GitRepoTests are only run with SystemIORunner (which does not support hardlink - // creation) so use a BashRunner instead. - this.FileSystem.SupportsHardlinkCreation.ShouldBeFalse( - "If this.FileSystem.SupportsHardlinkCreation is true, CreateHardLink no longer needs to create a BashRunner"); - FileSystemRunner runner = new BashRunner(); - - runner.CreateHardLink(virtualNewLinkFile, virtualExistingFile); - runner.CreateHardLink(controlNewLinkFile, controlExistingFile); + this.FileSystem.CreateHardLink(virtualNewLinkFile, virtualExistingFile); + this.FileSystem.CreateHardLink(controlNewLinkFile, controlExistingFile); } protected void SetFileAsReadOnly(string filePath) From bff758ce9a3219fa5b975c1bd4f036ce39c402e6 Mon Sep 17 00:00:00 2001 From: William Baker Date: Thu, 13 Sep 2018 14:29:33 -0700 Subject: [PATCH 129/272] Remove unused using statements --- .../Tests/EnlistmentPerTestCase/ModifiedPathsTests.cs | 2 -- 1 file changed, 2 deletions(-) diff --git a/GVFS/GVFS.FunctionalTests/Tests/EnlistmentPerTestCase/ModifiedPathsTests.cs b/GVFS/GVFS.FunctionalTests/Tests/EnlistmentPerTestCase/ModifiedPathsTests.cs index 0dc57253c..5348ce268 100644 --- a/GVFS/GVFS.FunctionalTests/Tests/EnlistmentPerTestCase/ModifiedPathsTests.cs +++ b/GVFS/GVFS.FunctionalTests/Tests/EnlistmentPerTestCase/ModifiedPathsTests.cs @@ -3,9 +3,7 @@ using GVFS.FunctionalTests.Tools; using GVFS.Tests.Should; using NUnit.Framework; -using System.Collections.Generic; using System.IO; -using System.Linq; namespace GVFS.FunctionalTests.Tests.EnlistmentPerTestCase { From 1b5bdb69f6d59e53cee48a2649983af6ea912f89 Mon Sep 17 00:00:00 2001 From: Saeed Noursalehi Date: Thu, 13 Sep 2018 15:32:07 -0700 Subject: [PATCH 130/272] Set the auth cache TTL to 0 when registering a virtualization root --- ProjFS.Mac/PrjFSKext/PrjFSKext/VirtualizationRoots.cpp | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/ProjFS.Mac/PrjFSKext/PrjFSKext/VirtualizationRoots.cpp b/ProjFS.Mac/PrjFSKext/PrjFSKext/VirtualizationRoots.cpp index d37281361..443a8d30f 100644 --- a/ProjFS.Mac/PrjFSKext/PrjFSKext/VirtualizationRoots.cpp +++ b/ProjFS.Mac/PrjFSKext/PrjFSKext/VirtualizationRoots.cpp @@ -105,7 +105,6 @@ int16_t VirtualizationRoots_LookupVnode(vnode_t vnode, vfs_context_t context) { // TODO: check xattr contents - char path[PrjFSMaxPath] = ""; int pathLength = sizeof(path); vn_getpath(vnode, path, &pathLength); @@ -283,6 +282,12 @@ VirtualizationRootResult VirtualizationRoot_RegisterProviderForPath(PrjFSProvide vnode_put(virtualizationRootVNode); } + if (rootIndex >= 0) + { + VirtualizationRoot* root = &s_virtualizationRoots[rootIndex]; + vfs_setauthcache_ttl(vnode_mount(root->rootVNode), 0); + } + vfs_context_rele(vfsContext); return VirtualizationRootResult { err, rootIndex }; From efa70da9cf608c75a8b9c1ec284c46762b164eb4 Mon Sep 17 00:00:00 2001 From: William Baker Date: Thu, 13 Sep 2018 16:47:59 -0700 Subject: [PATCH 131/272] Fix AddStageTests on Windows --- GVFS/GVFS.FunctionalTests/Tests/GitCommands/AddStageTests.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/GVFS/GVFS.FunctionalTests/Tests/GitCommands/AddStageTests.cs b/GVFS/GVFS.FunctionalTests/Tests/GitCommands/AddStageTests.cs index f0d4305f6..d4e834dad 100644 --- a/GVFS/GVFS.FunctionalTests/Tests/GitCommands/AddStageTests.cs +++ b/GVFS/GVFS.FunctionalTests/Tests/GitCommands/AddStageTests.cs @@ -56,7 +56,7 @@ public void StageAllowsPlaceholderCreation() private void CommandAllowsPlaceholderCreation(string command, params string[] fileToReadPathParts) { string fileToRead = Path.Combine(fileToReadPathParts); - this.EditFile($"Some new content for {command}.", "Readme.md"); + this.EditFile($"Some new content for {command}.", "Protocol.md"); ManualResetEventSlim resetEvent = GitHelpers.RunGitCommandWithWaitAndStdIn(this.Enlistment, resetTimeout: 3000, command: $"{command} -p", stdinToQuit: "q", processId: out _); this.FileContentsShouldMatch(fileToRead); this.ValidateGitCommand("--no-optional-locks status"); From 749d821e6765228bcb4ea9d867120ce4d8ed3999 Mon Sep 17 00:00:00 2001 From: William Baker Date: Thu, 13 Sep 2018 08:48:05 -0700 Subject: [PATCH 132/272] Recursively expand directories before rename and enable folder rename functional tests --- .../MoveRenameFolderTests.cs | 52 ++++++++++--- .../WorkingDirectoryTests.cs | 4 +- .../PrjFSKext/PrjFSKext/KauthHandler.cpp | 58 +++++++-------- ProjFS.Mac/PrjFSKext/public/Message.h | 1 + ProjFS.Mac/PrjFSLib/PrjFSLib.cpp | 74 ++++++++++++++++++- 5 files changed, 142 insertions(+), 47 deletions(-) diff --git a/GVFS/GVFS.FunctionalTests/Tests/EnlistmentPerFixture/MoveRenameFolderTests.cs b/GVFS/GVFS.FunctionalTests/Tests/EnlistmentPerFixture/MoveRenameFolderTests.cs index 7e5c70ba5..2f0071a04 100644 --- a/GVFS/GVFS.FunctionalTests/Tests/EnlistmentPerFixture/MoveRenameFolderTests.cs +++ b/GVFS/GVFS.FunctionalTests/Tests/EnlistmentPerFixture/MoveRenameFolderTests.cs @@ -1,5 +1,6 @@ using GVFS.FunctionalTests.FileSystemRunners; using GVFS.FunctionalTests.Should; +using GVFS.Tests.Should; using NUnit.Framework; using System.IO; @@ -58,10 +59,8 @@ public void RenameFolderShouldFail() this.Enlistment.GetVirtualPathTo(Path.Combine(oldFolderName, testFileName)).ShouldBeAFile(this.fileSystem).WithContents(TestFileContents); } - // This test requires expansion on PreDelete to be implemented // MacOnly because renames of partial folders are blocked on Windows [TestCase] - [Category(Categories.MacTODO.M2)] [Category(Categories.MacOnly)] public void ChangeUnhydratedFolderName() { @@ -79,10 +78,8 @@ public void ChangeUnhydratedFolderName() this.Enlistment.GetVirtualPathTo(Path.Combine(newFolderName, testFileName)).ShouldBeAFile(this.fileSystem).WithContents(TestFileContents); } - // This test requires expansion on PreDelete to be implemented // MacOnly because renames of partial folders are blocked on Windows [TestCase] - [Category(Categories.MacTODO.M2)] [Category(Categories.MacOnly)] public void MoveUnhydratedFolderToNewFolder() { @@ -104,10 +101,8 @@ public void MoveUnhydratedFolderToNewFolder() this.Enlistment.GetVirtualPathTo(Path.Combine(movedFolderPath, testFileName)).ShouldBeAFile(this.fileSystem).WithContents(TestFileContents); } - // This test requires expansion on PreDelete to be implemented // MacOnly because renames of partial folders are blocked on Windows [TestCase] - [Category(Categories.MacTODO.M2)] [Category(Categories.MacOnly)] public void MoveUnhydratedFolderToFullFolderInDotGitFolder() { @@ -156,10 +151,8 @@ public void MoveFullFolderToFullFolderInDotGitFolder() Path.Combine(movedFolderPath, testFileName).ShouldBeAFile(this.fileSystem).WithContents(fileContents); } - // This test requires expansion on PreDelete to be implemented // MacOnly because renames of partial folders are blocked on Windows [TestCase] - [Category(Categories.MacTODO.M2)] [Category(Categories.MacOnly)] public void MoveAndRenameUnhydratedFolderToNewFolder() { @@ -181,14 +174,12 @@ public void MoveAndRenameUnhydratedFolderToNewFolder() this.Enlistment.GetVirtualPathTo(Path.Combine(movedFolderPath, testFileName)).ShouldBeAFile(this.fileSystem).WithContents(TestFileContents); } - // This test requires expansion on PreDelete to be implemented // MacOnly because renames of partial folders are blocked on Windows [TestCase] - [Category(Categories.MacTODO.M2)] [Category(Categories.MacOnly)] public void MoveFolderWithUnhydratedAndFullContents() { - string testFileName = "MoveFolderWithUnhydratedAndFullContents.cs"; + string testFileName = "MoveFolderWithUnhydratedAndFullContents.cpp"; string oldFolderName = Path.Combine("Test_EPF_MoveRenameFolderTests", "MoveFolderWithUnhydratedAndFullContents"); string newFile = "TestFile.txt"; @@ -214,5 +205,44 @@ public void MoveFolderWithUnhydratedAndFullContents() this.Enlistment.GetVirtualPathTo(Path.Combine(oldFolderName, newFile)).ShouldNotExistOnDisk(this.fileSystem); this.Enlistment.GetVirtualPathTo(Path.Combine(movedFolderPath, newFile)).ShouldBeAFile(this.fileSystem).WithContents(newFileContents); } + + // MacOnly because renames of partial folders are blocked on Windows + [TestCase] + [Category(Categories.MacOnly)] + public void MoveFolderWithUnexpandedChildFolders() + { + string oldFolderPath = this.Enlistment.GetVirtualPathTo("Test_EPF_MoveRenameFileTests"); + string newFolderName = "Test_EPF_MoveRenameFileTests_Renamed"; + string newFolderPath = this.Enlistment.GetVirtualPathTo(newFolderName); + this.fileSystem.MoveDirectory(oldFolderPath, newFolderPath); + oldFolderPath.ShouldNotExistOnDisk(this.fileSystem); + + newFolderPath.ShouldBeADirectory(this.fileSystem); + this.Enlistment.GetVirtualPathTo(newFolderName, "ChangeNestedUnhydratedFileNameCase", "Program.cs") + .ShouldBeAFile(this.fileSystem) + .WithContents(MoveRenameFileTests.TestFileContents); + + this.Enlistment.GetVirtualPathTo(newFolderName, "ChangeUnhydratedFileName", "Program.cs") + .ShouldBeAFile(this.fileSystem) + .WithContents(MoveRenameFileTests.TestFileContents); + + this.Enlistment.GetVirtualPathTo(newFolderName, "MoveUnhydratedFileToDotGitFolder", "Program.cs") + .ShouldBeAFile(this.fileSystem) + .WithContents(MoveRenameFileTests.TestFileContents); + + // Test moving a folder with a very deep unhydrated child tree + oldFolderPath = this.Enlistment.GetVirtualPathTo("Test_EPF_WorkingDirectoryTests"); + + // But expand the folder we will be renaming (so that only the children have not been expanded) + oldFolderPath.ShouldBeADirectory(this.fileSystem).WithDirectories().ShouldContain(dir => dir.Name.Equals("1")); + + newFolderName = "Test_EPF_WorkingDirectoryTests_Renamed"; + newFolderPath = this.Enlistment.GetVirtualPathTo(newFolderName); + this.fileSystem.MoveDirectory(oldFolderPath, newFolderPath); + oldFolderPath.ShouldNotExistOnDisk(this.fileSystem); + this.Enlistment.GetVirtualPathTo(newFolderName, "1", "2", "3", "4", "ReadDeepProjectedFile.cpp") + .ShouldBeAFile(this.fileSystem) + .WithContents(WorkingDirectoryTests.TestFileContents); + } } } diff --git a/GVFS/GVFS.FunctionalTests/Tests/EnlistmentPerFixture/WorkingDirectoryTests.cs b/GVFS/GVFS.FunctionalTests/Tests/EnlistmentPerFixture/WorkingDirectoryTests.cs index d5088205e..019043462 100644 --- a/GVFS/GVFS.FunctionalTests/Tests/EnlistmentPerFixture/WorkingDirectoryTests.cs +++ b/GVFS/GVFS.FunctionalTests/Tests/EnlistmentPerFixture/WorkingDirectoryTests.cs @@ -16,8 +16,7 @@ namespace GVFS.FunctionalTests.Tests.EnlistmentPerFixture [TestFixtureSource(typeof(FileSystemRunner), FileSystemRunner.TestRunners)] public class WorkingDirectoryTests : TestsWithEnlistmentPerFixture { - private const int CurrentPlaceholderVersion = 1; - private const string TestFileContents = + public const string TestFileContents = @"// dllmain.cpp : Defines the entry point for the DLL application. #include ""stdafx.h"" @@ -41,6 +40,7 @@ LPVOID lpReserved } "; + private const int CurrentPlaceholderVersion = 1; private FileSystemRunner fileSystem; diff --git a/ProjFS.Mac/PrjFSKext/PrjFSKext/KauthHandler.cpp b/ProjFS.Mac/PrjFSKext/PrjFSKext/KauthHandler.cpp index acce65bce..ab53da185 100644 --- a/ProjFS.Mac/PrjFSKext/PrjFSKext/KauthHandler.cpp +++ b/ProjFS.Mac/PrjFSKext/PrjFSKext/KauthHandler.cpp @@ -222,6 +222,7 @@ void KauthHandler_HandleKernelMessageResponse(uint64_t messageId, MessageType re case MessageType_UtoK_StartVirtualizationInstance: case MessageType_UtoK_StopVirtualizationInstance: case MessageType_KtoU_EnumerateDirectory: + case MessageType_KtoU_RecursivelyEnumerateDirectory: case MessageType_KtoU_HydrateFile: case MessageType_KtoU_NotifyFileModified: case MessageType_KtoU_NotifyFilePreDelete: @@ -276,9 +277,27 @@ static int HandleVnodeOperation( goto CleanupAndReturn; } + if (ActionBitIsSet(action, KAUTH_VNODE_DELETE)) + { + if (!TrySendRequestAndWaitForResponse( + root, + VDIR == vnodeType ? MessageType_KtoU_NotifyDirectoryPreDelete : MessageType_KtoU_NotifyFilePreDelete, + currentVnode, + pid, + procname, + &kauthResult, + kauthError)) + { + goto CleanupAndReturn; + } + } + if (VDIR == vnodeType) { - if (ActionBitIsSet( + bool deleteAction = ActionBitIsSet(action, KAUTH_VNODE_DELETE); + + if (deleteAction || + ActionBitIsSet( action, KAUTH_VNODE_LIST_DIRECTORY | KAUTH_VNODE_SEARCH | @@ -286,11 +305,14 @@ static int HandleVnodeOperation( KAUTH_VNODE_READ_ATTRIBUTES | KAUTH_VNODE_READ_EXTATTRIBUTES)) { - if (FileFlagsBitIsSet(currentVnodeFileFlags, FileFlags_IsEmpty)) + // Recursively expand directory on delete to ensure child placeholders are created before rename operations + if (deleteAction || FileFlagsBitIsSet(currentVnodeFileFlags, FileFlags_IsEmpty)) { if (!TrySendRequestAndWaitForResponse( root, - MessageType_KtoU_EnumerateDirectory, + deleteAction ? + MessageType_KtoU_RecursivelyEnumerateDirectory : + MessageType_KtoU_EnumerateDirectory, currentVnode, pid, procname, @@ -301,39 +323,9 @@ static int HandleVnodeOperation( } } } - - if (ActionBitIsSet(action, KAUTH_VNODE_DELETE)) - { - if (!TrySendRequestAndWaitForResponse( - root, - MessageType_KtoU_NotifyDirectoryPreDelete, - currentVnode, - pid, - procname, - &kauthResult, - kauthError)) - { - goto CleanupAndReturn; - } - } } else { - if (ActionBitIsSet(action, KAUTH_VNODE_DELETE)) - { - if (!TrySendRequestAndWaitForResponse( - root, - MessageType_KtoU_NotifyFilePreDelete, - currentVnode, - pid, - procname, - &kauthResult, - kauthError)) - { - goto CleanupAndReturn; - } - } - if (ActionBitIsSet( action, KAUTH_VNODE_READ_ATTRIBUTES | diff --git a/ProjFS.Mac/PrjFSKext/public/Message.h b/ProjFS.Mac/PrjFSKext/public/Message.h index a4a200fb8..43a99ab06 100644 --- a/ProjFS.Mac/PrjFSKext/public/Message.h +++ b/ProjFS.Mac/PrjFSKext/public/Message.h @@ -14,6 +14,7 @@ typedef enum // Messages from kernel to user mode MessageType_KtoU_EnumerateDirectory, + MessageType_KtoU_RecursivelyEnumerateDirectory, MessageType_KtoU_HydrateFile, MessageType_KtoU_NotifyFileModified, diff --git a/ProjFS.Mac/PrjFSLib/PrjFSLib.cpp b/ProjFS.Mac/PrjFSLib/PrjFSLib.cpp index a858a7fe3..5fb1afec6 100644 --- a/ProjFS.Mac/PrjFSLib/PrjFSLib.cpp +++ b/ProjFS.Mac/PrjFSLib/PrjFSLib.cpp @@ -7,6 +7,8 @@ #include #include #include +#include +#include #include #include #include @@ -54,6 +56,7 @@ static errno_t RegisterVirtualizationRootPath(const char* path); static void HandleKernelRequest(Message requestSpec, void* messageMemory); static PrjFS_Result HandleEnumerateDirectoryRequest(const MessageHeader* request, const char* path); +static PrjFS_Result HandleRecursivelyEnumerateDirectoryRequest(const MessageHeader* request, const char* path); static PrjFS_Result HandleHydrateFileRequest(const MessageHeader* request, const char* path); static PrjFS_Result HandleFileNotification( const MessageHeader* request, @@ -491,6 +494,12 @@ static void HandleKernelRequest(Message request, void* messageMemory) result = HandleEnumerateDirectoryRequest(requestHeader, request.path); break; } + + case MessageType_KtoU_RecursivelyEnumerateDirectory: + { + result = HandleRecursivelyEnumerateDirectoryRequest(requestHeader, request.path); + break; + } case MessageType_KtoU_HydrateFile: { @@ -579,7 +588,7 @@ static PrjFS_Result HandleEnumerateDirectoryRequest(const MessageHeader* request if (!SetBitInFileFlags(fullPath, FileFlags_IsEmpty, false)) { - // TODO: how should we handle this scenario where the provider thinks it succeeded, but we were unable to + // TODO(Mac): how should we handle this scenario where the provider thinks it succeeded, but we were unable to // update placeholder metadata? return PrjFS_Result_EIOError; } @@ -588,6 +597,68 @@ static PrjFS_Result HandleEnumerateDirectoryRequest(const MessageHeader* request return callbackResult; } +static PrjFS_Result HandleRecursivelyEnumerateDirectoryRequest(const MessageHeader* request, const char* path) +{ +#ifdef DEBUG + std::cout << "PrjFSLib.HandleRecursivelyEnumerateDirectoryRequest: " << path << std::endl; +#endif + + DIR* directory = nullptr; + PrjFS_Result result = PrjFS_Result_Success; + std::list directoryRelativePaths; + directoryRelativePaths.push_back(path); + + // Walk each directory, expanding those that are found to be empty + char pathBuffer[PrjFSMaxPath]; + std::list::iterator relativePathIter = directoryRelativePaths.begin(); + while(relativePathIter != directoryRelativePaths.end()) + { + CombinePaths(s_virtualizationRootFullPath.c_str(), relativePathIter->c_str(), pathBuffer); + + // TODO(Mac): how should we handle scenarios where we were unable to fully expand the directory and + // its children? + if (IsBitSetInFileFlags(pathBuffer, FileFlags_IsEmpty)) + { + PrjFS_Result result = HandleEnumerateDirectoryRequest(request, relativePathIter->c_str()); + if (result != PrjFS_Result_Success) + { + goto CleanupAndReturn; + } + } + + DIR* directory = opendir(pathBuffer); + if (nullptr == directory) + { + result = PrjFS_Result_EIOError; + goto CleanupAndReturn; + } + + dirent* dirEntry = readdir(directory); + while(dirEntry != nullptr) + { + if (dirEntry->d_type == DT_DIR && + 0 != strncmp(".", dirEntry->d_name, sizeof(dirEntry->d_name)) && + 0 != strncmp("..", dirEntry->d_name, sizeof(dirEntry->d_name))) + { + CombinePaths(relativePathIter->c_str(), dirEntry->d_name, pathBuffer); + directoryRelativePaths.push_back(pathBuffer); + } + + dirEntry = readdir(directory); + } + + ++relativePathIter; + } + +CleanupAndReturn: + if (directory != nullptr) + { + closedir(directory); + } + + return result; +} + static PrjFS_Result HandleHydrateFileRequest(const MessageHeader* request, const char* path) { #ifdef DEBUG @@ -815,6 +886,7 @@ static inline PrjFS_NotificationType KUMessageTypeToNotificationType(MessageType case MessageType_UtoK_StartVirtualizationInstance: case MessageType_UtoK_StopVirtualizationInstance: case MessageType_KtoU_EnumerateDirectory: + case MessageType_KtoU_RecursivelyEnumerateDirectory: case MessageType_KtoU_HydrateFile: case MessageType_Response_Success: case MessageType_Response_Fail: From 22a176f5d12b16bb765146b1093c2947c6abfd56 Mon Sep 17 00:00:00 2001 From: Kevin Willford Date: Fri, 14 Sep 2018 14:38:34 -0600 Subject: [PATCH 133/272] Fix tests that still need a sparse-checkout to test upgrades --- .../Windows/Tests/DiskLayoutUpgradeTests.cs | 11 +++++++++++ .../Windows/Tests/SharedCacheUpgradeTests.cs | 10 ++++++++++ 2 files changed, 21 insertions(+) diff --git a/GVFS/GVFS.FunctionalTests.Windows/Windows/Tests/DiskLayoutUpgradeTests.cs b/GVFS/GVFS.FunctionalTests.Windows/Windows/Tests/DiskLayoutUpgradeTests.cs index 1abbbaafb..d3ea1f93b 100644 --- a/GVFS/GVFS.FunctionalTests.Windows/Windows/Tests/DiskLayoutUpgradeTests.cs +++ b/GVFS/GVFS.FunctionalTests.Windows/Windows/Tests/DiskLayoutUpgradeTests.cs @@ -27,6 +27,17 @@ public class DiskLayoutUpgradeTests : TestsWithEnlistmentPerTestCase private FileSystemRunner fileSystem = new SystemIORunner(); + [SetUp] + public override void CreateEnlistment() + { + base.CreateEnlistment(); + + // Since there isn't a sparse-checkout file that is used anymore one needs to be added + // in order to test the old upgrades that might have needed it + string sparseCheckoutPath = Path.Combine(this.Enlistment.RepoRoot, TestConstants.DotGit.Info.SparseCheckoutPath); + this.fileSystem.WriteAllText(sparseCheckoutPath, "/.gitattributes\r\n"); + } + [TestCase] public void MountUpgradesFromVersion7() { diff --git a/GVFS/GVFS.FunctionalTests.Windows/Windows/Tests/SharedCacheUpgradeTests.cs b/GVFS/GVFS.FunctionalTests.Windows/Windows/Tests/SharedCacheUpgradeTests.cs index f113b0012..5d6acf516 100644 --- a/GVFS/GVFS.FunctionalTests.Windows/Windows/Tests/SharedCacheUpgradeTests.cs +++ b/GVFS/GVFS.FunctionalTests.Windows/Windows/Tests/SharedCacheUpgradeTests.cs @@ -48,6 +48,11 @@ public void MountUpgradesLocalSizesToSharedCache() versionJsonPath.ShouldBeAFile(this.fileSystem); this.fileSystem.DeleteFile(versionJsonPath); + // Since there isn't a sparse-checkout file that is used anymore one needs to be added + // in order to test the old upgrades that might have needed it + string sparseCheckoutPath = Path.Combine(enlistment.RepoRoot, TestConstants.DotGit.Info.SparseCheckoutPath); + this.fileSystem.WriteAllText(sparseCheckoutPath, "/.gitattributes\r\n"); + // "13.0" was the last version before blob sizes were moved out of Esent string metadataPath = Path.Combine(enlistment.DotGVFSRoot, GVFSHelpers.RepoMetadataName); this.fileSystem.CreateEmptyFile(metadataPath); @@ -103,6 +108,11 @@ public void MountUpgradesLocalSizesToSharedCache() versionJsonPath.ShouldBeAFile(this.fileSystem); this.fileSystem.DeleteFile(versionJsonPath); + // Since there isn't a sparse-checkout file that is used anymore one needs to be added + // in order to test the old upgrades that might have needed it + string sparseCheckoutPath2 = Path.Combine(enlistment2.RepoRoot, TestConstants.DotGit.Info.SparseCheckoutPath); + this.fileSystem.WriteAllText(sparseCheckoutPath2, "/.gitattributes\r\n"); + // "13.0" was the last version before blob sizes were moved out of Esent metadataPath = Path.Combine(enlistment2.DotGVFSRoot, GVFSHelpers.RepoMetadataName); this.fileSystem.CreateEmptyFile(metadataPath); From cba03055c6a931ddf3133888f51f5042feaf67f3 Mon Sep 17 00:00:00 2001 From: Kevin Willford Date: Fri, 14 Sep 2018 14:39:52 -0600 Subject: [PATCH 134/272] Remove setting skip worktree bit --- GVFS/GVFS.Common/Prefetch/Git/GitIndexGenerator.cs | 11 ++--------- 1 file changed, 2 insertions(+), 9 deletions(-) diff --git a/GVFS/GVFS.Common/Prefetch/Git/GitIndexGenerator.cs b/GVFS/GVFS.Common/Prefetch/Git/GitIndexGenerator.cs index 98f0ac577..66ae2d3d9 100644 --- a/GVFS/GVFS.Common/Prefetch/Git/GitIndexGenerator.cs +++ b/GVFS/GVFS.Common/Prefetch/Git/GitIndexGenerator.cs @@ -109,8 +109,7 @@ private void WriteAllEntries(uint version) LsTreeEntry entry; while (this.entryQueue.TryTake(out entry, Timeout.Infinite)) { - bool skipWorkTree = false; - this.WriteEntry(writer, version, entry.Sha, entry.Filename, skipWorkTree, ref lastStringLength); + this.WriteEntry(writer, version, entry.Sha, entry.Filename, ref lastStringLength); } // Update entry count @@ -140,7 +139,7 @@ private string GetDirectoryNameForGitPath(string filename) return filename.Substring(0, idx + 1); } - private void WriteEntry(BinaryWriter writer, uint version, string sha, string filename, bool skipWorktree, ref uint lastStringLength) + private void WriteEntry(BinaryWriter writer, uint version, string sha, string filename, ref uint lastStringLength) { long startPosition = writer.BaseStream.Position; @@ -153,14 +152,8 @@ private void WriteEntry(BinaryWriter writer, uint version, string sha, string fi byte[] filenameBytes = Encoding.UTF8.GetBytes(filename); ushort flags = (ushort)(filenameBytes.Length & 0xFFF); - flags |= version >= 3 && skipWorktree ? ExtendedBit : (ushort)0; writer.Write(EndianHelper.Swap(flags)); - if (version >= 3 && skipWorktree) - { - writer.Write(EndianHelper.Swap(SkipWorktreeBit)); - } - if (version >= 4) { this.WriteReplaceLength(writer, lastStringLength); From bc6f8962f4b638b7ca4114c4cf892757d78fe663 Mon Sep 17 00:00:00 2001 From: William Baker Date: Fri, 14 Sep 2018 08:23:58 -0700 Subject: [PATCH 135/272] Temporary fix: look up vnode path earlier in callback --- .../PrjFSKext/PrjFSKext/KauthHandler.cpp | 31 ++++++++++++++----- 1 file changed, 24 insertions(+), 7 deletions(-) diff --git a/ProjFS.Mac/PrjFSKext/PrjFSKext/KauthHandler.cpp b/ProjFS.Mac/PrjFSKext/PrjFSKext/KauthHandler.cpp index acce65bce..3fbf03713 100644 --- a/ProjFS.Mac/PrjFSKext/PrjFSKext/KauthHandler.cpp +++ b/ProjFS.Mac/PrjFSKext/PrjFSKext/KauthHandler.cpp @@ -48,6 +48,7 @@ static bool TrySendRequestAndWaitForResponse( const VirtualizationRoot* root, MessageType messageType, const vnode_t vnode, + const char* vnodePath, int pid, const char* procname, int* kauthResult, @@ -253,14 +254,23 @@ static int HandleVnodeOperation( vnode_t currentVnode = reinterpret_cast(arg1); // arg2 is the (vnode_t) parent vnode int* kauthError = reinterpret_cast(arg3); + int kauthResult = KAUTH_RESULT_DEFER; + + const char* vnodePath = nullptr; + char vnodePathBuffer[PrjFSMaxPath]; + int vnodePathLength = PrjFSMaxPath; VirtualizationRoot* root = nullptr; vtype vnodeType; uint32_t currentVnodeFileFlags; int pid; char procname[MAXCOMLEN + 1]; - - int kauthResult = KAUTH_RESULT_DEFER; + + // TODO(Mac): Issue #271 - Reduce reliance on vn_getpath + if (0 == vn_getpath(currentVnode, vnodePathBuffer, &vnodePathLength)) + { + vnodePath = vnodePathBuffer; + } if (!ShouldHandleVnodeOpEvent( context, @@ -292,6 +302,7 @@ static int HandleVnodeOperation( root, MessageType_KtoU_EnumerateDirectory, currentVnode, + vnodePath, pid, procname, &kauthResult, @@ -308,6 +319,7 @@ static int HandleVnodeOperation( root, MessageType_KtoU_NotifyDirectoryPreDelete, currentVnode, + vnodePath, pid, procname, &kauthResult, @@ -325,6 +337,7 @@ static int HandleVnodeOperation( root, MessageType_KtoU_NotifyFilePreDelete, currentVnode, + vnodePath, pid, procname, &kauthResult, @@ -351,6 +364,7 @@ static int HandleVnodeOperation( root, MessageType_KtoU_HydrateFile, currentVnode, + vnodePath, pid, procname, &kauthResult, @@ -428,6 +442,7 @@ static int HandleFileOpOperation( root, messageType, currentVnodeFromPath, + newPath, pid, procname, &kauthResult, @@ -439,7 +454,7 @@ static int HandleFileOpOperation( else if (KAUTH_FILEOP_CLOSE == action) { vnode_t currentVnode = reinterpret_cast(arg0); - // arg1 is the (const char *) path + const char* path = (const char*)arg1; int closeFlags = static_cast(arg2); if (vnode_isdir(currentVnode)) @@ -476,6 +491,7 @@ static int HandleFileOpOperation( root, MessageType_KtoU_NotifyFileModified, currentVnode, + path, pid, procname, &kauthResult, @@ -492,6 +508,7 @@ static int HandleFileOpOperation( root, MessageType_KtoU_NotifyFileCreated, currentVnode, + path, pid, procname, &kauthResult, @@ -647,6 +664,7 @@ static bool TrySendRequestAndWaitForResponse( const VirtualizationRoot* root, MessageType messageType, const vnode_t vnode, + const char* vnodePath, int pid, const char* procname, int* kauthResult, @@ -657,11 +675,10 @@ static bool TrySendRequestAndWaitForResponse( OutstandingMessage message; message.receivedResponse = false; - char vnodePath[PrjFSMaxPath]; - int vnodePathLength = PrjFSMaxPath; - if (vn_getpath(vnode, vnodePath, &vnodePathLength)) + if (nullptr == vnodePath) { - KextLog_Error("Unable to resolve a vnode to its path"); + // Default error code is EACCES. See errno.h for more codes. + *kauthError = EAGAIN; *kauthResult = KAUTH_RESULT_DENY; return false; } From 1d0beff80599463ee944bd42a518494e04ee292d Mon Sep 17 00:00:00 2001 From: William Baker Date: Mon, 17 Sep 2018 08:40:03 -0700 Subject: [PATCH 136/272] Add comment --- ProjFS.Mac/PrjFSKext/PrjFSKext/KauthHandler.cpp | 2 ++ 1 file changed, 2 insertions(+) diff --git a/ProjFS.Mac/PrjFSKext/PrjFSKext/KauthHandler.cpp b/ProjFS.Mac/PrjFSKext/PrjFSKext/KauthHandler.cpp index 3fbf03713..6868047a9 100644 --- a/ProjFS.Mac/PrjFSKext/PrjFSKext/KauthHandler.cpp +++ b/ProjFS.Mac/PrjFSKext/PrjFSKext/KauthHandler.cpp @@ -267,6 +267,8 @@ static int HandleVnodeOperation( char procname[MAXCOMLEN + 1]; // TODO(Mac): Issue #271 - Reduce reliance on vn_getpath + // Call vn_getpath first when the cache is hottest to increase the chances + // of successfully getting the path if (0 == vn_getpath(currentVnode, vnodePathBuffer, &vnodePathLength)) { vnodePath = vnodePathBuffer; From 89cf95b718816394f9eb2d5e55156dbab004a732 Mon Sep 17 00:00:00 2001 From: William Baker Date: Tue, 18 Sep 2018 13:53:06 -0700 Subject: [PATCH 137/272] Make LaunchPrefetchJobIfIdle thread safe --- .../Prefetch/BackgroundPrefetcher.cs | 53 ++++++++++++------- .../Prefetch/BackgroundPrefetcherTests.cs | 44 +++++++++++++++ .../GVFS.UnitTests/Virtual/CommonRepoSetup.cs | 7 +-- 3 files changed, 83 insertions(+), 21 deletions(-) diff --git a/GVFS/GVFS.Common/Prefetch/BackgroundPrefetcher.cs b/GVFS/GVFS.Common/Prefetch/BackgroundPrefetcher.cs index e9927393a..fd2fa1987 100644 --- a/GVFS/GVFS.Common/Prefetch/BackgroundPrefetcher.cs +++ b/GVFS/GVFS.Common/Prefetch/BackgroundPrefetcher.cs @@ -12,6 +12,7 @@ public class BackgroundPrefetcher : IDisposable private const string TelemetryKey = nameof(BackgroundPrefetcher); private readonly TimeSpan timerPeriod = TimeSpan.FromMinutes(15); private readonly TimeSpan timeBetweenPrefetches = TimeSpan.FromMinutes(70); + private readonly object launchPrefetchLock = new object(); private ITracer tracer; private GVFSEnlistment enlistment; @@ -48,19 +49,30 @@ public void Dispose() public bool LaunchPrefetchJobIfIdle() { - if (this.prefetchJobThread?.IsAlive == true) + try { - this.tracer.RelatedInfo(nameof(BackgroundPrefetcher) + ": background thread not idle, skipping timed start"); + lock (this.launchPrefetchLock) + { + if (this.prefetchJobThread?.IsAlive == true) + { + this.tracer.RelatedInfo(nameof(BackgroundPrefetcher) + ": background thread not idle, skipping timed start"); + } + else + { + this.prefetchJobThread = new Thread(() => this.BackgroundPrefetch()); + this.prefetchJobThread.IsBackground = true; + this.prefetchJobThread.Start(); + return true; + } + + return false; + } } - else + catch (Exception e) { - this.prefetchJobThread = new Thread(() => this.BackgroundPrefetch()); - this.prefetchJobThread.IsBackground = true; - this.prefetchJobThread.Start(); - return true; + this.LogUnhandledExceptionAndExit(nameof(this.LaunchPrefetchJobIfIdle), e); + return false; } - - return false; } /// @@ -119,16 +131,21 @@ private void BackgroundPrefetch() } catch (Exception e) { - EventMetadata metadata = new EventMetadata(); - metadata.Add("Method", nameof(this.BackgroundPrefetch)); - metadata.Add("ExceptionMessage", e.Message); - metadata.Add("StackTrace", e.StackTrace); - this.tracer.RelatedError( - metadata: metadata, - message: TelemetryKey + ": Unexpected Exception while running prefetch background thread (fatal): " + e.Message, - keywords: Keywords.Telemetry); - Environment.Exit((int)ReturnCode.GenericError); + this.LogUnhandledExceptionAndExit(nameof(this.BackgroundPrefetch), e); } } + + private void LogUnhandledExceptionAndExit(string methodName, Exception e) + { + EventMetadata metadata = new EventMetadata(); + metadata.Add("Method", methodName); + metadata.Add("ExceptionMessage", e.Message); + metadata.Add("StackTrace", e.StackTrace); + this.tracer.RelatedError( + metadata: metadata, + message: TelemetryKey + ": Unexpected Exception while running prefetch background thread (fatal): " + e.Message, + keywords: Keywords.Telemetry); + Environment.Exit((int)ReturnCode.GenericError); + } } } diff --git a/GVFS/GVFS.UnitTests/Prefetch/BackgroundPrefetcherTests.cs b/GVFS/GVFS.UnitTests/Prefetch/BackgroundPrefetcherTests.cs index 667376fbe..e5e829e39 100644 --- a/GVFS/GVFS.UnitTests/Prefetch/BackgroundPrefetcherTests.cs +++ b/GVFS/GVFS.UnitTests/Prefetch/BackgroundPrefetcherTests.cs @@ -1,7 +1,9 @@ using GVFS.Common.Prefetch; using GVFS.Tests.Should; +using GVFS.UnitTests.Mock.FileSystem; using GVFS.UnitTests.Virtual; using NUnit.Framework; +using System.Threading; namespace GVFS.UnitTests.Prefetch { @@ -32,5 +34,47 @@ public void RestartBackgroundJobSucceeds() prefetcher.WaitForPrefetchToFinish(); } } + + [TestCase] + public void LaunchPrefetchJobIfIdleDoesNotLaunchSecondThreadIfFirstInProgress() + { + using (CommonRepoSetup setup = new CommonRepoSetup()) + { + BlockedDirectoryExistsFileSystem fileSystem = new BlockedDirectoryExistsFileSystem(setup.FileSystem.RootDirectory); + using (BackgroundPrefetcher prefetcher = new BackgroundPrefetcher( + setup.Context.Tracer, + setup.Context.Enlistment, + fileSystem, + setup.GitObjects)) + { + prefetcher.LaunchPrefetchJobIfIdle().ShouldBeTrue(); + prefetcher.LaunchPrefetchJobIfIdle().ShouldBeFalse(); + fileSystem.UnblockDirectoryExists(); + prefetcher.WaitForPrefetchToFinish(); + } + } + } + + private class BlockedDirectoryExistsFileSystem : MockFileSystem + { + private ManualResetEvent unblockDirectoryExists; + + public BlockedDirectoryExistsFileSystem(MockDirectory rootDirectory) + : base(rootDirectory) + { + this.unblockDirectoryExists = new ManualResetEvent(initialState: false); + } + + public void UnblockDirectoryExists() + { + this.unblockDirectoryExists.Set(); + } + + public override bool DirectoryExists(string path) + { + this.unblockDirectoryExists.WaitOne(); + return base.DirectoryExists(path); + } + } } } diff --git a/GVFS/GVFS.UnitTests/Virtual/CommonRepoSetup.cs b/GVFS/GVFS.UnitTests/Virtual/CommonRepoSetup.cs index 9594d9da0..ab9731039 100644 --- a/GVFS/GVFS.UnitTests/Virtual/CommonRepoSetup.cs +++ b/GVFS/GVFS.UnitTests/Virtual/CommonRepoSetup.cs @@ -34,14 +34,14 @@ public CommonRepoSetup() enlistmentDirectory.CreateFile(Path.Combine(this.GitParentPath, ".git", "info", "always_exclude"), "always_exclude Contents", createDirectories: true); enlistmentDirectory.CreateDirectory(enlistment.GitPackRoot); - MockFileSystem fileSystem = new MockFileSystem(enlistmentDirectory); + this.FileSystem = new MockFileSystem(enlistmentDirectory); this.Repository = new MockGitRepo( tracer, enlistment, - fileSystem); + this.FileSystem); CreateStandardGitTree(this.Repository); - this.Context = new GVFSContext(tracer, fileSystem, this.Repository, enlistment); + this.Context = new GVFSContext(tracer, this.FileSystem, this.Repository, enlistment); this.HttpObjects = new MockHttpGitObjects(tracer, enlistment); this.GitObjects = new MockGVFSGitObjects(this.Context, this.HttpObjects); @@ -56,6 +56,7 @@ public CommonRepoSetup() public MockGitRepo Repository { get; private set; } public MockHttpGitObjects HttpObjects { get; private set; } + public MockFileSystem FileSystem { get; private set; } public void Dispose() { From 4509af5f935878fbe6dbea4758c1d634aef3251d Mon Sep 17 00:00:00 2001 From: William Baker Date: Tue, 18 Sep 2018 14:36:50 -0700 Subject: [PATCH 138/272] Fix unit test bug --- .../Prefetch/BackgroundPrefetcherTests.cs | 22 +++++++++---------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/GVFS/GVFS.UnitTests/Prefetch/BackgroundPrefetcherTests.cs b/GVFS/GVFS.UnitTests/Prefetch/BackgroundPrefetcherTests.cs index e5e829e39..9d3f838a2 100644 --- a/GVFS/GVFS.UnitTests/Prefetch/BackgroundPrefetcherTests.cs +++ b/GVFS/GVFS.UnitTests/Prefetch/BackgroundPrefetcherTests.cs @@ -40,7 +40,7 @@ public void LaunchPrefetchJobIfIdleDoesNotLaunchSecondThreadIfFirstInProgress() { using (CommonRepoSetup setup = new CommonRepoSetup()) { - BlockedDirectoryExistsFileSystem fileSystem = new BlockedDirectoryExistsFileSystem(setup.FileSystem.RootDirectory); + BlockedCreateDirectoryFileSystem fileSystem = new BlockedCreateDirectoryFileSystem(setup.FileSystem.RootDirectory); using (BackgroundPrefetcher prefetcher = new BackgroundPrefetcher( setup.Context.Tracer, setup.Context.Enlistment, @@ -49,31 +49,31 @@ public void LaunchPrefetchJobIfIdleDoesNotLaunchSecondThreadIfFirstInProgress() { prefetcher.LaunchPrefetchJobIfIdle().ShouldBeTrue(); prefetcher.LaunchPrefetchJobIfIdle().ShouldBeFalse(); - fileSystem.UnblockDirectoryExists(); + fileSystem.UnblockCreateDirectory(); prefetcher.WaitForPrefetchToFinish(); } } } - private class BlockedDirectoryExistsFileSystem : MockFileSystem + private class BlockedCreateDirectoryFileSystem : MockFileSystem { - private ManualResetEvent unblockDirectoryExists; + private ManualResetEvent unblockCreateDirectory; - public BlockedDirectoryExistsFileSystem(MockDirectory rootDirectory) + public BlockedCreateDirectoryFileSystem(MockDirectory rootDirectory) : base(rootDirectory) { - this.unblockDirectoryExists = new ManualResetEvent(initialState: false); + this.unblockCreateDirectory = new ManualResetEvent(initialState: false); } - public void UnblockDirectoryExists() + public void UnblockCreateDirectory() { - this.unblockDirectoryExists.Set(); + this.unblockCreateDirectory.Set(); } - public override bool DirectoryExists(string path) + public override void CreateDirectory(string path) { - this.unblockDirectoryExists.WaitOne(); - return base.DirectoryExists(path); + this.unblockCreateDirectory.WaitOne(); + base.CreateDirectory(path); } } } From 2c2d116711e613752d556378ecd3f9b3d5ca03e7 Mon Sep 17 00:00:00 2001 From: William Baker Date: Tue, 18 Sep 2018 15:19:38 -0700 Subject: [PATCH 139/272] Catch and log exceptions thrown by ConvertDirectoryToVirtualizationRoot --- GVFS/GVFS.Common/FileSystem/IKernelDriver.cs | 3 +- GVFS/GVFS.Platform.Mac/ProjFSKext.cs | 3 +- GVFS/GVFS.Platform.Windows/ProjFSFilter.cs | 35 +++++++++++++++++--- GVFS/GVFS/CommandLine/CloneVerb.cs | 12 +++++-- GVFS/GVFS/CommandLine/DehydrateVerb.cs | 11 +++++- 5 files changed, 55 insertions(+), 9 deletions(-) diff --git a/GVFS/GVFS.Common/FileSystem/IKernelDriver.cs b/GVFS/GVFS.Common/FileSystem/IKernelDriver.cs index 270019da4..35fbdd1c1 100644 --- a/GVFS/GVFS.Common/FileSystem/IKernelDriver.cs +++ b/GVFS/GVFS.Common/FileSystem/IKernelDriver.cs @@ -1,4 +1,5 @@ using GVFS.Common.Tracing; +using System; namespace GVFS.Common.FileSystem { @@ -8,7 +9,7 @@ public interface IKernelDriver string DriverLogFolderName { get; } bool IsSupported(string normalizedEnlistmentRootPath, out string warning, out string error); string FlushDriverLogs(); - bool TryPrepareFolderForCallbacks(string folderPath, out string error); + bool TryPrepareFolderForCallbacks(string folderPath, out string error, out Exception exception); bool IsReady(JsonTracer tracer, string enlistmentRoot, out string error); } } diff --git a/GVFS/GVFS.Platform.Mac/ProjFSKext.cs b/GVFS/GVFS.Platform.Mac/ProjFSKext.cs index 9adc1d68c..ede5623fc 100644 --- a/GVFS/GVFS.Platform.Mac/ProjFSKext.cs +++ b/GVFS/GVFS.Platform.Mac/ProjFSKext.cs @@ -46,8 +46,9 @@ public bool IsReady(JsonTracer tracer, string enlistmentRoot, out string error) return true; } - public bool TryPrepareFolderForCallbacks(string folderPath, out string error) + public bool TryPrepareFolderForCallbacks(string folderPath, out string error, out Exception exception) { + exception = null; error = string.Empty; Result result = VirtualizationInstance.ConvertDirectoryToVirtualizationRoot(folderPath); if (result != Result.Success) diff --git a/GVFS/GVFS.Platform.Windows/ProjFSFilter.cs b/GVFS/GVFS.Platform.Windows/ProjFSFilter.cs index 544b5678a..7df193d96 100644 --- a/GVFS/GVFS.Platform.Windows/ProjFSFilter.cs +++ b/GVFS/GVFS.Platform.Windows/ProjFSFilter.cs @@ -30,6 +30,7 @@ public class ProjFSFilter : IKernelDriver private const string FilterLoggerSessionName = "Microsoft-Windows-ProjFS-Filter-Log"; private const string ProjFSNativeLibFileName = "ProjectedFSLib.dll"; + private const string ProjFSManagedLibFileName = "ProjectedFSLib.Managed.dll"; private const uint OkResult = 0; private const uint NameCollisionErrorResult = 0x801F0012; @@ -320,14 +321,40 @@ public string FlushDriverLogs() return sb.ToString(); } - public bool TryPrepareFolderForCallbacks(string folderPath, out string error) + public bool TryPrepareFolderForCallbacks(string folderPath, out string error, out Exception exception) { + exception = null; error = string.Empty; Guid virtualizationInstanceGuid = Guid.NewGuid(); - HResult result = VirtualizationInstance.ConvertDirectoryToVirtualizationRoot(virtualizationInstanceGuid, folderPath); - if (result != HResult.Ok) + + try + { + HResult result = VirtualizationInstance.ConvertDirectoryToVirtualizationRoot(virtualizationInstanceGuid, folderPath); + if (result != HResult.Ok) + { + error = "Failed to prepare \"" + folderPath + "\" for callbacks, error: " + result.ToString("F"); + return false; + } + } + catch (FileNotFoundException e) + { + exception = e; + + if (e.FileName.Equals(ProjFSManagedLibFileName, StringComparison.OrdinalIgnoreCase)) + { + error = $"Failed to load {ProjFSManagedLibFileName}. Ensure that ProjFS is installed and enabled"; + } + else + { + error = $"FileNotFoundException while trying to prepare \"{folderPath}\" for callbacks: {e.Message}"; + } + + return false; + } + catch (Exception e) { - error = "Failed to prepare \"" + folderPath + "\" for callbacks, error: " + result.ToString("F"); + exception = e; + error = $"Exception while trying to prepare \"{folderPath}\" for callbacks: {e.Message}"; return false; } diff --git a/GVFS/GVFS/CommandLine/CloneVerb.cs b/GVFS/GVFS/CommandLine/CloneVerb.cs index 00a429c07..5d86b909f 100644 --- a/GVFS/GVFS/CommandLine/CloneVerb.cs +++ b/GVFS/GVFS/CommandLine/CloneVerb.cs @@ -609,10 +609,18 @@ private Result CreateClone( } // Prepare the working directory folder for GVFS last to ensure that gvfs mount will fail if gvfs clone has failed + Exception exception; string prepFileSystemError; - if (!GVFSPlatform.Instance.KernelDriver.TryPrepareFolderForCallbacks(enlistment.WorkingDirectoryRoot, out prepFileSystemError)) + if (!GVFSPlatform.Instance.KernelDriver.TryPrepareFolderForCallbacks(enlistment.WorkingDirectoryRoot, out prepFileSystemError, out exception)) { - tracer.RelatedError(prepFileSystemError); + EventMetadata metadata = new EventMetadata(); + metadata.Add(nameof(prepFileSystemError), prepFileSystemError); + if (exception != null) + { + metadata.Add("Exception", exception.ToString()); + } + + tracer.RelatedError(metadata, $"{nameof(this.CreateClone)}: TryPrepareFolderForCallbacks failed"); return new Result(prepFileSystemError); } diff --git a/GVFS/GVFS/CommandLine/DehydrateVerb.cs b/GVFS/GVFS/CommandLine/DehydrateVerb.cs index 71342283a..906745f3c 100644 --- a/GVFS/GVFS/CommandLine/DehydrateVerb.cs +++ b/GVFS/GVFS/CommandLine/DehydrateVerb.cs @@ -221,9 +221,18 @@ private void Mount(ITracer tracer) private void PrepareSrcFolder(ITracer tracer, GVFSEnlistment enlistment) { + Exception exception; string error; - if (!GVFSPlatform.Instance.KernelDriver.TryPrepareFolderForCallbacks(enlistment.WorkingDirectoryRoot, out error)) + if (!GVFSPlatform.Instance.KernelDriver.TryPrepareFolderForCallbacks(enlistment.WorkingDirectoryRoot, out error, out exception)) { + EventMetadata metadata = new EventMetadata(); + metadata.Add(nameof(error), error); + if (exception != null) + { + metadata.Add("Exception", exception.ToString()); + } + + tracer.RelatedError(metadata, $"{nameof(this.PrepareSrcFolder)}: TryPrepareFolderForCallbacks failed"); this.ReportErrorAndExit(tracer, "Failed to recreate the virtualization root: " + error); } } From f2bda4254a57e3218a0dc1c74da6f8543a76a96e Mon Sep 17 00:00:00 2001 From: William Baker Date: Tue, 18 Sep 2018 15:53:38 -0700 Subject: [PATCH 140/272] Catch exceptions one level higher --- GVFS/GVFS.Common/FileSystem/IKernelDriver.cs | 3 +- GVFS/GVFS.Platform.Mac/ProjFSKext.cs | 3 +- GVFS/GVFS.Platform.Windows/ProjFSFilter.cs | 35 +++----------------- GVFS/GVFS/CommandLine/CloneVerb.cs | 27 +++++++++------ GVFS/GVFS/CommandLine/DehydrateVerb.cs | 22 +++++++----- 5 files changed, 36 insertions(+), 54 deletions(-) diff --git a/GVFS/GVFS.Common/FileSystem/IKernelDriver.cs b/GVFS/GVFS.Common/FileSystem/IKernelDriver.cs index 35fbdd1c1..270019da4 100644 --- a/GVFS/GVFS.Common/FileSystem/IKernelDriver.cs +++ b/GVFS/GVFS.Common/FileSystem/IKernelDriver.cs @@ -1,5 +1,4 @@ using GVFS.Common.Tracing; -using System; namespace GVFS.Common.FileSystem { @@ -9,7 +8,7 @@ public interface IKernelDriver string DriverLogFolderName { get; } bool IsSupported(string normalizedEnlistmentRootPath, out string warning, out string error); string FlushDriverLogs(); - bool TryPrepareFolderForCallbacks(string folderPath, out string error, out Exception exception); + bool TryPrepareFolderForCallbacks(string folderPath, out string error); bool IsReady(JsonTracer tracer, string enlistmentRoot, out string error); } } diff --git a/GVFS/GVFS.Platform.Mac/ProjFSKext.cs b/GVFS/GVFS.Platform.Mac/ProjFSKext.cs index ede5623fc..9adc1d68c 100644 --- a/GVFS/GVFS.Platform.Mac/ProjFSKext.cs +++ b/GVFS/GVFS.Platform.Mac/ProjFSKext.cs @@ -46,9 +46,8 @@ public bool IsReady(JsonTracer tracer, string enlistmentRoot, out string error) return true; } - public bool TryPrepareFolderForCallbacks(string folderPath, out string error, out Exception exception) + public bool TryPrepareFolderForCallbacks(string folderPath, out string error) { - exception = null; error = string.Empty; Result result = VirtualizationInstance.ConvertDirectoryToVirtualizationRoot(folderPath); if (result != Result.Success) diff --git a/GVFS/GVFS.Platform.Windows/ProjFSFilter.cs b/GVFS/GVFS.Platform.Windows/ProjFSFilter.cs index 7df193d96..544b5678a 100644 --- a/GVFS/GVFS.Platform.Windows/ProjFSFilter.cs +++ b/GVFS/GVFS.Platform.Windows/ProjFSFilter.cs @@ -30,7 +30,6 @@ public class ProjFSFilter : IKernelDriver private const string FilterLoggerSessionName = "Microsoft-Windows-ProjFS-Filter-Log"; private const string ProjFSNativeLibFileName = "ProjectedFSLib.dll"; - private const string ProjFSManagedLibFileName = "ProjectedFSLib.Managed.dll"; private const uint OkResult = 0; private const uint NameCollisionErrorResult = 0x801F0012; @@ -321,40 +320,14 @@ public string FlushDriverLogs() return sb.ToString(); } - public bool TryPrepareFolderForCallbacks(string folderPath, out string error, out Exception exception) + public bool TryPrepareFolderForCallbacks(string folderPath, out string error) { - exception = null; error = string.Empty; Guid virtualizationInstanceGuid = Guid.NewGuid(); - - try - { - HResult result = VirtualizationInstance.ConvertDirectoryToVirtualizationRoot(virtualizationInstanceGuid, folderPath); - if (result != HResult.Ok) - { - error = "Failed to prepare \"" + folderPath + "\" for callbacks, error: " + result.ToString("F"); - return false; - } - } - catch (FileNotFoundException e) - { - exception = e; - - if (e.FileName.Equals(ProjFSManagedLibFileName, StringComparison.OrdinalIgnoreCase)) - { - error = $"Failed to load {ProjFSManagedLibFileName}. Ensure that ProjFS is installed and enabled"; - } - else - { - error = $"FileNotFoundException while trying to prepare \"{folderPath}\" for callbacks: {e.Message}"; - } - - return false; - } - catch (Exception e) + HResult result = VirtualizationInstance.ConvertDirectoryToVirtualizationRoot(virtualizationInstanceGuid, folderPath); + if (result != HResult.Ok) { - exception = e; - error = $"Exception while trying to prepare \"{folderPath}\" for callbacks: {e.Message}"; + error = "Failed to prepare \"" + folderPath + "\" for callbacks, error: " + result.ToString("F"); return false; } diff --git a/GVFS/GVFS/CommandLine/CloneVerb.cs b/GVFS/GVFS/CommandLine/CloneVerb.cs index 5d86b909f..6cf233ba2 100644 --- a/GVFS/GVFS/CommandLine/CloneVerb.cs +++ b/GVFS/GVFS/CommandLine/CloneVerb.cs @@ -609,22 +609,29 @@ private Result CreateClone( } // Prepare the working directory folder for GVFS last to ensure that gvfs mount will fail if gvfs clone has failed - Exception exception; - string prepFileSystemError; - if (!GVFSPlatform.Instance.KernelDriver.TryPrepareFolderForCallbacks(enlistment.WorkingDirectoryRoot, out prepFileSystemError, out exception)) + Result prepForCallbacksResult = new Result(true); + try { - EventMetadata metadata = new EventMetadata(); - metadata.Add(nameof(prepFileSystemError), prepFileSystemError); - if (exception != null) + string prepFileSystemError; + if (!GVFSPlatform.Instance.KernelDriver.TryPrepareFolderForCallbacks(enlistment.WorkingDirectoryRoot, out prepFileSystemError)) { - metadata.Add("Exception", exception.ToString()); + prepForCallbacksResult = new Result(prepFileSystemError); } - + } + catch (Exception e) + { + EventMetadata metadata = new EventMetadata(); + metadata.Add("Exception", e.ToString()); tracer.RelatedError(metadata, $"{nameof(this.CreateClone)}: TryPrepareFolderForCallbacks failed"); - return new Result(prepFileSystemError); + prepForCallbacksResult = new Result($"Failed to prepare \"{enlistment.WorkingDirectoryRoot}\" for callbacks, exception: {e.Message}"); } - return new Result(true); + if (!prepForCallbacksResult.Success) + { + tracer.RelatedError($"TryPrepareFolderForCallbacks failed, error: {prepForCallbacksResult.ErrorMessage}"); + } + + return prepForCallbacksResult; } private void CreateGitScript(GVFSEnlistment enlistment) diff --git a/GVFS/GVFS/CommandLine/DehydrateVerb.cs b/GVFS/GVFS/CommandLine/DehydrateVerb.cs index 906745f3c..a82e7cd13 100644 --- a/GVFS/GVFS/CommandLine/DehydrateVerb.cs +++ b/GVFS/GVFS/CommandLine/DehydrateVerb.cs @@ -221,19 +221,23 @@ private void Mount(ITracer tracer) private void PrepareSrcFolder(ITracer tracer, GVFSEnlistment enlistment) { - Exception exception; - string error; - if (!GVFSPlatform.Instance.KernelDriver.TryPrepareFolderForCallbacks(enlistment.WorkingDirectoryRoot, out error, out exception)) + try { - EventMetadata metadata = new EventMetadata(); - metadata.Add(nameof(error), error); - if (exception != null) + string error; + if (!GVFSPlatform.Instance.KernelDriver.TryPrepareFolderForCallbacks(enlistment.WorkingDirectoryRoot, out error)) { - metadata.Add("Exception", exception.ToString()); + EventMetadata metadata = new EventMetadata(); + metadata.Add(nameof(error), error); + tracer.RelatedError(metadata, $"{nameof(this.PrepareSrcFolder)}: TryPrepareFolderForCallbacks failed"); + this.ReportErrorAndExit(tracer, "Failed to recreate the virtualization root: " + error); } - + } + catch (Exception e) + { + EventMetadata metadata = new EventMetadata(); + metadata.Add("Exception", e.ToString()); tracer.RelatedError(metadata, $"{nameof(this.PrepareSrcFolder)}: TryPrepareFolderForCallbacks failed"); - this.ReportErrorAndExit(tracer, "Failed to recreate the virtualization root: " + error); + this.ReportErrorAndExit(tracer, "Failed to recreate the virtualization root: " + e.Message); } } From 6bc7066de5e3e7a609eb7f5c1f68bbb1aa286f22 Mon Sep 17 00:00:00 2001 From: Nick Graczyk Date: Thu, 20 Sep 2018 15:02:26 -0700 Subject: [PATCH 141/272] Handle read only files when hydrating with the mirror provider by only requesting Read access --- MirrorProvider/MirrorProvider/FileSystemVirtualizer.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/MirrorProvider/MirrorProvider/FileSystemVirtualizer.cs b/MirrorProvider/MirrorProvider/FileSystemVirtualizer.cs index 07ddf3955..40efddbe9 100644 --- a/MirrorProvider/MirrorProvider/FileSystemVirtualizer.cs +++ b/MirrorProvider/MirrorProvider/FileSystemVirtualizer.cs @@ -81,7 +81,7 @@ protected FileSystemResult HydrateFile(string relativePath, int bufferSize, Func return FileSystemResult.EFileNotFound; } - using (FileStream fs = new FileStream(fullPathInMirror, FileMode.Open)) + using (FileStream fs = new FileStream(fullPathInMirror, FileMode.Open, FileAccess.Read)) { long remainingData = fs.Length; byte[] buffer = new byte[bufferSize]; From 3aac4138b4c6e08b86ae736fa501f42f30d5b19c Mon Sep 17 00:00:00 2001 From: Jameson Miller Date: Tue, 11 Sep 2018 11:33:30 -0400 Subject: [PATCH 142/272] Tests demonstrating issue with newline in Git commands --- .../Tests/GitCommands/GitCommandsTests.cs | 11 +++++++++++ .../Tests/GitCommands/UpdateIndexTests.cs | 11 +++++++++++ 2 files changed, 22 insertions(+) diff --git a/GVFS/GVFS.FunctionalTests/Tests/GitCommands/GitCommandsTests.cs b/GVFS/GVFS.FunctionalTests/Tests/GitCommands/GitCommandsTests.cs index f079e5464..f7f5a6d47 100644 --- a/GVFS/GVFS.FunctionalTests/Tests/GitCommands/GitCommandsTests.cs +++ b/GVFS/GVFS.FunctionalTests/Tests/GitCommands/GitCommandsTests.cs @@ -381,6 +381,17 @@ public void AddFileInSubfolderAndCommitOnNewBranchSwitchDeleteFolderAndSwitchBac this.FileShouldHaveContents(newFileContents, newFilePath); } + [TestCase] + [Ignore("Currently failing regression test for how GVFS handles newlines in git command line")] + public void CommitWithNewlinesInMessage() + { + this.ValidateGitCommand("checkout -b tests/functional/commit_with_uncommon_arguments"); + this.CreateFile(); + this.ValidateGitCommand("status"); + this.ValidateGitCommand("add ."); + this.RunGitCommand("commit -m \"Message that contains \na\nnew\nline\""); + } + [TestCase] public void CaseOnlyRenameFileAndChangeBranches() { diff --git a/GVFS/GVFS.FunctionalTests/Tests/GitCommands/UpdateIndexTests.cs b/GVFS/GVFS.FunctionalTests/Tests/GitCommands/UpdateIndexTests.cs index 5ca28eaa4..df6871f24 100644 --- a/GVFS/GVFS.FunctionalTests/Tests/GitCommands/UpdateIndexTests.cs +++ b/GVFS/GVFS.FunctionalTests/Tests/GitCommands/UpdateIndexTests.cs @@ -63,6 +63,17 @@ public void UpdateIndexRemoveAddFileOpenForWrite() GitHelpers.InvokeGitAgainstGVFSRepo(this.Enlistment.RepoRoot, "update-index --add Test_ConflictTests/AddedFiles/AddedByBothDifferentContent.txt"); } + [TestCase] + [Category(Categories.MacTODO.M4)] + [Ignore("Currently failing regression test for how GVFS handles newlines in git command line")] + public void UpdateIndexWithCacheInfo() + { + // Update Protocol.md with the contents from blob 583f1... + string command = $"update-index --cacheinfo 100644 \"583f1a56db7cc884d54534c5d9c56b93a1e00a2b\n\" Protocol.md"; + + this.ValidateGitCommand(command); + } + protected override void CreateEnlistment() { base.CreateEnlistment(); From fb44c71f86bfb1304bed0321930f76be11651f2b Mon Sep 17 00:00:00 2001 From: Jameson Miller Date: Fri, 7 Sep 2018 10:37:22 -0400 Subject: [PATCH 143/272] Tweak the protocol GVFS used to communicate over named pipes GVFS currently sends a text based data across a named pipe with lines / messages separated by newline characters. This prohibits sending messages that include a newline as part of the message content. The main current manifestation of this problem is that GVFS does not handle git command lines that contain a newline character. There is another problem that GVFS needs to send paths across the named pipe (for ModifiedPathsList queries), and these paths can include newline characters on certain filesystems (macOS, Linux). Additionally, the protocol that GVFS uses to communicate across the named pipe is currently not consistent. It usually uses a text based stream, but for the ModifiedPathsList response, it uses null bytes to seperate entries in a list. To address these issues, this change tweaks the protocol used to communicate via the named pipe to use the 0x3 byte to indicate the end of a line / message (this is the End of text ASCII code). --- .../GVFS.Common/NamedPipes/NamedPipeClient.cs | 13 ++- .../GVFS.Common/NamedPipes/NamedPipeServer.cs | 45 +++++---- .../NamedPipes/NamedPipeStreamReader.cs | 92 +++++++++++++++++++ .../NamedPipes/NamedPipeStreamWriter.cs | 24 +++++ .../NamedPipes/StreamWriterExtensions.cs | 16 ---- .../Tests/GitCommands/GitCommandsTests.cs | 1 - .../Tests/GitCommands/UpdateIndexTests.cs | 1 - GVFS/GVFS.Hooks/GVFS.Hooks.Mac.csproj | 7 +- GVFS/GVFS.Hooks/GVFS.Hooks.Windows.csproj | 7 +- GVFS/GVFS.ReadObjectHook/main.cpp | 6 +- GVFS/GVFS.VirtualFileSystemHook/main.cpp | 5 +- 11 files changed, 164 insertions(+), 53 deletions(-) create mode 100644 GVFS/GVFS.Common/NamedPipes/NamedPipeStreamReader.cs create mode 100644 GVFS/GVFS.Common/NamedPipes/NamedPipeStreamWriter.cs delete mode 100644 GVFS/GVFS.Common/NamedPipes/StreamWriterExtensions.cs diff --git a/GVFS/GVFS.Common/NamedPipes/NamedPipeClient.cs b/GVFS/GVFS.Common/NamedPipes/NamedPipeClient.cs index 69e27b111..18745788f 100644 --- a/GVFS/GVFS.Common/NamedPipes/NamedPipeClient.cs +++ b/GVFS/GVFS.Common/NamedPipes/NamedPipeClient.cs @@ -8,8 +8,8 @@ public class NamedPipeClient : IDisposable { private string pipeName; private NamedPipeClientStream clientStream; - private StreamReader reader; - private StreamWriter writer; + private NamedPipeStreamReader reader; + private NamedPipeStreamWriter writer; public NamedPipeClient(string pipeName) { @@ -37,8 +37,8 @@ public bool Connect(int timeoutMilliseconds = 3000) return false; } - this.reader = new StreamReader(this.clientStream); - this.writer = new StreamWriter(this.clientStream); + this.reader = new NamedPipeStreamReader(this.clientStream); + this.writer = new NamedPipeStreamWriter(this.clientStream); return true; } @@ -68,8 +68,7 @@ public void SendRequest(string message) try { - this.writer.WritePlatformIndependentLine(message); - this.writer.Flush(); + this.writer.WriteMessage(message); } catch (IOException e) { @@ -81,7 +80,7 @@ public string ReadRawResponse() { try { - string response = this.reader.ReadLine(); + string response = this.reader.ReadMessage(); if (response == null) { throw new BrokenPipeException("Unable to read from pipe", null); diff --git a/GVFS/GVFS.Common/NamedPipes/NamedPipeServer.cs b/GVFS/GVFS.Common/NamedPipes/NamedPipeServer.cs index d6eebe711..83ac690e3 100644 --- a/GVFS/GVFS.Common/NamedPipes/NamedPipeServer.cs +++ b/GVFS/GVFS.Common/NamedPipes/NamedPipeServer.cs @@ -88,7 +88,7 @@ private void OpenListeningPipe() private void OnNewConnection(IAsyncResult ar) { if (!this.isStopping) - { + { this.OnNewConnection(ar, createNewThreadIfSynchronous: true); } } @@ -137,14 +137,14 @@ private void OnNewConnection(IAsyncResult ar, bool createNewThreadIfSynchronous) if (!connectionBroken) { - try - { - this.handleConnection(new Connection(pipe, () => this.isStopping)); - } - catch (Exception e) - { - this.LogErrorAndExit("Unhandled exception in connection handler", e); - } + try + { + this.handleConnection(new Connection(pipe, this.tracer, () => this.isStopping)); + } + catch (Exception e) + { + this.LogErrorAndExit("Unhandled exception in connection handler", e); + } } } } @@ -174,16 +174,18 @@ private void LogErrorAndExit(string message, Exception e) public class Connection { private NamedPipeServerStream serverStream; - private StreamReader reader; - private StreamWriter writer; + private NamedPipeStreamReader reader; + private NamedPipeStreamWriter writer; + private ITracer tracer; private Func isStopping; - public Connection(NamedPipeServerStream serverStream, Func isStopping) + public Connection(NamedPipeServerStream serverStream, ITracer tracer, Func isStopping) { this.serverStream = serverStream; + this.tracer = tracer; this.isStopping = isStopping; - this.reader = new StreamReader(this.serverStream); - this.writer = new StreamWriter(this.serverStream); + this.reader = new NamedPipeStreamReader(this.serverStream); + this.writer = new NamedPipeStreamWriter(this.serverStream); } public bool IsConnected @@ -200,10 +202,17 @@ public string ReadRequest() { try { - return this.reader.ReadLine(); + return this.reader.ReadMessage(); } - catch (IOException) + catch (IOException e) { + EventMetadata metadata = new EventMetadata(); + metadata.Add("ExceptionMessage", e.Message); + metadata.Add("StackTrace", e.StackTrace); + this.tracer.RelatedError( + metadata: metadata, + message: $"Error reading message from NamedPipe: {e.Message}"); + return null; } } @@ -212,9 +221,7 @@ public bool TrySendResponse(string message) { try { - this.writer.WritePlatformIndependentLine(message); - this.writer.Flush(); - + this.writer.WriteMessage(message); return true; } catch (IOException) diff --git a/GVFS/GVFS.Common/NamedPipes/NamedPipeStreamReader.cs b/GVFS/GVFS.Common/NamedPipes/NamedPipeStreamReader.cs new file mode 100644 index 000000000..17ced2e20 --- /dev/null +++ b/GVFS/GVFS.Common/NamedPipes/NamedPipeStreamReader.cs @@ -0,0 +1,92 @@ +using System; +using System.Collections.Generic; +using System.IO; +using System.Linq; +using System.Text; + +namespace GVFS.Common.NamedPipes +{ + /// + /// Implements the NamedPipe protocol as described in NamedPipeServer. + /// + public class NamedPipeStreamReader + { + private const int DefaultBufferSize = 1024; + private const byte TerminatorByte = 0x3; + + private int bufferSize; + private byte[] buffer; + private Stream stream; + + public NamedPipeStreamReader(Stream stream, int bufferSize) + { + this.stream = stream; + this.bufferSize = bufferSize; + this.buffer = new byte[this.bufferSize]; + } + + public NamedPipeStreamReader(Stream stream) + : this(stream, DefaultBufferSize) + { + } + + /// + /// Read a message from the stream. + /// + /// The message read from the stream, or null if the end of the input stream has been reached. + public string ReadMessage() + { + int bytesRead = this.Read(); + if (bytesRead == 0) + { + // The end of the stream has been reached - return null to indicate this. + return null; + } + + // If we have read in the entire message in the first read (mainline scenario), + // then just process the data directly from the buffer. + if (this.buffer[bytesRead - 1] == TerminatorByte) + { + return Encoding.UTF8.GetString(this.buffer, 0, bytesRead - 1); + } + + // We need to process multiple chunks - collect data from multiple chunks + // into a single list + List bytes = new List(this.bufferSize * 2); + + while (true) + { + bool encounteredTerminatorByte = this.buffer[bytesRead - 1] == TerminatorByte; + int lengthToCopy = encounteredTerminatorByte ? bytesRead - 1 : bytesRead; + + bytes.AddRange(new ArraySegment(this.buffer, 0, lengthToCopy)); + if (encounteredTerminatorByte) + { + break; + } + + bytesRead = this.Read(); + + if (bytesRead == 0) + { + // We have read a partial message (the last byte received does not indicate that + // this was the end of the message), but the stream has been closed. Throw an exception + // and let upper layer deal with this condition. + + throw new IOException("Incomplete message read from stream. The end of the stream was reached without the expected terminating byte."); + } + } + + return Encoding.UTF8.GetString(bytes.ToArray()); + } + + /// + /// Read the next chunk of bytes from the stream. + /// + /// The number of bytes read. + private int Read() + { + return this.stream.Read(this.buffer, 0, this.buffer.Length); + } + } +} diff --git a/GVFS/GVFS.Common/NamedPipes/NamedPipeStreamWriter.cs b/GVFS/GVFS.Common/NamedPipes/NamedPipeStreamWriter.cs new file mode 100644 index 000000000..f3f11ecbe --- /dev/null +++ b/GVFS/GVFS.Common/NamedPipes/NamedPipeStreamWriter.cs @@ -0,0 +1,24 @@ +using System.IO; +using System.Text; + +namespace GVFS.Common.NamedPipes +{ + public class NamedPipeStreamWriter + { + private const byte TerminatorByte = 0x3; + private const string TerminatorByteString = "\x3"; + private Stream stream; + + public NamedPipeStreamWriter(Stream stream) + { + this.stream = stream; + } + + public void WriteMessage(string message) + { + byte[] byteBuffer = Encoding.UTF8.GetBytes(message + TerminatorByteString); + this.stream.Write(byteBuffer, 0, byteBuffer.Length); + this.stream.Flush(); + } + } +} diff --git a/GVFS/GVFS.Common/NamedPipes/StreamWriterExtensions.cs b/GVFS/GVFS.Common/NamedPipes/StreamWriterExtensions.cs deleted file mode 100644 index 58db185d6..000000000 --- a/GVFS/GVFS.Common/NamedPipes/StreamWriterExtensions.cs +++ /dev/null @@ -1,16 +0,0 @@ -using System.IO; - -namespace GVFS.Common.NamedPipes -{ - public static class StreamWriterExtensions - { - public const int Foo = 0; - - public static void WritePlatformIndependentLine(this StreamWriter writer, string value) - { - // WriteLine is not platform independent as on some platforms it terminates lines with \r\n - // and on others it uses \n - writer.Write(value + "\n"); - } - } -} diff --git a/GVFS/GVFS.FunctionalTests/Tests/GitCommands/GitCommandsTests.cs b/GVFS/GVFS.FunctionalTests/Tests/GitCommands/GitCommandsTests.cs index f7f5a6d47..0b3ccccce 100644 --- a/GVFS/GVFS.FunctionalTests/Tests/GitCommands/GitCommandsTests.cs +++ b/GVFS/GVFS.FunctionalTests/Tests/GitCommands/GitCommandsTests.cs @@ -382,7 +382,6 @@ public void AddFileInSubfolderAndCommitOnNewBranchSwitchDeleteFolderAndSwitchBac } [TestCase] - [Ignore("Currently failing regression test for how GVFS handles newlines in git command line")] public void CommitWithNewlinesInMessage() { this.ValidateGitCommand("checkout -b tests/functional/commit_with_uncommon_arguments"); diff --git a/GVFS/GVFS.FunctionalTests/Tests/GitCommands/UpdateIndexTests.cs b/GVFS/GVFS.FunctionalTests/Tests/GitCommands/UpdateIndexTests.cs index df6871f24..d729822d0 100644 --- a/GVFS/GVFS.FunctionalTests/Tests/GitCommands/UpdateIndexTests.cs +++ b/GVFS/GVFS.FunctionalTests/Tests/GitCommands/UpdateIndexTests.cs @@ -65,7 +65,6 @@ public void UpdateIndexRemoveAddFileOpenForWrite() [TestCase] [Category(Categories.MacTODO.M4)] - [Ignore("Currently failing regression test for how GVFS handles newlines in git command line")] public void UpdateIndexWithCacheInfo() { // Update Protocol.md with the contents from blob 583f1... diff --git a/GVFS/GVFS.Hooks/GVFS.Hooks.Mac.csproj b/GVFS/GVFS.Hooks/GVFS.Hooks.Mac.csproj index ddde81618..dccf8bdd3 100644 --- a/GVFS/GVFS.Hooks/GVFS.Hooks.Mac.csproj +++ b/GVFS/GVFS.Hooks/GVFS.Hooks.Mac.csproj @@ -59,8 +59,11 @@ Common\NamedPipes\NamedPipeClient.cs - - Common\NamedPipes\StreamWriterExtensions.cs + + Common\NamedPipes\NamedPipeStreamReader.cs + + + Common\NamedPipes\NamedPipeStreamWriter.cs Common\NativeMethods.Shared.cs diff --git a/GVFS/GVFS.Hooks/GVFS.Hooks.Windows.csproj b/GVFS/GVFS.Hooks/GVFS.Hooks.Windows.csproj index 692c8832a..05629c39e 100644 --- a/GVFS/GVFS.Hooks/GVFS.Hooks.Windows.csproj +++ b/GVFS/GVFS.Hooks/GVFS.Hooks.Windows.csproj @@ -83,8 +83,11 @@ Common\NamedPipes\NamedPipeClient.cs - - Common\NamedPipes\StreamWriterExtensions.cs + + Common\NamedPipes\NamedPipeStreamReader.cs + + + Common\NamedPipes\NamedPipeStreamWriter.cs Common\NativeMethods.Shared.cs diff --git a/GVFS/GVFS.ReadObjectHook/main.cpp b/GVFS/GVFS.ReadObjectHook/main.cpp index 5fcb05af4..7a3f60887 100644 --- a/GVFS/GVFS.ReadObjectHook/main.cpp +++ b/GVFS/GVFS.ReadObjectHook/main.cpp @@ -18,8 +18,8 @@ #define DLO_REQUEST_LENGTH (4 + SHA1_LENGTH + 1) // Expected response: -// "S\n" -> Success -// "F\n" -> Failure +// "S\x3" -> Success +// "F\x3" -> Failure #define DLO_RESPONSE_LENGTH 2 enum ReadObjectHookErrorReturnCode @@ -33,7 +33,7 @@ int DownloadSHA(PIPE_HANDLE pipeHandle, const char *sha1) // Format: "DLO|<40 character SHA>" // Example: "DLO|920C34DCDDFC8F07AC4704C8C0D087D6F2095729" char request[DLO_REQUEST_LENGTH+1]; - if (snprintf(request, DLO_REQUEST_LENGTH+1, "DLO|%s\n", sha1) != DLO_REQUEST_LENGTH) + if (snprintf(request, DLO_REQUEST_LENGTH+1, "DLO|%s\x3", sha1) != DLO_REQUEST_LENGTH) { die(ReturnCode::InvalidSHA, "First argument must be a 40 character SHA, actual value: %s\n", sha1); } diff --git a/GVFS/GVFS.VirtualFileSystemHook/main.cpp b/GVFS/GVFS.VirtualFileSystemHook/main.cpp index d0c1aea18..60498900e 100644 --- a/GVFS/GVFS.VirtualFileSystemHook/main.cpp +++ b/GVFS/GVFS.VirtualFileSystemHook/main.cpp @@ -31,7 +31,7 @@ int main(int argc, char *argv[]) int error = 0; bool success = WriteToPipe( pipeHandle, - "MPL|1\n", + "MPL|1\x3", messageLength, &bytesWritten, &error); @@ -49,6 +49,7 @@ int main(int argc, char *argv[]) int lastError; bool finishedReading = false; bool firstRead = true; + do { char *pMessage = &message[0]; @@ -80,7 +81,7 @@ int main(int argc, char *argv[]) messageLength -= 2; } - if (*(pMessage + messageLength - 1) == '\n') + if (*(pMessage + messageLength - 1) == '\x3') { finishedReading = true; messageLength -= 1; From 659c2d4ba5e2059f0372226f8824778c0ba80e8a Mon Sep 17 00:00:00 2001 From: Jameson Miller Date: Fri, 21 Sep 2018 15:33:23 -0400 Subject: [PATCH 144/272] Add automated tests around NamedPipeStream protocol --- .../NamedPipeStreamReaderWriterTests.cs | 108 ++++++++++++++++++ 1 file changed, 108 insertions(+) create mode 100644 GVFS/GVFS.UnitTests/Common/NamedPipeStreamReaderWriterTests.cs diff --git a/GVFS/GVFS.UnitTests/Common/NamedPipeStreamReaderWriterTests.cs b/GVFS/GVFS.UnitTests/Common/NamedPipeStreamReaderWriterTests.cs new file mode 100644 index 000000000..003ca048f --- /dev/null +++ b/GVFS/GVFS.UnitTests/Common/NamedPipeStreamReaderWriterTests.cs @@ -0,0 +1,108 @@ +using GVFS.Common.NamedPipes; +using GVFS.Tests.Should; +using NUnit.Framework; +using System.IO; + +namespace GVFS.UnitTests.Common +{ + [TestFixture] + public class NamedPipeStreamReaderWriterTests + { + private const int BufferSize = 256; + + private MemoryStream stream; + private NamedPipeStreamWriter streamWriter; + private NamedPipeStreamReader streamReader; + + [SetUp] + public void Setup() + { + this.stream = new MemoryStream(); + this.streamWriter = new NamedPipeStreamWriter(this.stream); + this.streamReader = new NamedPipeStreamReader(this.stream, BufferSize); + } + + [Test] + [Description("Verify that we can transmit multiple messages")] + public void CanWriteAndReadMessages() + { + string firstMessage = @"This is a new message"; + this.TestTransmitMessage(firstMessage); + + string secondMessage = @"This is another message"; + this.TestTransmitMessage(secondMessage); + + string thirdMessage = @"This is the third message in a series of messages"; + this.TestTransmitMessage(thirdMessage); + + string longMessage = new string('T', 1024 * 5); + this.TestTransmitMessage(longMessage); + } + + [Test] + [Description("Verify that we can transmit a message that contains content that is the size of a NamedPipeStreamReader's buffer")] + public void CanSendBufferSizedContent() + { + string longMessage = new string('T', BufferSize); + this.TestTransmitMessage(longMessage); + } + + [Test] + [Description("Verify that we can transmit message that is the same size a NamedPipeStreamReader's buffer")] + public void CanSendBufferSizedMessage() + { + int numBytesInMessageTerminator = 1; + string longMessage = new string('T', BufferSize - numBytesInMessageTerminator); + this.TestTransmitMessage(longMessage); + } + + [Test] + [Description("Verify that the expected exception is thrown if message is not terminated with expected byte.")] + public void ReadingPartialMessgeThrows() + { + byte[] bytes = System.Text.Encoding.ASCII.GetBytes("This is a partial message"); + + this.stream.Write(bytes, 0, bytes.Length); + this.stream.Seek(0, SeekOrigin.Begin); + + Assert.Throws(() => this.streamReader.ReadMessage()); + } + + [Test] + [Description("Verify that we can transmit message that is larger than the buffer")] + public void CanSendMultiBufferSizedMessage() + { + string longMessage = new string('T', BufferSize * 3); + this.TestTransmitMessage(longMessage); + } + + [Test] + [Description("Verify that we can transmit message that newline characters")] + public void CanSendNewLines() + { + string messageWithNewLines = "This is a \nstringwith\nnewlines"; + this.TestTransmitMessage(messageWithNewLines); + } + + private void TestTransmitMessage(string message) + { + long pos = this.ReadStreamPosition(); + this.streamWriter.WriteMessage(message); + + this.SetStreamPosition(pos); + + string readMessage = this.streamReader.ReadMessage(); + readMessage.ShouldEqual(message, "The message read from the stream reader is not the same as the message that was sent."); + } + + private long ReadStreamPosition() + { + return this.stream.Position; + } + + private void SetStreamPosition(long position) + { + this.stream.Seek(position, SeekOrigin.Begin); + } + } +} From d67eac1a4d0f7e280d5a7d348434fd93de8a4693 Mon Sep 17 00:00:00 2001 From: William Baker Date: Fri, 21 Sep 2018 16:52:24 -0700 Subject: [PATCH 145/272] PR Feedback: Move exception handling into TryPrepareFolderForCallbacks --- GVFS/GVFS.Common/FileSystem/IKernelDriver.cs | 3 +- GVFS/GVFS.Platform.Mac/ProjFSKext.cs | 3 +- GVFS/GVFS.Platform.Windows/ProjFSFilter.cs | 51 ++++++++++++++++---- GVFS/GVFS/CommandLine/CloneVerb.cs | 27 ++++------- GVFS/GVFS/CommandLine/DehydrateVerb.cs | 22 ++++----- 5 files changed, 65 insertions(+), 41 deletions(-) diff --git a/GVFS/GVFS.Common/FileSystem/IKernelDriver.cs b/GVFS/GVFS.Common/FileSystem/IKernelDriver.cs index 270019da4..35fbdd1c1 100644 --- a/GVFS/GVFS.Common/FileSystem/IKernelDriver.cs +++ b/GVFS/GVFS.Common/FileSystem/IKernelDriver.cs @@ -1,4 +1,5 @@ using GVFS.Common.Tracing; +using System; namespace GVFS.Common.FileSystem { @@ -8,7 +9,7 @@ public interface IKernelDriver string DriverLogFolderName { get; } bool IsSupported(string normalizedEnlistmentRootPath, out string warning, out string error); string FlushDriverLogs(); - bool TryPrepareFolderForCallbacks(string folderPath, out string error); + bool TryPrepareFolderForCallbacks(string folderPath, out string error, out Exception exception); bool IsReady(JsonTracer tracer, string enlistmentRoot, out string error); } } diff --git a/GVFS/GVFS.Platform.Mac/ProjFSKext.cs b/GVFS/GVFS.Platform.Mac/ProjFSKext.cs index 9adc1d68c..ede5623fc 100644 --- a/GVFS/GVFS.Platform.Mac/ProjFSKext.cs +++ b/GVFS/GVFS.Platform.Mac/ProjFSKext.cs @@ -46,8 +46,9 @@ public bool IsReady(JsonTracer tracer, string enlistmentRoot, out string error) return true; } - public bool TryPrepareFolderForCallbacks(string folderPath, out string error) + public bool TryPrepareFolderForCallbacks(string folderPath, out string error, out Exception exception) { + exception = null; error = string.Empty; Result result = VirtualizationInstance.ConvertDirectoryToVirtualizationRoot(folderPath); if (result != Result.Success) diff --git a/GVFS/GVFS.Platform.Windows/ProjFSFilter.cs b/GVFS/GVFS.Platform.Windows/ProjFSFilter.cs index 544b5678a..3d7b4ce92 100644 --- a/GVFS/GVFS.Platform.Windows/ProjFSFilter.cs +++ b/GVFS/GVFS.Platform.Windows/ProjFSFilter.cs @@ -30,6 +30,7 @@ public class ProjFSFilter : IKernelDriver private const string FilterLoggerSessionName = "Microsoft-Windows-ProjFS-Filter-Log"; private const string ProjFSNativeLibFileName = "ProjectedFSLib.dll"; + private const string ProjFSManagedLibFileName = "ProjectedFSLib.Managed.dll"; private const uint OkResult = 0; private const uint NameCollisionErrorResult = 0x801F0012; @@ -320,18 +321,34 @@ public string FlushDriverLogs() return sb.ToString(); } - public bool TryPrepareFolderForCallbacks(string folderPath, out string error) + public bool TryPrepareFolderForCallbacks(string folderPath, out string error, out Exception exception) { - error = string.Empty; - Guid virtualizationInstanceGuid = Guid.NewGuid(); - HResult result = VirtualizationInstance.ConvertDirectoryToVirtualizationRoot(virtualizationInstanceGuid, folderPath); - if (result != HResult.Ok) + exception = null; + try { - error = "Failed to prepare \"" + folderPath + "\" for callbacks, error: " + result.ToString("F"); - return false; + return this.TryPrepareFolderForCallbacksImpl(folderPath, out error); } + catch (FileNotFoundException e) + { + exception = e; - return true; + if (e.FileName.Equals(ProjFSManagedLibFileName, StringComparison.OrdinalIgnoreCase)) + { + error = $"Failed to load {ProjFSManagedLibFileName}. Ensure that ProjFS is installed and enabled"; + } + else + { + error = $"FileNotFoundException while trying to prepare \"{folderPath}\" for callbacks: {e.Message}"; + } + + return false; + } + catch (Exception e) + { + exception = e; + error = $"Exception while trying to prepare \"{folderPath}\" for callbacks: {e.Message}"; + return false; + } } // TODO 1050199: Once the service is an optional component, GVFS should only attempt to attach @@ -562,7 +579,23 @@ private static EventMetadata CreateEventMetadata(Exception e = null) private static ProcessResult CallPowershellCommand(string command) { return ProcessHelper.Run("powershell.exe", "-NonInteractive -NoProfile -Command \"& { " + command + " }\""); - } + } + + // Using an Impl method allows TryPrepareFolderForCallbacks to catch any ProjFS dependency related exceptions + // thrown in the process of calling this method. + private bool TryPrepareFolderForCallbacksImpl(string folderPath, out string error) + { + error = string.Empty; + Guid virtualizationInstanceGuid = Guid.NewGuid(); + HResult result = VirtualizationInstance.ConvertDirectoryToVirtualizationRoot(virtualizationInstanceGuid, folderPath); + if (result != HResult.Ok) + { + error = "Failed to prepare \"" + folderPath + "\" for callbacks, error: " + result.ToString("F"); + return false; + } + + return true; + } private static class NativeMethods { diff --git a/GVFS/GVFS/CommandLine/CloneVerb.cs b/GVFS/GVFS/CommandLine/CloneVerb.cs index 6cf233ba2..5d86b909f 100644 --- a/GVFS/GVFS/CommandLine/CloneVerb.cs +++ b/GVFS/GVFS/CommandLine/CloneVerb.cs @@ -609,29 +609,22 @@ private Result CreateClone( } // Prepare the working directory folder for GVFS last to ensure that gvfs mount will fail if gvfs clone has failed - Result prepForCallbacksResult = new Result(true); - try + Exception exception; + string prepFileSystemError; + if (!GVFSPlatform.Instance.KernelDriver.TryPrepareFolderForCallbacks(enlistment.WorkingDirectoryRoot, out prepFileSystemError, out exception)) { - string prepFileSystemError; - if (!GVFSPlatform.Instance.KernelDriver.TryPrepareFolderForCallbacks(enlistment.WorkingDirectoryRoot, out prepFileSystemError)) + EventMetadata metadata = new EventMetadata(); + metadata.Add(nameof(prepFileSystemError), prepFileSystemError); + if (exception != null) { - prepForCallbacksResult = new Result(prepFileSystemError); + metadata.Add("Exception", exception.ToString()); } - } - catch (Exception e) - { - EventMetadata metadata = new EventMetadata(); - metadata.Add("Exception", e.ToString()); - tracer.RelatedError(metadata, $"{nameof(this.CreateClone)}: TryPrepareFolderForCallbacks failed"); - prepForCallbacksResult = new Result($"Failed to prepare \"{enlistment.WorkingDirectoryRoot}\" for callbacks, exception: {e.Message}"); - } - if (!prepForCallbacksResult.Success) - { - tracer.RelatedError($"TryPrepareFolderForCallbacks failed, error: {prepForCallbacksResult.ErrorMessage}"); + tracer.RelatedError(metadata, $"{nameof(this.CreateClone)}: TryPrepareFolderForCallbacks failed"); + return new Result(prepFileSystemError); } - return prepForCallbacksResult; + return new Result(true); } private void CreateGitScript(GVFSEnlistment enlistment) diff --git a/GVFS/GVFS/CommandLine/DehydrateVerb.cs b/GVFS/GVFS/CommandLine/DehydrateVerb.cs index a82e7cd13..906745f3c 100644 --- a/GVFS/GVFS/CommandLine/DehydrateVerb.cs +++ b/GVFS/GVFS/CommandLine/DehydrateVerb.cs @@ -221,23 +221,19 @@ private void Mount(ITracer tracer) private void PrepareSrcFolder(ITracer tracer, GVFSEnlistment enlistment) { - try + Exception exception; + string error; + if (!GVFSPlatform.Instance.KernelDriver.TryPrepareFolderForCallbacks(enlistment.WorkingDirectoryRoot, out error, out exception)) { - string error; - if (!GVFSPlatform.Instance.KernelDriver.TryPrepareFolderForCallbacks(enlistment.WorkingDirectoryRoot, out error)) + EventMetadata metadata = new EventMetadata(); + metadata.Add(nameof(error), error); + if (exception != null) { - EventMetadata metadata = new EventMetadata(); - metadata.Add(nameof(error), error); - tracer.RelatedError(metadata, $"{nameof(this.PrepareSrcFolder)}: TryPrepareFolderForCallbacks failed"); - this.ReportErrorAndExit(tracer, "Failed to recreate the virtualization root: " + error); + metadata.Add("Exception", exception.ToString()); } - } - catch (Exception e) - { - EventMetadata metadata = new EventMetadata(); - metadata.Add("Exception", e.ToString()); + tracer.RelatedError(metadata, $"{nameof(this.PrepareSrcFolder)}: TryPrepareFolderForCallbacks failed"); - this.ReportErrorAndExit(tracer, "Failed to recreate the virtualization root: " + e.Message); + this.ReportErrorAndExit(tracer, "Failed to recreate the virtualization root: " + error); } } From 71f5406f918f9145260042adf3b60c661609cae8 Mon Sep 17 00:00:00 2001 From: William Baker Date: Fri, 21 Sep 2018 17:15:23 -0700 Subject: [PATCH 146/272] PR Feedback: Move delete action check and use a queue for processing directories --- .../PrjFSKext/PrjFSKext/KauthHandler.cpp | 9 +++---- ProjFS.Mac/PrjFSLib/PrjFSLib.cpp | 25 +++++++++---------- 2 files changed, 16 insertions(+), 18 deletions(-) diff --git a/ProjFS.Mac/PrjFSKext/PrjFSKext/KauthHandler.cpp b/ProjFS.Mac/PrjFSKext/PrjFSKext/KauthHandler.cpp index ab53da185..d46aa411f 100644 --- a/ProjFS.Mac/PrjFSKext/PrjFSKext/KauthHandler.cpp +++ b/ProjFS.Mac/PrjFSKext/PrjFSKext/KauthHandler.cpp @@ -294,17 +294,16 @@ static int HandleVnodeOperation( if (VDIR == vnodeType) { - bool deleteAction = ActionBitIsSet(action, KAUTH_VNODE_DELETE); - - if (deleteAction || - ActionBitIsSet( + if (ActionBitIsSet( action, KAUTH_VNODE_LIST_DIRECTORY | KAUTH_VNODE_SEARCH | KAUTH_VNODE_READ_SECURITY | KAUTH_VNODE_READ_ATTRIBUTES | - KAUTH_VNODE_READ_EXTATTRIBUTES)) + KAUTH_VNODE_READ_EXTATTRIBUTES | + KAUTH_VNODE_DELETE)) { + bool deleteAction = ActionBitIsSet(action, KAUTH_VNODE_DELETE); // Recursively expand directory on delete to ensure child placeholders are created before rename operations if (deleteAction || FileFlagsBitIsSet(currentVnodeFileFlags, FileFlags_IsEmpty)) { diff --git a/ProjFS.Mac/PrjFSLib/PrjFSLib.cpp b/ProjFS.Mac/PrjFSLib/PrjFSLib.cpp index 5fb1afec6..eb67b38dc 100644 --- a/ProjFS.Mac/PrjFSLib/PrjFSLib.cpp +++ b/ProjFS.Mac/PrjFSLib/PrjFSLib.cpp @@ -9,6 +9,7 @@ #include #include #include +#include #include #include #include @@ -605,21 +606,21 @@ static PrjFS_Result HandleRecursivelyEnumerateDirectoryRequest(const MessageHead DIR* directory = nullptr; PrjFS_Result result = PrjFS_Result_Success; - std::list directoryRelativePaths; - directoryRelativePaths.push_back(path); + std::queue directoryRelativePaths; + directoryRelativePaths.push(path); // Walk each directory, expanding those that are found to be empty char pathBuffer[PrjFSMaxPath]; - std::list::iterator relativePathIter = directoryRelativePaths.begin(); - while(relativePathIter != directoryRelativePaths.end()) + while (!directoryRelativePaths.empty()) { - CombinePaths(s_virtualizationRootFullPath.c_str(), relativePathIter->c_str(), pathBuffer); + string directoryRelativePath(directoryRelativePaths.front()); + directoryRelativePaths.pop(); + + CombinePaths(s_virtualizationRootFullPath.c_str(), directoryRelativePath.c_str(), pathBuffer); - // TODO(Mac): how should we handle scenarios where we were unable to fully expand the directory and - // its children? if (IsBitSetInFileFlags(pathBuffer, FileFlags_IsEmpty)) { - PrjFS_Result result = HandleEnumerateDirectoryRequest(request, relativePathIter->c_str()); + PrjFS_Result result = HandleEnumerateDirectoryRequest(request, directoryRelativePath.c_str()); if (result != PrjFS_Result_Success) { goto CleanupAndReturn; @@ -634,20 +635,18 @@ static PrjFS_Result HandleRecursivelyEnumerateDirectoryRequest(const MessageHead } dirent* dirEntry = readdir(directory); - while(dirEntry != nullptr) + while (dirEntry != nullptr) { if (dirEntry->d_type == DT_DIR && 0 != strncmp(".", dirEntry->d_name, sizeof(dirEntry->d_name)) && 0 != strncmp("..", dirEntry->d_name, sizeof(dirEntry->d_name))) { - CombinePaths(relativePathIter->c_str(), dirEntry->d_name, pathBuffer); - directoryRelativePaths.push_back(pathBuffer); + CombinePaths(directoryRelativePath.c_str(), dirEntry->d_name, pathBuffer); + directoryRelativePaths.emplace(pathBuffer); } dirEntry = readdir(directory); } - - ++relativePathIter; } CleanupAndReturn: From eab7934d187c2ce82a9e7775b23727d2373b15d0 Mon Sep 17 00:00:00 2001 From: William Baker Date: Mon, 24 Sep 2018 10:12:18 -0700 Subject: [PATCH 147/272] Track both file type and file mode --- .../MacFileSystemVirtualizer.cs | 8 ++--- .../Projection/MockGitIndexProjection.cs | 8 ++--- .../MacFileSystemVirtualizerTests.cs | 13 +++---- .../GitIndexProjection.GitIndexEntry.cs | 2 +- .../GitIndexProjection.GitIndexParser.cs | 35 +++++++++++++++++-- .../Projection/GitIndexProjection.cs | 31 +++++++++++----- 6 files changed, 71 insertions(+), 26 deletions(-) diff --git a/GVFS/GVFS.Platform.Mac/MacFileSystemVirtualizer.cs b/GVFS/GVFS.Platform.Mac/MacFileSystemVirtualizer.cs index 3d22f7f34..c8d954bc7 100644 --- a/GVFS/GVFS.Platform.Mac/MacFileSystemVirtualizer.cs +++ b/GVFS/GVFS.Platform.Mac/MacFileSystemVirtualizer.cs @@ -80,13 +80,13 @@ public override FileSystemResult WritePlaceholderFile( string sha) { // TODO(Mac): Add functional tests that validate file mode is set correctly - ushort fileMode = this.FileSystemCallbacks.GitIndexProjection.GetFilePathMode(relativePath); + ushort fileTypeAndMode = this.FileSystemCallbacks.GitIndexProjection.GetFileTypeAndMode(relativePath); Result result = this.virtualizationInstance.WritePlaceholderFile( relativePath, PlaceholderVersionId, ToVersionIdByteArray(FileSystemVirtualizer.ConvertShaToContentId(sha)), (ulong)endOfFile, - fileMode); + (ushort)(fileTypeAndMode & GitIndexProjection.FileModeMask)); return new FileSystemResult(ResultToFSResult(result), unchecked((int)result)); } @@ -114,14 +114,14 @@ public override FileSystemResult UpdatePlaceholderIfNeeded( // TODO(Mac): Add functional tests that include: // - Mode + content changes between commits // - Mode only changes (without any change to content, see issue #223) - ushort fileMode = this.FileSystemCallbacks.GitIndexProjection.GetFilePathMode(relativePath); + ushort fileTypeAndMode = this.FileSystemCallbacks.GitIndexProjection.GetFileTypeAndMode(relativePath); Result result = this.virtualizationInstance.UpdatePlaceholderIfNeeded( relativePath, PlaceholderVersionId, ToVersionIdByteArray(ConvertShaToContentId(shaContentId)), (ulong)endOfFile, - fileMode, + (ushort)(fileTypeAndMode & GitIndexProjection.FileModeMask), (UpdateType)updateFlags, out failureCause); failureReason = (UpdateFailureReason)failureCause; diff --git a/GVFS/GVFS.UnitTests/Mock/Virtualization/Projection/MockGitIndexProjection.cs b/GVFS/GVFS.UnitTests/Mock/Virtualization/Projection/MockGitIndexProjection.cs index 904946610..9bd7c5f5a 100644 --- a/GVFS/GVFS.UnitTests/Mock/Virtualization/Projection/MockGitIndexProjection.cs +++ b/GVFS/GVFS.UnitTests/Mock/Virtualization/Projection/MockGitIndexProjection.cs @@ -36,7 +36,7 @@ public MockGitIndexProjection(IEnumerable projectedFiles) this.PlaceholdersCreated = new ConcurrentHashSet(); this.ExpandedFolders = new ConcurrentHashSet(); - this.MockFileModes = new ConcurrentDictionary(); + this.MockFileTypesAndModes = new ConcurrentDictionary(); this.unblockGetProjectedItems = new ManualResetEvent(true); this.waitForGetProjectedItems = new ManualResetEvent(true); @@ -56,7 +56,7 @@ public MockGitIndexProjection(IEnumerable projectedFiles) public ConcurrentHashSet ExpandedFolders { get; } - public ConcurrentDictionary MockFileModes { get; } + public ConcurrentDictionary MockFileTypesAndModes { get; } public bool ThrowOperationCanceledExceptionOnProjectionRequest { get; set; } @@ -161,10 +161,10 @@ public override bool TryGetProjectedItemsFromMemory(string folderPath, out List< return false; } - public override ushort GetFilePathMode(string path) + public override ushort GetFileTypeAndMode(string path) { ushort result; - if (this.MockFileModes.TryGetValue(path, out result)) + if (this.MockFileTypesAndModes.TryGetValue(path, out result)) { return result; } diff --git a/GVFS/GVFS.UnitTests/Platform.Mac/MacFileSystemVirtualizerTests.cs b/GVFS/GVFS.UnitTests/Platform.Mac/MacFileSystemVirtualizerTests.cs index 675ae8a93..96cb89f60 100644 --- a/GVFS/GVFS.UnitTests/Platform.Mac/MacFileSystemVirtualizerTests.cs +++ b/GVFS/GVFS.UnitTests/Platform.Mac/MacFileSystemVirtualizerTests.cs @@ -21,6 +21,7 @@ namespace GVFS.UnitTests.Platform.Mac [TestFixture] public class MacFileSystemVirtualizerTests : TestsWithCommonRepo { + private static readonly ushort RegularFileType = 0x8000; private static readonly ushort FileMode644 = Convert.ToUInt16("644", 8); private static readonly ushort FileMode664 = Convert.ToUInt16("664", 8); private static readonly ushort FileMode755 = Convert.ToUInt16("755", 8); @@ -98,7 +99,7 @@ public void UpdatePlaceholderIfNeeded() backgroundTaskRunner, virtualizer)) { - gitIndexProjection.MockFileModes.TryAdd("test" + Path.DirectorySeparatorChar + "test.txt", FileMode644); + gitIndexProjection.MockFileTypesAndModes.TryAdd("test" + Path.DirectorySeparatorChar + "test.txt", (ushort)(RegularFileType | FileMode644)); string error; fileSystemCallbacks.TryStart(out error).ShouldEqual(true); @@ -189,7 +190,7 @@ public void OnEnumerateDirectoryReturnsSuccessWhenResultsNotInMemory() backgroundFileSystemTaskRunner: backgroundTaskRunner, fileSystemVirtualizer: virtualizer)) { - gitIndexProjection.MockFileModes.TryAdd("test" + Path.DirectorySeparatorChar + "test.txt", FileMode644); + gitIndexProjection.MockFileTypesAndModes.TryAdd("test" + Path.DirectorySeparatorChar + "test.txt", (ushort)(RegularFileType | FileMode644)); string error; fileSystemCallbacks.TryStart(out error).ShouldEqual(true); @@ -219,7 +220,7 @@ public void OnEnumerateDirectoryReturnsSuccessWhenResultsInMemory() backgroundFileSystemTaskRunner: backgroundTaskRunner, fileSystemVirtualizer: virtualizer)) { - gitIndexProjection.MockFileModes.TryAdd("test" + Path.DirectorySeparatorChar + "test.txt", FileMode644); + gitIndexProjection.MockFileTypesAndModes.TryAdd("test" + Path.DirectorySeparatorChar + "test.txt", (ushort)(RegularFileType | FileMode644)); string error; fileSystemCallbacks.TryStart(out error).ShouldEqual(true); @@ -250,9 +251,9 @@ public void OnEnumerateDirectorySetsFileModes() backgroundFileSystemTaskRunner: backgroundTaskRunner, fileSystemVirtualizer: virtualizer)) { - gitIndexProjection.MockFileModes.TryAdd("test" + Path.DirectorySeparatorChar + "test644.txt", FileMode644); - gitIndexProjection.MockFileModes.TryAdd("test" + Path.DirectorySeparatorChar + "test664.txt", FileMode664); - gitIndexProjection.MockFileModes.TryAdd("test" + Path.DirectorySeparatorChar + "test755.txt", FileMode755); + gitIndexProjection.MockFileTypesAndModes.TryAdd("test" + Path.DirectorySeparatorChar + "test644.txt", (ushort)(RegularFileType | FileMode644)); + gitIndexProjection.MockFileTypesAndModes.TryAdd("test" + Path.DirectorySeparatorChar + "test664.txt", (ushort)(RegularFileType | FileMode664)); + gitIndexProjection.MockFileTypesAndModes.TryAdd("test" + Path.DirectorySeparatorChar + "test755.txt", (ushort)(RegularFileType | FileMode755)); string error; fileSystemCallbacks.TryStart(out error).ShouldEqual(true); diff --git a/GVFS/GVFS.Virtualization/Projection/GitIndexProjection.GitIndexEntry.cs b/GVFS/GVFS.Virtualization/Projection/GitIndexProjection.GitIndexEntry.cs index 1da6d6642..612931242 100644 --- a/GVFS/GVFS.Virtualization/Projection/GitIndexProjection.GitIndexEntry.cs +++ b/GVFS/GVFS.Virtualization/Projection/GitIndexProjection.GitIndexEntry.cs @@ -27,7 +27,7 @@ public GitIndexEntry() public byte[] Sha { get; } = new byte[20]; public bool SkipWorktree { get; set; } - public ushort FileMode { get; set; } + public ushort FileTypeAndMode { get; set; } public GitIndexParser.MergeStage MergeState { get; set; } public int ReplaceIndex { get; set; } diff --git a/GVFS/GVFS.Virtualization/Projection/GitIndexProjection.GitIndexParser.cs b/GVFS/GVFS.Virtualization/Projection/GitIndexProjection.GitIndexParser.cs index 682f9ec3b..cd0cc2031 100644 --- a/GVFS/GVFS.Virtualization/Projection/GitIndexProjection.GitIndexParser.cs +++ b/GVFS/GVFS.Virtualization/Projection/GitIndexProjection.GitIndexParser.cs @@ -16,6 +16,10 @@ internal partial class GitIndexParser private const ushort ExtendedBit = 0x4000; private const ushort SkipWorktreeBit = 0x4000; + private static readonly ushort FileMode755 = Convert.ToUInt16("755", 8); + private static readonly ushort FileMode664 = Convert.ToUInt16("644", 8); + private static readonly ushort FileMode644 = Convert.ToUInt16("644", 8); + private Stream indexStream; private byte[] page; private int nextByteIndex; @@ -181,8 +185,35 @@ private FileSystemTaskResult ParseIndex(ITracer tracer, Stream indexStream, Func // 3-bit unused // 9-bit unix permission. Only 0755 and 0644 are valid for regular files. (Legacy repos can also contain 664) // Symbolic links and gitlinks have value 0 in this field. - ushort mode = this.ReadUInt16(); - this.resuableParsedIndexEntry.FileMode = (ushort)(mode & 0x1FF); + ushort typeAndMode = this.ReadUInt16(); + + ushort type = (ushort)(typeAndMode & FileTypeMask); + ushort mode = (ushort)(typeAndMode & FileModeMask); + + switch ((FileType)type) + { + case FileType.Regular: + if (mode != FileMode755 && mode != FileMode644 && mode != FileMode664) + { + throw new InvalidDataException($"Invalid file mode {Convert.ToString(mode, 8)} found for regular file in index"); + } + + break; + + case FileType.SymLink: + case FileType.GitLink: + if (mode != 0) + { + throw new InvalidDataException($"Invalid file mode {Convert.ToString(mode, 8)} found for link file({type:X}) in index"); + } + + break; + + default: + throw new InvalidDataException($"Invalid object type {type:X} found in index"); + } + + this.resuableParsedIndexEntry.FileTypeAndMode = typeAndMode; this.Skip(12); } diff --git a/GVFS/GVFS.Virtualization/Projection/GitIndexProjection.cs b/GVFS/GVFS.Virtualization/Projection/GitIndexProjection.cs index 66b139615..8c1dbe924 100644 --- a/GVFS/GVFS.Virtualization/Projection/GitIndexProjection.cs +++ b/GVFS/GVFS.Virtualization/Projection/GitIndexProjection.cs @@ -22,7 +22,11 @@ public partial class GitIndexProjection : IDisposable, IProfilerOnlyIndexProject { public const string ProjectionIndexBackupName = "GVFS_projection"; - protected static readonly ushort FileMode644 = Convert.ToUInt16("644", 8); + // Bitmasks for extracting file type and mode from the ushort stored in the index + public const ushort FileTypeMask = 0xF000; + public const ushort FileModeMask = 0x1FF; + + protected static readonly ushort Regular644File = (ushort)((ushort)FileType.Regular | Convert.ToUInt16("644", 8)); private const int IndexFileStreamBufferSize = 512 * 1024; @@ -54,7 +58,7 @@ public partial class GitIndexProjection : IDisposable, IProfilerOnlyIndexProject // nonDefaultFileModes is only populated when the platform supports file mode // On platforms that support file modes, file paths that are not in nonDefaultFileModes have mode 644 - private Dictionary nonDefaultFileModes = new Dictionary(StringComparer.OrdinalIgnoreCase); + private Dictionary nonDefaultFileTypesAndModes = new Dictionary(StringComparer.OrdinalIgnoreCase); private BlobSizes blobSizes; private PlaceholderListDatabase placeholderList; @@ -118,6 +122,15 @@ protected GitIndexProjection() { } + public enum FileType : ushort + { + Invalid = 0, + + Regular = 0x8000, + SymLink = 0xA000, + GitLink = 0xE000, + } + public int EstimatedPlaceholderCount { get @@ -356,11 +369,11 @@ public virtual bool TryGetProjectedItemsFromMemory(string folderPath, out List

")); } From f0185654fee7453647637e81ff5e1b550acdca72 Mon Sep 17 00:00:00 2001 From: William Baker Date: Mon, 24 Sep 2018 11:16:04 -0700 Subject: [PATCH 148/272] Add symlink projection code --- .../FileSystem/PhysicalFileSystem.cs | 4 +- GVFS/GVFS.Common/Git/GVFSGitObjects.cs | 1 + GVFS/GVFS.Common/NativeMethods.cs | 2 +- .../Windows/Tests/DiskLayoutUpgradeTests.cs | 8 +- .../MacFileSystemVirtualizer.cs | 164 ++++++++++++++++-- ...skLayout12to13Upgrade_FolderPlaceholder.cs | 2 +- .../MacFileSystemVirtualizerTests.cs | 9 +- .../Background/FileSystemTask.cs | 8 +- .../FileSystemCallbacks.cs | 6 + .../GitIndexProjection.GitIndexParser.cs | 2 +- .../PrjFSLib.Mac.Managed/Interop/PrjFSLib.cs | 14 +- .../VirtualizationInstance.cs | 24 +++ ProjFS.Mac/PrjFSLib/PrjFSLib.cpp | 55 ++++++ ProjFS.Mac/PrjFSLib/PrjFSLib.h | 10 ++ 14 files changed, 276 insertions(+), 33 deletions(-) diff --git a/GVFS/GVFS.Common/FileSystem/PhysicalFileSystem.cs b/GVFS/GVFS.Common/FileSystem/PhysicalFileSystem.cs index 9d0b23998..cbe88f851 100644 --- a/GVFS/GVFS.Common/FileSystem/PhysicalFileSystem.cs +++ b/GVFS/GVFS.Common/FileSystem/PhysicalFileSystem.cs @@ -103,9 +103,9 @@ public virtual void DeleteDirectory(string path, bool recursive = false) RecursiveDelete(path); } - public virtual bool IsSymlink(string path) + public virtual bool IsSymLink(string path) { - return (this.GetAttributes(path) & FileAttributes.ReparsePoint) == FileAttributes.ReparsePoint && NativeMethods.IsSymlink(path); + return (this.GetAttributes(path) & FileAttributes.ReparsePoint) == FileAttributes.ReparsePoint && NativeMethods.IsSymLink(path); } public virtual IEnumerable ItemsInDirectory(string path) diff --git a/GVFS/GVFS.Common/Git/GVFSGitObjects.cs b/GVFS/GVFS.Common/Git/GVFSGitObjects.cs index 6c77a211d..6f3335f61 100644 --- a/GVFS/GVFS.Common/Git/GVFSGitObjects.cs +++ b/GVFS/GVFS.Common/Git/GVFSGitObjects.cs @@ -28,6 +28,7 @@ public enum RequestSource FileStreamCallback, GVFSVerb, NamedPipeMessage, + SymLinkCreation, } protected GVFSContext Context { get; private set; } diff --git a/GVFS/GVFS.Common/NativeMethods.cs b/GVFS/GVFS.Common/NativeMethods.cs index 4f19f1e25..65074fdb0 100644 --- a/GVFS/GVFS.Common/NativeMethods.cs +++ b/GVFS/GVFS.Common/NativeMethods.cs @@ -127,7 +127,7 @@ public static uint GetWindowsBuildNumber() return versionInfo.BuildNumber; } - public static bool IsSymlink(string path) + public static bool IsSymLink(string path) { using (SafeFileHandle output = CreateFile( path, diff --git a/GVFS/GVFS.FunctionalTests.Windows/Windows/Tests/DiskLayoutUpgradeTests.cs b/GVFS/GVFS.FunctionalTests.Windows/Windows/Tests/DiskLayoutUpgradeTests.cs index d3ea1f93b..e29598bfb 100644 --- a/GVFS/GVFS.FunctionalTests.Windows/Windows/Tests/DiskLayoutUpgradeTests.cs +++ b/GVFS/GVFS.FunctionalTests.Windows/Windows/Tests/DiskLayoutUpgradeTests.cs @@ -361,14 +361,14 @@ private void PerformIOBeforePlaceholderDatabaseUpgradeTest() this.fileSystem.DeleteDirectory(Path.Combine(this.Enlistment.RepoRoot, "GVFS\\GVFS.Tests\\Properties")); string junctionTarget = Path.Combine(this.Enlistment.EnlistmentRoot, "DirJunction"); - string symlinkTarget = Path.Combine(this.Enlistment.EnlistmentRoot, "DirSymlink"); + string symLinkTarget = Path.Combine(this.Enlistment.EnlistmentRoot, "DirSymLink"); Directory.CreateDirectory(junctionTarget); - Directory.CreateDirectory(symlinkTarget); + Directory.CreateDirectory(symLinkTarget); string junctionLink = Path.Combine(this.Enlistment.RepoRoot, "DirJunction"); - string symlink = Path.Combine(this.Enlistment.RepoRoot, "DirLink"); + string symLink = Path.Combine(this.Enlistment.RepoRoot, "DirLink"); ProcessHelper.Run("CMD.exe", "/C mklink /J " + junctionLink + " " + junctionTarget); - ProcessHelper.Run("CMD.exe", "/C mklink /D " + symlink + " " + symlinkTarget); + ProcessHelper.Run("CMD.exe", "/C mklink /D " + symLink + " " + symLinkTarget); string target = Path.Combine(this.Enlistment.EnlistmentRoot, "GVFS", "GVFS", "GVFS.UnitTests"); string link = Path.Combine(this.Enlistment.RepoRoot, "UnitTests"); diff --git a/GVFS/GVFS.Platform.Mac/MacFileSystemVirtualizer.cs b/GVFS/GVFS.Platform.Mac/MacFileSystemVirtualizer.cs index c8d954bc7..610133d85 100644 --- a/GVFS/GVFS.Platform.Mac/MacFileSystemVirtualizer.cs +++ b/GVFS/GVFS.Platform.Mac/MacFileSystemVirtualizer.cs @@ -8,6 +8,7 @@ using System; using System.Collections.Generic; using System.IO; +using System.Text; using System.Threading; namespace GVFS.Platform.Mac @@ -81,14 +82,43 @@ public override FileSystemResult WritePlaceholderFile( { // TODO(Mac): Add functional tests that validate file mode is set correctly ushort fileTypeAndMode = this.FileSystemCallbacks.GitIndexProjection.GetFileTypeAndMode(relativePath); - Result result = this.virtualizationInstance.WritePlaceholderFile( - relativePath, - PlaceholderVersionId, - ToVersionIdByteArray(FileSystemVirtualizer.ConvertShaToContentId(sha)), - (ulong)endOfFile, - (ushort)(fileTypeAndMode & GitIndexProjection.FileModeMask)); + GitIndexProjection.FileType fileType = (GitIndexProjection.FileType)(fileTypeAndMode & GitIndexProjection.FileTypeMask); - return new FileSystemResult(ResultToFSResult(result), unchecked((int)result)); + if (fileType == GitIndexProjection.FileType.Regular) + { + Result result = this.virtualizationInstance.WritePlaceholderFile( + relativePath, + PlaceholderVersionId, + ToVersionIdByteArray(FileSystemVirtualizer.ConvertShaToContentId(sha)), + (ulong)endOfFile, + (ushort)(fileTypeAndMode & GitIndexProjection.FileModeMask)); + + return new FileSystemResult(ResultToFSResult(result), unchecked((int)result)); + } + else if (fileType == GitIndexProjection.FileType.SymLink) + { + string symLinkContents; + if (this.TryGetSymLinkObjectContents(sha, out symLinkContents)) + { + Result result = this.virtualizationInstance.WriteSymLink(relativePath, symLinkContents); + + this.FileSystemCallbacks.OnFileSymLinkCreated(relativePath); + + return new FileSystemResult(ResultToFSResult(result), unchecked((int)result)); + } + + EventMetadata metadata = this.CreateEventMetadata(relativePath); + metadata.Add(nameof(sha), sha); + this.Context.Tracer.RelatedError(metadata, $"{nameof(this.WritePlaceholderFile)}: Failed to read contents of symlink object"); + return new FileSystemResult(FSResult.IOError, 0); + } + else + { + EventMetadata metadata = this.CreateEventMetadata(relativePath); + metadata.Add(nameof(fileType), fileType); + this.Context.Tracer.RelatedError(metadata, $"{nameof(this.WritePlaceholderFile)}: Unsupported fileType"); + return new FileSystemResult(FSResult.IOError, 0); + } } public override FileSystemResult WritePlaceholderDirectory(string relativePath) @@ -115,17 +145,54 @@ public override FileSystemResult UpdatePlaceholderIfNeeded( // - Mode + content changes between commits // - Mode only changes (without any change to content, see issue #223) ushort fileTypeAndMode = this.FileSystemCallbacks.GitIndexProjection.GetFileTypeAndMode(relativePath); + GitIndexProjection.FileType fileType = (GitIndexProjection.FileType)(fileTypeAndMode & GitIndexProjection.FileTypeMask); + + if (fileType == GitIndexProjection.FileType.Regular) + { + Result result = this.virtualizationInstance.UpdatePlaceholderIfNeeded( + relativePath, + PlaceholderVersionId, + ToVersionIdByteArray(ConvertShaToContentId(shaContentId)), + (ulong)endOfFile, + (ushort)(fileTypeAndMode & GitIndexProjection.FileModeMask), + (UpdateType)updateFlags, + out failureCause); + + failureReason = (UpdateFailureReason)failureCause; + return new FileSystemResult(ResultToFSResult(result), unchecked((int)result)); + } + else if (fileType == GitIndexProjection.FileType.SymLink) + { + string symLinkContents; + if (this.TryGetSymLinkObjectContents(shaContentId, out symLinkContents)) + { + Result result = this.virtualizationInstance.ReplacePlaceholderFileWithSymLink( + relativePath, + symLinkContents, + (UpdateType)updateFlags, + out failureCause); + + this.FileSystemCallbacks.OnFileSymLinkCreated(relativePath); + + failureReason = (UpdateFailureReason)failureCause; + return new FileSystemResult(ResultToFSResult(result), unchecked((int)result)); + } - Result result = this.virtualizationInstance.UpdatePlaceholderIfNeeded( - relativePath, - PlaceholderVersionId, - ToVersionIdByteArray(ConvertShaToContentId(shaContentId)), - (ulong)endOfFile, - (ushort)(fileTypeAndMode & GitIndexProjection.FileModeMask), - (UpdateType)updateFlags, - out failureCause); - failureReason = (UpdateFailureReason)failureCause; - return new FileSystemResult(ResultToFSResult(result), unchecked((int)result)); + EventMetadata metadata = this.CreateEventMetadata(relativePath); + metadata.Add(nameof(shaContentId), shaContentId); + this.Context.Tracer.RelatedError(metadata, $"{nameof(this.UpdatePlaceholderIfNeeded)}: Failed to read contents of symlink object"); + failureReason = UpdateFailureReason.NoFailure; + return new FileSystemResult(FSResult.IOError, 0); + } + else + { + EventMetadata metadata = this.CreateEventMetadata(relativePath); + metadata.Add(nameof(fileType), fileType); + metadata.Add(nameof(fileTypeAndMode), fileTypeAndMode); + this.Context.Tracer.RelatedError(metadata, $"{nameof(this.UpdatePlaceholderIfNeeded)}: Unsupported fileType"); + failureReason = UpdateFailureReason.NoFailure; + return new FileSystemResult(FSResult.IOError, 0); + } } protected override bool TryStart(out string error) @@ -165,6 +232,67 @@ private static byte[] ToVersionIdByteArray(byte[] version) return bytes; } + private bool TryGetSymLinkObjectContents(string sha, out string contents) + { + contents = null; + StringBuilder objectContents = new StringBuilder(); + + try + { + if (!this.GitObjects.TryCopyBlobContentStream( + sha, + CancellationToken.None, + GVFSGitObjects.RequestSource.SymLinkCreation, + (stream, blobLength) => + { + // TODO(Mac): Find a better solution than reading from the stream one byte at at time + byte[] buffer = new byte[4096]; + uint bufferIndex = 0; + int nextByte = stream.ReadByte(); + while (nextByte != -1) + { + while (bufferIndex < buffer.Length && nextByte != -1) + { + buffer[bufferIndex] = (byte)nextByte; + nextByte = stream.ReadByte(); + ++bufferIndex; + } + + while (bufferIndex < buffer.Length) + { + buffer[bufferIndex] = 0; + ++bufferIndex; + } + + objectContents.Append(System.Text.Encoding.ASCII.GetString(buffer)); + + if (bufferIndex == buffer.Length) + { + bufferIndex = 0; + } + } + })) + { + EventMetadata metadata = this.CreateEventMetadata(); + metadata.Add(nameof(sha), sha); + this.Context.Tracer.RelatedError(metadata, $"{nameof(this.TryGetSymLinkObjectContents)}: TryCopyBlobContentStream failed"); + + return false; + } + } + catch (GetFileStreamException e) + { + EventMetadata metadata = this.CreateEventMetadata(relativePath: null, exception: e); + metadata.Add(nameof(sha), sha); + this.Context.Tracer.RelatedError(metadata, $"{nameof(this.TryGetSymLinkObjectContents)}: TryCopyBlobContentStream caught GetFileStreamException"); + + return false; + } + + contents = objectContents.ToString(); + return true; + } + private Result OnGetFileStream( ulong commandId, string relativePath, @@ -258,7 +386,7 @@ private Result OnGetFileStream( { activity.RelatedError(metadata, $"{nameof(this.OnGetFileStream)}: TryCopyBlobContentStream failed"); - // TODO: Is this the correct Result to return? + // TODO(Mac): Is this the correct Result to return? return Result.EFileNotFound; } } diff --git a/GVFS/GVFS.Platform.Windows/DiskLayoutUpgrades/DiskLayout12to13Upgrade_FolderPlaceholder.cs b/GVFS/GVFS.Platform.Windows/DiskLayoutUpgrades/DiskLayout12to13Upgrade_FolderPlaceholder.cs index 28653438c..e2d88bf09 100644 --- a/GVFS/GVFS.Platform.Windows/DiskLayoutUpgrades/DiskLayout12to13Upgrade_FolderPlaceholder.cs +++ b/GVFS/GVFS.Platform.Windows/DiskLayoutUpgrades/DiskLayout12to13Upgrade_FolderPlaceholder.cs @@ -75,7 +75,7 @@ public override bool TryUpgrade(ITracer tracer, string enlistmentRoot) private static IEnumerable GetFolderPlaceholdersFromDisk(ITracer tracer, PhysicalFileSystem fileSystem, string path) { - if (!fileSystem.IsSymlink(path)) + if (!fileSystem.IsSymLink(path)) { foreach (string directory in fileSystem.EnumerateDirectories(path)) { diff --git a/GVFS/GVFS.UnitTests/Platform.Mac/MacFileSystemVirtualizerTests.cs b/GVFS/GVFS.UnitTests/Platform.Mac/MacFileSystemVirtualizerTests.cs index 96cb89f60..6d4d19d6d 100644 --- a/GVFS/GVFS.UnitTests/Platform.Mac/MacFileSystemVirtualizerTests.cs +++ b/GVFS/GVFS.UnitTests/Platform.Mac/MacFileSystemVirtualizerTests.cs @@ -99,7 +99,8 @@ public void UpdatePlaceholderIfNeeded() backgroundTaskRunner, virtualizer)) { - gitIndexProjection.MockFileTypesAndModes.TryAdd("test" + Path.DirectorySeparatorChar + "test.txt", (ushort)(RegularFileType | FileMode644)); + string filePath = "test" + Path.DirectorySeparatorChar + "test.txt"; + gitIndexProjection.MockFileTypesAndModes.TryAdd(filePath, (ushort)(RegularFileType | FileMode644)); string error; fileSystemCallbacks.TryStart(out error).ShouldEqual(true); @@ -109,7 +110,7 @@ public void UpdatePlaceholderIfNeeded() mockVirtualization.UpdatePlaceholderIfNeededFailureCause = UpdateFailureCause.NoFailure; virtualizer .UpdatePlaceholderIfNeeded( - "test.txt", + filePath, DateTime.Now, DateTime.Now, DateTime.Now, @@ -126,7 +127,7 @@ public void UpdatePlaceholderIfNeeded() mockVirtualization.UpdatePlaceholderIfNeededFailureCause = UpdateFailureCause.NoFailure; virtualizer .UpdatePlaceholderIfNeeded( - "test.txt", + filePath, DateTime.Now, DateTime.Now, DateTime.Now, @@ -146,7 +147,7 @@ public void UpdatePlaceholderIfNeeded() // TODO: The result should probably be VirtualizationInvalidOperation but for now it's IOError virtualizer .UpdatePlaceholderIfNeeded( - "test.txt", + filePath, DateTime.Now, DateTime.Now, DateTime.Now, diff --git a/GVFS/GVFS.Virtualization/Background/FileSystemTask.cs b/GVFS/GVFS.Virtualization/Background/FileSystemTask.cs index b5644bf86..ff8950b4f 100644 --- a/GVFS/GVFS.Virtualization/Background/FileSystemTask.cs +++ b/GVFS/GVFS.Virtualization/Background/FileSystemTask.cs @@ -29,7 +29,8 @@ public enum OperationType OnFolderFirstWrite, OnIndexWriteWithoutProjectionChange, OnPlaceholderCreationsBlockedForGit, - OnFileHardLinkCreated + OnFileHardLinkCreated, + OnFileSymLinkCreated } public OperationType Operation { get; } @@ -52,6 +53,11 @@ public static FileSystemTask OnFileHardLinkCreated(string newLinkRelativePath) return new FileSystemTask(OperationType.OnFileHardLinkCreated, newLinkRelativePath, oldVirtualPath: null); } + public static FileSystemTask OnFileSymLinkCreated(string newLinkRelativePath) + { + return new FileSystemTask(OperationType.OnFileSymLinkCreated, newLinkRelativePath, oldVirtualPath: null); + } + public static FileSystemTask OnFileDeleted(string virtualPath) { return new FileSystemTask(OperationType.OnFileDeleted, virtualPath, oldVirtualPath: null); diff --git a/GVFS/GVFS.Virtualization/FileSystemCallbacks.cs b/GVFS/GVFS.Virtualization/FileSystemCallbacks.cs index a9ea5200e..b8ea79d83 100644 --- a/GVFS/GVFS.Virtualization/FileSystemCallbacks.cs +++ b/GVFS/GVFS.Virtualization/FileSystemCallbacks.cs @@ -428,6 +428,11 @@ public virtual void OnFileHardLinkCreated(string newLinkRelativePath) this.backgroundFileSystemTaskRunner.Enqueue(FileSystemTask.OnFileHardLinkCreated(newLinkRelativePath)); } + public virtual void OnFileSymLinkCreated(string newLinkRelativePath) + { + this.backgroundFileSystemTaskRunner.Enqueue(FileSystemTask.OnFileSymLinkCreated(newLinkRelativePath)); + } + public void OnFileDeleted(string relativePath) { this.backgroundFileSystemTaskRunner.Enqueue(FileSystemTask.OnFileDeleted(relativePath)); @@ -615,6 +620,7 @@ private FileSystemTaskResult ExecuteBackgroundOperation(FileSystemTask gitUpdate case FileSystemTask.OperationType.OnFileCreated: case FileSystemTask.OperationType.OnFailedPlaceholderDelete: case FileSystemTask.OperationType.OnFileHardLinkCreated: + case FileSystemTask.OperationType.OnFileSymLinkCreated: metadata.Add("virtualPath", gitUpdate.VirtualPath); result = this.AddModifiedPathAndRemoveFromPlaceholderList(gitUpdate.VirtualPath); break; diff --git a/GVFS/GVFS.Virtualization/Projection/GitIndexProjection.GitIndexParser.cs b/GVFS/GVFS.Virtualization/Projection/GitIndexProjection.GitIndexParser.cs index cd0cc2031..ca5f88dad 100644 --- a/GVFS/GVFS.Virtualization/Projection/GitIndexProjection.GitIndexParser.cs +++ b/GVFS/GVFS.Virtualization/Projection/GitIndexProjection.GitIndexParser.cs @@ -210,7 +210,7 @@ private FileSystemTaskResult ParseIndex(ITracer tracer, Stream indexStream, Func break; default: - throw new InvalidDataException($"Invalid object type {type:X} found in index"); + throw new InvalidDataException($"Invalid file type {type:X} found in index"); } this.resuableParsedIndexEntry.FileTypeAndMode = typeAndMode; diff --git a/ProjFS.Mac/PrjFSLib.Mac.Managed/Interop/PrjFSLib.cs b/ProjFS.Mac/PrjFSLib.Mac.Managed/Interop/PrjFSLib.cs index 6edba146f..786426139 100644 --- a/ProjFS.Mac/PrjFSLib.Mac.Managed/Interop/PrjFSLib.cs +++ b/ProjFS.Mac/PrjFSLib.Mac.Managed/Interop/PrjFSLib.cs @@ -32,8 +32,13 @@ public static extern Result WritePlaceholderFile( ulong fileSize, ushort fileMode); + [DllImport(PrjFSLibPath, EntryPoint = "PrjFS_WriteSymLink")] + public static extern Result WriteSymLink( + string relativePath, + string symLinkContents); + [DllImport(PrjFSLibPath, EntryPoint = "PrjFS_UpdatePlaceholderFileIfNeeded")] - public static extern Result UpdatePlaceholderFileIfNeeded( + public static extern Result UpdatePlaceholderFileIfNeeded( string relativePath, [MarshalAs(UnmanagedType.LPArray, SizeConst = PlaceholderIdLength)] byte[] providerId, @@ -44,6 +49,13 @@ public static extern Result UpdatePlaceholderFileIfNeeded( UpdateType updateType, ref UpdateFailureCause failureCause); + [DllImport(PrjFSLibPath, EntryPoint = "PrjFS_ReplacePlaceholderFileWithSymLink")] + public static extern Result ReplacePlaceholderFileWithSymLink( + string relativePath, + string symLinkContents, + UpdateType updateType, + ref UpdateFailureCause failureCause); + [DllImport(PrjFSLibPath, EntryPoint = "PrjFS_DeleteFile")] public static extern Result DeleteFile( string relativePath, diff --git a/ProjFS.Mac/PrjFSLib.Mac.Managed/VirtualizationInstance.cs b/ProjFS.Mac/PrjFSLib.Mac.Managed/VirtualizationInstance.cs index 3b7cf74b4..8a2ef8a72 100644 --- a/ProjFS.Mac/PrjFSLib.Mac.Managed/VirtualizationInstance.cs +++ b/ProjFS.Mac/PrjFSLib.Mac.Managed/VirtualizationInstance.cs @@ -108,6 +108,13 @@ public virtual Result WritePlaceholderFile( fileMode); } + public virtual Result WriteSymLink( + string relativePath, + string symLinkContents) + { + return Interop.PrjFSLib.WriteSymLink(relativePath, symLinkContents); + } + public virtual Result UpdatePlaceholderIfNeeded( string relativePath, byte[] providerId, @@ -137,6 +144,23 @@ public virtual Result UpdatePlaceholderIfNeeded( return result; } + public virtual Result ReplacePlaceholderFileWithSymLink( + string relativePath, + string symLinkContents, + UpdateType updateFlags, + out UpdateFailureCause failureCause) + { + UpdateFailureCause updateFailureCause = UpdateFailureCause.NoFailure; + Result result = Interop.PrjFSLib.ReplacePlaceholderFileWithSymLink( + relativePath, + symLinkContents, + updateFlags, + ref updateFailureCause); + + failureCause = updateFailureCause; + return result; + } + public virtual Result CompleteCommand( ulong commandId, Result result) diff --git a/ProjFS.Mac/PrjFSLib/PrjFSLib.cpp b/ProjFS.Mac/PrjFSLib/PrjFSLib.cpp index a858a7fe3..755d0fc9b 100644 --- a/ProjFS.Mac/PrjFSLib/PrjFSLib.cpp +++ b/ProjFS.Mac/PrjFSLib/PrjFSLib.cpp @@ -355,6 +355,38 @@ PrjFS_Result PrjFS_WritePlaceholderFile( return PrjFS_Result_EIOError; } +PrjFS_Result PrjFS_WriteSymLink( + _In_ const char* relativePath, + _In_ const char* symLinkContents) +{ +#ifdef DEBUG + std::cout + << "PrjFS_WriteSymLink(" + << relativePath << ", " + << symLinkContents << ")" << std::endl; +#endif + + if (nullptr == relativePath || nullptr == symLinkContents) + { + return PrjFS_Result_EInvalidArgs; + } + + char fullPath[PrjFSMaxPath]; + CombinePaths(s_virtualizationRootFullPath.c_str(), relativePath, fullPath); + + if(symlink(symLinkContents, fullPath)) + { + goto CleanupAndFail; + } + + return PrjFS_Result_Success; + +CleanupAndFail: + + return PrjFS_Result_EIOError; + +} + PrjFS_Result PrjFS_UpdatePlaceholderFileIfNeeded( _In_ const char* relativePath, _In_ unsigned char providerId[PrjFS_PlaceholderIdLength], @@ -387,6 +419,29 @@ PrjFS_Result PrjFS_UpdatePlaceholderFileIfNeeded( return PrjFS_WritePlaceholderFile(relativePath, providerId, contentId, fileSize, fileMode); } +PrjFS_Result PrjFS_ReplacePlaceholderFileWithSymLink( + _In_ const char* relativePath, + _In_ const char* symLinkContents, + _In_ PrjFS_UpdateType updateFlags, + _Out_ PrjFS_UpdateFailureCause* failureCause) +{ +#ifdef DEBUG + std::cout + << "PrjFS_ReplacePlaceholderFileWithSymLink(" + << relativePath << ", " + << symLinkContents << ", " + << std::hex << updateFlags << std::dec << ")" << std::endl; +#endif + + PrjFS_Result result = PrjFS_DeleteFile(relativePath, updateFlags, failureCause); + if (result != PrjFS_Result_Success) + { + return result; + } + + return PrjFS_WriteSymLink(relativePath, symLinkContents); +} + PrjFS_Result PrjFS_DeleteFile( _In_ const char* relativePath, _In_ PrjFS_UpdateType updateFlags, diff --git a/ProjFS.Mac/PrjFSLib/PrjFSLib.h b/ProjFS.Mac/PrjFSLib/PrjFSLib.h index 6e7a49a0b..5c7eff983 100644 --- a/ProjFS.Mac/PrjFSLib/PrjFSLib.h +++ b/ProjFS.Mac/PrjFSLib/PrjFSLib.h @@ -85,6 +85,10 @@ extern "C" PrjFS_Result PrjFS_WritePlaceholderFile( _In_ unsigned long fileSize, _In_ uint16_t fileMode); +extern "C" PrjFS_Result PrjFS_WriteSymLink( + _In_ const char* relativePath, + _In_ const char* symLinkContents); + typedef enum { PrjFS_UpdateType_Invalid = 0x00000000, @@ -111,6 +115,12 @@ extern "C" PrjFS_Result PrjFS_UpdatePlaceholderFileIfNeeded( _In_ PrjFS_UpdateType updateFlags, _Out_ PrjFS_UpdateFailureCause* failureCause); +extern "C" PrjFS_Result PrjFS_ReplacePlaceholderFileWithSymLink( + _In_ const char* relativePath, + _In_ const char* symLinkContents, + _In_ PrjFS_UpdateType updateFlags, + _Out_ PrjFS_UpdateFailureCause* failureCause); + extern "C" PrjFS_Result PrjFS_DeleteFile( _In_ const char* relativePath, _In_ PrjFS_UpdateType updateFlags, From 64feb7c04c66bcd920ca15e2926e63c340e5d479 Mon Sep 17 00:00:00 2001 From: Yehezkel Bernat Date: Tue, 25 Sep 2018 17:46:25 +0300 Subject: [PATCH 149/272] trivial: Fix a typo in error messages --- GVFS/GVFS/CommandLine/ServiceVerb.cs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/GVFS/GVFS/CommandLine/ServiceVerb.cs b/GVFS/GVFS/CommandLine/ServiceVerb.cs index f440206ac..13b4a5689 100644 --- a/GVFS/GVFS/CommandLine/ServiceVerb.cs +++ b/GVFS/GVFS/CommandLine/ServiceVerb.cs @@ -51,11 +51,11 @@ public override void Execute() int optionCount = new[] { this.MountAll, this.UnmountAll, this.List }.Count(flag => flag); if (optionCount == 0) { - this.ReportErrorAndExit("Error: You must specify an argument. Run 'gvfs serivce --help' for details."); + this.ReportErrorAndExit($"Error: You must specify an argument. Run 'gvfs {ServiceVerbName} --help' for details."); } else if (optionCount > 1) { - this.ReportErrorAndExit("Error: You cannot specify multiple arguments. Run 'gvfs serivce --help' for details."); + this.ReportErrorAndExit($"Error: You cannot specify multiple arguments. Run 'gvfs {ServiceVerbName} --help' for details."); } string errorMessage; From 3f8fd9e44a4b31fa328f2abd3bb6bd7358f38659 Mon Sep 17 00:00:00 2001 From: Kevin Willford Date: Wed, 22 Aug 2018 12:38:04 -0600 Subject: [PATCH 150/272] Add tests for removing entries from the modified files database --- .../Windows/Tests/DiskLayoutUpgradeTests.cs | 42 ++++--- .../ModifiedPathsTests.cs | 105 ++++++++++++++---- 2 files changed, 108 insertions(+), 39 deletions(-) diff --git a/GVFS/GVFS.FunctionalTests.Windows/Windows/Tests/DiskLayoutUpgradeTests.cs b/GVFS/GVFS.FunctionalTests.Windows/Windows/Tests/DiskLayoutUpgradeTests.cs index d3ea1f93b..eb1ce473d 100644 --- a/GVFS/GVFS.FunctionalTests.Windows/Windows/Tests/DiskLayoutUpgradeTests.cs +++ b/GVFS/GVFS.FunctionalTests.Windows/Windows/Tests/DiskLayoutUpgradeTests.cs @@ -324,24 +324,30 @@ public void MountCreatesModifiedPathsDatabase() this.Enlistment.MountGVFS(); this.Enlistment.UnmountGVFS(); - string expectedModifiedPaths = @"A .gitattributes -A developer/me/ -A developer/me/JLANGE9._prerazzle -A developer/me/StateSwitch.Save -A tools/x86/remote.exe -A tools/x86/runelevated.exe -A tools/amd64/remote.exe -A tools/amd64/runelevated.exe -A tools/perllib/MS/TraceLogging.dll -A tools/managed/v2.0/midldd.CheckedInExe -A tools/managed/v4.0/sdapi.dll -A tools/managed/v2.0/midlpars.dll -A tools/managed/v2.0/RPCDataSupport.dll -A tools/managed/v2.0/MidlStaticAnalysis.dll -A tools/perllib/MS/Somefile.txt -"; - - modifiedPathsDatabasePath.ShouldBeAFile(this.fileSystem).WithContents(expectedModifiedPaths); + string[] expectedModifiedPaths = + { + "A .gitattributes", + "A developer/me/", + "A developer/me/JLANGE9._prerazzle", + "A developer/me/StateSwitch.Save", + "A tools/x86/remote.exe", + "A tools/x86/runelevated.exe", + "A tools/amd64/remote.exe", + "A tools/amd64/runelevated.exe", + "A tools/perllib/MS/TraceLogging.dll", + "A tools/managed/v2.0/midldd.CheckedInExe", + "A tools/managed/v4.0/sdapi.dll", + "A tools/managed/v2.0/midlpars.dll", + "A tools/managed/v2.0/RPCDataSupport.dll", + "A tools/managed/v2.0/MidlStaticAnalysis.dll", + "A tools/perllib/MS/Somefile.txt", + }; + + modifiedPathsDatabasePath.ShouldBeAFile(this.fileSystem); + this.fileSystem.ReadAllText(modifiedPathsDatabasePath) + .Split(new[] { '\r', '\n' }, StringSplitOptions.RemoveEmptyEntries) + .OrderBy(x => x) + .ShouldMatchInOrder(expectedModifiedPaths.OrderBy(x => x)); this.ValidatePersistedVersionMatchesCurrentVersion(); } diff --git a/GVFS/GVFS.FunctionalTests/Tests/EnlistmentPerTestCase/ModifiedPathsTests.cs b/GVFS/GVFS.FunctionalTests/Tests/EnlistmentPerTestCase/ModifiedPathsTests.cs index c57bffccd..62fa2a9f9 100644 --- a/GVFS/GVFS.FunctionalTests/Tests/EnlistmentPerTestCase/ModifiedPathsTests.cs +++ b/GVFS/GVFS.FunctionalTests/Tests/EnlistmentPerTestCase/ModifiedPathsTests.cs @@ -3,7 +3,9 @@ using GVFS.FunctionalTests.Tools; using GVFS.Tests.Should; using NUnit.Framework; +using System; using System.IO; +using System.Linq; namespace GVFS.FunctionalTests.Tests.EnlistmentPerTestCase { @@ -23,26 +25,64 @@ public class ModifiedPathsTests : TestsWithEnlistmentPerTestCase private static readonly string FileToCreateOutsideRepo = $"{nameof(ModifiedPathsTests)}_outsideRepo.txt"; private static readonly string FolderToCreateOutsideRepo = $"{nameof(ModifiedPathsTests)}_outsideFolder"; private static readonly string FolderToDelete = "Scripts"; - private static readonly string ExpectedModifiedFilesContentsAfterRemount = -$@"A .gitattributes -A {GVFSHelpers.ConvertPathToGitFormat(FileToAdd)} -A {GVFSHelpers.ConvertPathToGitFormat(FileToUpdate)} -A {FileToDelete} -A {GVFSHelpers.ConvertPathToGitFormat(FileToRename)} -A {GVFSHelpers.ConvertPathToGitFormat(RenameFileTarget)} -A {FolderToCreate}/ -A {FolderToRename}/ -A {RenameFolderTarget}/ -A {RenameNewDotGitFileTarget} -A {FileToCreateOutsideRepo} -A {FolderToCreateOutsideRepo}/ -A {FolderToDelete}/CreateCommonAssemblyVersion.bat -A {FolderToDelete}/CreateCommonCliAssemblyVersion.bat -A {FolderToDelete}/CreateCommonVersionHeader.bat -A {FolderToDelete}/RunFunctionalTests.bat -A {FolderToDelete}/RunUnitTests.bat -A {FolderToDelete}/ -"; + private static readonly string[] ExpectedModifiedFilesContentsAfterRemount = + { + $"A .gitattributes", + $"A {GVFSHelpers.ConvertPathToGitFormat(FileToAdd)}", + $"A {GVFSHelpers.ConvertPathToGitFormat(FileToUpdate)}", + $"A {FileToDelete}", + $"A {GVFSHelpers.ConvertPathToGitFormat(FileToRename)}", + $"A {GVFSHelpers.ConvertPathToGitFormat(RenameFileTarget)}", + $"A {FolderToCreate}/", + $"A {FolderToRename}/", + $"A {RenameFolderTarget}/", + $"A {RenameNewDotGitFileTarget}", + $"A {FileToCreateOutsideRepo}", + $"A {FolderToCreateOutsideRepo}/", + $"A {FolderToDelete}/CreateCommonAssemblyVersion.bat", + $"A {FolderToDelete}/CreateCommonCliAssemblyVersion.bat", + $"A {FolderToDelete}/CreateCommonVersionHeader.bat", + $"A {FolderToDelete}/RunFunctionalTests.bat", + $"A {FolderToDelete}/RunUnitTests.bat", + $"A {FolderToDelete}/", + }; + + [TestCaseSource(typeof(FileSystemRunner), FileSystemRunner.TestRunners)] + public void DeletedTempFileIsRemovedFromModifiedFiles(FileSystemRunner fileSystem) + { + string tempFile = this.CreateFile(fileSystem, "temp.txt"); + fileSystem.DeleteFile(tempFile); + tempFile.ShouldNotExistOnDisk(fileSystem); + + this.Enlistment.UnmountGVFS(); + this.ValidateModifiedPathsDoNotContain(fileSystem, "temp.txt"); + } + + [TestCaseSource(typeof(FileSystemRunner), FileSystemRunner.TestRunners)] + public void DeletedTempFolderIsRemovedFromModifiedFiles(FileSystemRunner fileSystem) + { + string tempFolder = this.CreateDirectory(fileSystem, "Temp"); + fileSystem.DeleteDirectory(tempFolder); + tempFolder.ShouldNotExistOnDisk(fileSystem); + + this.Enlistment.UnmountGVFS(); + this.ValidateModifiedPathsDoNotContain(fileSystem, "Temp/"); + } + + [TestCaseSource(typeof(FileSystemRunner), FileSystemRunner.TestRunners)] + public void DeletedTempFolderDeletesFilesFromModifiedFiles(FileSystemRunner fileSystem) + { + string tempFolder = this.CreateDirectory(fileSystem, "Temp"); + string tempFile1 = this.CreateFile(fileSystem, Path.Combine("Temp", "temp1.txt")); + string tempFile2 = this.CreateFile(fileSystem, Path.Combine("Temp", "temp2.txt")); + fileSystem.DeleteDirectory(tempFolder); + tempFolder.ShouldNotExistOnDisk(fileSystem); + tempFile1.ShouldNotExistOnDisk(fileSystem); + tempFile2.ShouldNotExistOnDisk(fileSystem); + + this.Enlistment.UnmountGVFS(); + this.ValidateModifiedPathsDoNotContain(fileSystem, "Temp/", "Temp/temp1.txt", "Temp/temp2.txt"); + } [Category(Categories.MacTODO.M2)] [TestCaseSource(typeof(FileSystemRunner), FileSystemRunner.TestRunners)] @@ -114,7 +154,8 @@ public void ModifiedPathsSavedAfterRemount(FileSystemRunner fileSystem) modifiedPathsDatabase.ShouldBeAFile(fileSystem); using (StreamReader reader = new StreamReader(File.Open(modifiedPathsDatabase, FileMode.Open, FileAccess.Read, FileShare.ReadWrite))) { - reader.ReadToEnd().ShouldEqual(ExpectedModifiedFilesContentsAfterRemount); + reader.ReadToEnd().Split(new char[] { '\r', '\n' }, StringSplitOptions.RemoveEmptyEntries).OrderBy(x => x) + .ShouldMatchInOrder(ExpectedModifiedFilesContentsAfterRemount.OrderBy(x => x)); } } @@ -160,5 +201,27 @@ A LinkToFileOutsideSrc.txt reader.ReadToEnd().ShouldEqual(ExpectedModifiedFilesContentsAfterHardlinks); } } + + private string CreateDirectory(FileSystemRunner fileSystem, string relativePath) + { + string tempFolder = this.Enlistment.GetVirtualPathTo(relativePath); + fileSystem.CreateDirectory(tempFolder); + tempFolder.ShouldBeADirectory(fileSystem); + return tempFolder; + } + + private string CreateFile(FileSystemRunner fileSystem, string relativePath) + { + string tempFile = this.Enlistment.GetVirtualPathTo(relativePath); + fileSystem.WriteAllText(tempFile, $"Contents for the {relativePath} file"); + tempFile.ShouldBeAFile(fileSystem); + return tempFile; + } + + private void ValidateModifiedPathsDoNotContain(FileSystemRunner fileSystem, params string[] paths) + { + GVFSHelpers.ModifiedPathsShouldNotContain(fileSystem, this.Enlistment.DotGVFSRoot, paths.Select(x => $"A {x}" + Environment.NewLine).ToArray()); + GVFSHelpers.ModifiedPathsShouldNotContain(fileSystem, this.Enlistment.DotGVFSRoot, paths.Select(x => $"D {x}" + Environment.NewLine).ToArray()); + } } } From a0f9b408c228b89fb7d6e5000d914bd14fa6408f Mon Sep 17 00:00:00 2001 From: Kevin Willford Date: Tue, 21 Aug 2018 11:00:38 -0600 Subject: [PATCH 151/272] Add TryRemove method to the ModifiedPathsDatabase --- GVFS/GVFS.Common/ModifiedPathsDatabase.cs | 99 ++++++++++++++++++----- 1 file changed, 80 insertions(+), 19 deletions(-) diff --git a/GVFS/GVFS.Common/ModifiedPathsDatabase.cs b/GVFS/GVFS.Common/ModifiedPathsDatabase.cs index 043a1e3bc..409b36a3c 100644 --- a/GVFS/GVFS.Common/ModifiedPathsDatabase.cs +++ b/GVFS/GVFS.Common/ModifiedPathsDatabase.cs @@ -75,38 +75,99 @@ public bool TryAdd(string path, bool isFolder, out bool isRetryable) } catch (IOException e) { - if (this.Tracer != null) - { - EventMetadata metadata = new EventMetadata(); - metadata.Add("Area", "ModifiedPathsDatabase"); - metadata.Add(nameof(entry), entry); - metadata.Add(nameof(isFolder), isFolder); - metadata.Add("Exception", e.ToString()); - this.Tracer.RelatedWarning(metadata, $"IOException caught while processing {nameof(this.TryAdd)}"); - } - + this.TraceWarning(isFolder, entry, e, nameof(this.TryAdd)); return false; } catch (Exception e) { - if (this.Tracer != null) - { - EventMetadata metadata = new EventMetadata(); - metadata.Add("Area", "ModifiedPathsDatabase"); - metadata.Add(nameof(entry), entry); - metadata.Add(nameof(isFolder), isFolder); - metadata.Add("Exception", e.ToString()); - this.Tracer.RelatedError(metadata, $"Exception caught while processing {nameof(this.TryAdd)}"); - } + this.TraceError(isFolder, entry, e, nameof(this.TryAdd)); + isRetryable = false; + return false; + } + + return true; + } + return true; + } + + public bool TryRemove(string path, bool isFolder, out bool isRetryable) + { + isRetryable = true; + string entry = this.NormalizeEntryString(path, isFolder); + if (this.modifiedPaths.Contains(entry)) + { + isRetryable = true; + try + { + this.WriteRemoveEntry(entry, () => this.modifiedPaths.TryRemove(entry)); + } + catch (IOException e) + { + this.TraceWarning(isFolder, entry, e, nameof(this.TryRemove)); + return false; + } + catch (Exception e) + { + this.TraceError(isFolder, entry, e, nameof(this.TryRemove)); isRetryable = false; return false; } + + return true; } return true; } + public void WriteAllEntriesAndFlush() + { + try + { + this.WriteAndReplaceDataFile(this.GenerateDataLines); + } + catch (Exception e) + { + throw new FileBasedCollectionException(e); + } + } + + private static EventMetadata CreateEventMetadata(bool isFolder, string entry, Exception e) + { + EventMetadata metadata = new EventMetadata(); + metadata.Add("Area", "ModifiedPathsDatabase"); + metadata.Add(nameof(entry), entry); + metadata.Add(nameof(isFolder), isFolder); + metadata.Add("Exception", e.ToString()); + return metadata; + } + + private IEnumerable GenerateDataLines() + { + foreach (string entry in this.modifiedPaths) + { + yield return this.FormatAddLine(entry); + } + } + + private void TraceWarning(bool isFolder, string entry, Exception e, string method) + { + if (this.Tracer != null) + { + EventMetadata metadata = CreateEventMetadata(isFolder, entry, e); + this.Tracer.RelatedWarning(metadata, $"{e.GetType().Name} caught while processing {method}"); + } + } + + private void TraceError(bool isFolder, string entry, Exception e, string method) + { + if (this.Tracer != null) + { + EventMetadata metadata = CreateEventMetadata(isFolder, entry, e); + this.Tracer.RelatedError(metadata, $"{e.GetType().Name} caught while processing {method}"); + } + } + private bool TryParseAddLine(string line, out string key, out string value, out string error) { key = line; From ae5f9bace1a4da28fea769761d3cfc80c610753e Mon Sep 17 00:00:00 2001 From: Kevin Willford Date: Wed, 5 Sep 2018 14:28:36 -0600 Subject: [PATCH 152/272] Handle the file system callbacks to track created then deleted files --- .../FileSystemCallbacks.cs | 46 ++++++++++++++++++- 1 file changed, 44 insertions(+), 2 deletions(-) diff --git a/GVFS/GVFS.Virtualization/FileSystemCallbacks.cs b/GVFS/GVFS.Virtualization/FileSystemCallbacks.cs index a9ea5200e..d954ae6c4 100644 --- a/GVFS/GVFS.Virtualization/FileSystemCallbacks.cs +++ b/GVFS/GVFS.Virtualization/FileSystemCallbacks.cs @@ -33,6 +33,7 @@ public class FileSystemCallbacks : IDisposable, IHeartBeatMetadataProvider private GVFSContext context; private GVFSGitObjects gitObjects; private ModifiedPathsDatabase modifiedPaths; + private ConcurrentHashSet newlyCreatedFileAndFolderPaths; private ConcurrentDictionary placeHolderCreationCount; private BackgroundFileSystemTaskRunner backgroundFileSystemTaskRunner; private FileSystemVirtualizer fileSystemVirtualizer; @@ -75,6 +76,7 @@ public FileSystemCallbacks( this.fileSystemVirtualizer = fileSystemVirtualizer; this.placeHolderCreationCount = new ConcurrentDictionary(StringComparer.OrdinalIgnoreCase); + this.newlyCreatedFileAndFolderPaths = new ConcurrentHashSet(StringComparer.OrdinalIgnoreCase); string error; if (!ModifiedPathsDatabase.TryLoadOrCreate( @@ -364,6 +366,8 @@ public virtual void OnIndexFileChange() this.GitIndexProjection.InvalidateProjection(); this.InvalidateGitStatusCache(); } + + this.newlyCreatedFileAndFolderPaths.Clear(); } public void InvalidateGitStatusCache() @@ -400,6 +404,7 @@ public void OnExcludeFileChanged() public void OnFileCreated(string relativePath) { + this.newlyCreatedFileAndFolderPaths.Add(relativePath); this.backgroundFileSystemTaskRunner.Enqueue(FileSystemTask.OnFileCreated(relativePath)); } @@ -435,6 +440,7 @@ public void OnFileDeleted(string relativePath) public void OnFolderCreated(string relativePath) { + this.newlyCreatedFileAndFolderPaths.Add(relativePath); this.backgroundFileSystemTaskRunner.Enqueue(FileSystemTask.OnFolderCreated(relativePath)); } @@ -639,7 +645,15 @@ private FileSystemTaskResult ExecuteBackgroundOperation(FileSystemTask gitUpdate case FileSystemTask.OperationType.OnFileDeleted: metadata.Add("virtualPath", gitUpdate.VirtualPath); - result = this.AddModifiedPathAndRemoveFromPlaceholderList(gitUpdate.VirtualPath); + if (this.newlyCreatedFileAndFolderPaths.Contains(gitUpdate.VirtualPath)) + { + result = this.TryRemoveModifiedPath(gitUpdate.VirtualPath, isFolder: false); + } + else + { + result = this.AddModifiedPathAndRemoveFromPlaceholderList(gitUpdate.VirtualPath); + } + break; case FileSystemTask.OperationType.OnFileOverwritten: @@ -736,7 +750,15 @@ private FileSystemTaskResult ExecuteBackgroundOperation(FileSystemTask gitUpdate case FileSystemTask.OperationType.OnFolderDeleted: metadata.Add("virtualPath", gitUpdate.VirtualPath); - result = this.TryAddModifiedPath(gitUpdate.VirtualPath, isFolder: true); + if (this.newlyCreatedFileAndFolderPaths.Contains(gitUpdate.VirtualPath)) + { + result = this.TryRemoveModifiedPath(gitUpdate.VirtualPath, isFolder: true); + } + else + { + result = this.TryAddModifiedPath(gitUpdate.VirtualPath, isFolder: true); + } + break; case FileSystemTask.OperationType.OnFolderFirstWrite: @@ -768,6 +790,26 @@ private FileSystemTaskResult ExecuteBackgroundOperation(FileSystemTask gitUpdate return result; } + private FileSystemTaskResult TryRemoveModifiedPath(string virtualPath, bool isFolder) + { + string fullPathToItem = Path.Combine(this.context.Enlistment.WorkingDirectoryRoot, virtualPath); + if ((isFolder && this.context.FileSystem.DirectoryExists(fullPathToItem)) || + (!isFolder && this.context.FileSystem.FileExists(fullPathToItem))) + { + return FileSystemTaskResult.Success; + } + + if (!this.modifiedPaths.TryRemove(virtualPath, isFolder, out bool isRetryable)) + { + return isRetryable ? FileSystemTaskResult.RetryableError : FileSystemTaskResult.FatalError; + } + + this.newlyCreatedFileAndFolderPaths.TryRemove(virtualPath); + + this.InvalidateGitStatusCache(); + return FileSystemTaskResult.Success; + } + private FileSystemTaskResult TryAddModifiedPath(string virtualPath, bool isFolder) { if (!this.modifiedPaths.TryAdd(virtualPath, isFolder, out bool isRetryable)) From b50dd1a0396042229940beb790493ad61799364f Mon Sep 17 00:00:00 2001 From: Kevin Willford Date: Thu, 6 Sep 2018 13:03:05 -0600 Subject: [PATCH 153/272] Add handling of rename of files for modified paths cleanup --- GVFS/GVFS.Virtualization/FileSystemCallbacks.cs | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/GVFS/GVFS.Virtualization/FileSystemCallbacks.cs b/GVFS/GVFS.Virtualization/FileSystemCallbacks.cs index d954ae6c4..de5541d4f 100644 --- a/GVFS/GVFS.Virtualization/FileSystemCallbacks.cs +++ b/GVFS/GVFS.Virtualization/FileSystemCallbacks.cs @@ -425,6 +425,7 @@ public void OnFileConvertedToFull(string relativePath) public virtual void OnFileRenamed(string oldRelativePath, string newRelativePath) { + this.newlyCreatedFileAndFolderPaths.Add(newRelativePath); this.backgroundFileSystemTaskRunner.Enqueue(FileSystemTask.OnFileRenamed(oldRelativePath, newRelativePath)); } @@ -631,7 +632,14 @@ private FileSystemTaskResult ExecuteBackgroundOperation(FileSystemTask gitUpdate result = FileSystemTaskResult.Success; if (!string.IsNullOrEmpty(gitUpdate.OldVirtualPath) && !IsPathInsideDotGit(gitUpdate.OldVirtualPath)) { - result = this.AddModifiedPathAndRemoveFromPlaceholderList(gitUpdate.OldVirtualPath); + if (this.newlyCreatedFileAndFolderPaths.Contains(gitUpdate.OldVirtualPath)) + { + result = this.TryRemoveModifiedPath(gitUpdate.OldVirtualPath, isFolder: false); + } + else + { + result = this.AddModifiedPathAndRemoveFromPlaceholderList(gitUpdate.OldVirtualPath); + } } if (result == FileSystemTaskResult.Success && @@ -844,7 +852,7 @@ private FileSystemTaskResult AddModifiedPathAndRemoveFromPlaceholderList(string private FileSystemTaskResult PostBackgroundOperation() { - this.modifiedPaths.ForceFlush(); + this.modifiedPaths.WriteAllEntriesAndFlush(); this.gitStatusCache.RefreshAsynchronously(); return this.GitIndexProjection.CloseIndex(); } From 848e6d38fb4840aa1f715fa5b821763f9211a949 Mon Sep 17 00:00:00 2001 From: Kevin Willford Date: Fri, 7 Sep 2018 12:37:43 -0600 Subject: [PATCH 154/272] Handle cleaning up modified paths when a folder is renamed --- .../FileSystemCallbacks.cs | 28 +++++++++++++++---- 1 file changed, 22 insertions(+), 6 deletions(-) diff --git a/GVFS/GVFS.Virtualization/FileSystemCallbacks.cs b/GVFS/GVFS.Virtualization/FileSystemCallbacks.cs index de5541d4f..c640d41a0 100644 --- a/GVFS/GVFS.Virtualization/FileSystemCallbacks.cs +++ b/GVFS/GVFS.Virtualization/FileSystemCallbacks.cs @@ -691,10 +691,20 @@ private FileSystemTaskResult ExecuteBackgroundOperation(FileSystemTask gitUpdate result = this.TryAddModifiedPath(gitUpdate.VirtualPath, isFolder: true); if (result == FileSystemTaskResult.Success) { + if (!this.newlyCreatedFileAndFolderPaths.Contains(gitUpdate.VirtualPath)) + { + this.newlyCreatedFileAndFolderPaths.Add(gitUpdate.VirtualPath); + } + + if (this.newlyCreatedFileAndFolderPaths.Contains(gitUpdate.OldVirtualPath)) + { + this.TryRemoveModifiedPath(gitUpdate.OldVirtualPath, isFolder: true); + } + Queue relativeFolderPaths = new Queue(); relativeFolderPaths.Enqueue(gitUpdate.VirtualPath); - // Add all the files in the renamed folder to the always_exclude file + // Remove old paths from modified paths if in the newly created list while (relativeFolderPaths.Count > 0) { string folderPath = relativeFolderPaths.Dequeue(); @@ -705,14 +715,20 @@ private FileSystemTaskResult ExecuteBackgroundOperation(FileSystemTask gitUpdate foreach (DirectoryItemInfo itemInfo in this.context.FileSystem.ItemsInDirectory(Path.Combine(this.context.Enlistment.WorkingDirectoryRoot, folderPath))) { string itemVirtualPath = Path.Combine(folderPath, itemInfo.Name); - if (itemInfo.IsDirectory) + string oldItemVirtualPath = gitUpdate.OldVirtualPath + itemVirtualPath.Substring(gitUpdate.VirtualPath.Length); + if (!this.newlyCreatedFileAndFolderPaths.Contains(itemVirtualPath)) { - relativeFolderPaths.Enqueue(itemVirtualPath); + this.newlyCreatedFileAndFolderPaths.Add(itemVirtualPath); } - else + + if (this.newlyCreatedFileAndFolderPaths.Contains(oldItemVirtualPath)) { - string oldItemVirtualPath = gitUpdate.OldVirtualPath + itemVirtualPath.Substring(gitUpdate.VirtualPath.Length); - result = this.TryAddModifiedPath(itemVirtualPath, isFolder: false); + this.TryRemoveModifiedPath(oldItemVirtualPath, isFolder: itemInfo.IsDirectory); + } + + if (itemInfo.IsDirectory) + { + relativeFolderPaths.Enqueue(itemVirtualPath); } } } From c189b2c356dcd3e279cbfd7024a43c66935fde18 Mon Sep 17 00:00:00 2001 From: Kevin Willford Date: Fri, 7 Sep 2018 22:09:03 -0600 Subject: [PATCH 155/272] Add null check when creating trace metadata using and exception --- GVFS/GVFS.Common/ModifiedPathsDatabase.cs | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/GVFS/GVFS.Common/ModifiedPathsDatabase.cs b/GVFS/GVFS.Common/ModifiedPathsDatabase.cs index 409b36a3c..945613115 100644 --- a/GVFS/GVFS.Common/ModifiedPathsDatabase.cs +++ b/GVFS/GVFS.Common/ModifiedPathsDatabase.cs @@ -138,7 +138,11 @@ private static EventMetadata CreateEventMetadata(bool isFolder, string entry, Ex metadata.Add("Area", "ModifiedPathsDatabase"); metadata.Add(nameof(entry), entry); metadata.Add(nameof(isFolder), isFolder); - metadata.Add("Exception", e.ToString()); + if (e != null) + { + metadata.Add("Exception", e.ToString()); + } + return metadata; } From 1772519160fefa052275c7238875f918f4fbbf7b Mon Sep 17 00:00:00 2001 From: Kevin Willford Date: Tue, 11 Sep 2018 13:45:24 -0600 Subject: [PATCH 156/272] Fix broken tests --- .../Tests/EnlistmentPerFixture/GitFilesTests.cs | 10 ++-------- .../Tests/EnlistmentPerTestCase/ModifiedPathsTests.cs | 2 +- .../Tests/GitCommands/GitCommandsTests.cs | 2 +- 3 files changed, 4 insertions(+), 10 deletions(-) diff --git a/GVFS/GVFS.FunctionalTests/Tests/EnlistmentPerFixture/GitFilesTests.cs b/GVFS/GVFS.FunctionalTests/Tests/EnlistmentPerFixture/GitFilesTests.cs index eca57a1c2..88dbf1709 100644 --- a/GVFS/GVFS.FunctionalTests/Tests/EnlistmentPerFixture/GitFilesTests.cs +++ b/GVFS/GVFS.FunctionalTests/Tests/EnlistmentPerFixture/GitFilesTests.cs @@ -88,7 +88,6 @@ public void RenameEmptyFolderTest() string renamedFolderName = "folder3b"; string[] expectedModifiedEntries = { - folderName + "/", renamedFolderName + "/", }; @@ -110,19 +109,14 @@ public void RenameFolderTest() string[] fileNames = { "a", "b", "c" }; string[] expectedModifiedEntries = { - renamedFolderName + "/" + fileNames[0], - renamedFolderName + "/" + fileNames[1], - renamedFolderName + "/" + fileNames[2], - folderName + "/" + fileNames[0], - folderName + "/" + fileNames[1], - folderName + "/" + fileNames[2], + renamedFolderName + "/", }; this.Enlistment.GetVirtualPathTo(folderName).ShouldNotExistOnDisk(this.fileSystem); this.fileSystem.CreateDirectory(this.Enlistment.GetVirtualPathTo(folderName)); foreach (string fileName in fileNames) { - string filePath = folderName + "\\" + fileName; + string filePath = Path.Combine(folderName, fileName); this.fileSystem.CreateEmptyFile(this.Enlistment.GetVirtualPathTo(filePath)); this.Enlistment.GetVirtualPathTo(filePath).ShouldBeAFile(this.fileSystem); } diff --git a/GVFS/GVFS.FunctionalTests/Tests/EnlistmentPerTestCase/ModifiedPathsTests.cs b/GVFS/GVFS.FunctionalTests/Tests/EnlistmentPerTestCase/ModifiedPathsTests.cs index 62fa2a9f9..14fe1a5be 100644 --- a/GVFS/GVFS.FunctionalTests/Tests/EnlistmentPerTestCase/ModifiedPathsTests.cs +++ b/GVFS/GVFS.FunctionalTests/Tests/EnlistmentPerTestCase/ModifiedPathsTests.cs @@ -109,7 +109,7 @@ public void ModifiedPathsSavedAfterRemount(FileSystemRunner fileSystem) string folderToRenameTarget = this.Enlistment.GetVirtualPathTo(RenameFolderTarget); fileSystem.MoveDirectory(folderToRename, folderToRenameTarget); - // Moving the new folder out of the repo should not change the modified paths + // Moving the new folder out of the repo should not change the modified paths file string folderTargetOutsideSrc = Path.Combine(this.Enlistment.EnlistmentRoot, RenameFolderTarget); folderTargetOutsideSrc.ShouldNotExistOnDisk(fileSystem); fileSystem.MoveDirectory(folderToRenameTarget, folderTargetOutsideSrc); diff --git a/GVFS/GVFS.FunctionalTests/Tests/GitCommands/GitCommandsTests.cs b/GVFS/GVFS.FunctionalTests/Tests/GitCommands/GitCommandsTests.cs index 0b3ccccce..c623f5ea7 100644 --- a/GVFS/GVFS.FunctionalTests/Tests/GitCommands/GitCommandsTests.cs +++ b/GVFS/GVFS.FunctionalTests/Tests/GitCommands/GitCommandsTests.cs @@ -1041,7 +1041,7 @@ private void BasicCommit(Action fileSystemAction, string addCommand, [CallerMemb fileSystemAction(); this.ValidateGitCommand("status"); this.ValidateGitCommand(addCommand); - this.RunGitCommand("commit -m \"BasicCommit for {test}\""); + this.RunGitCommand($"commit -m \"BasicCommit for {test}\""); } private void SwitchBranch(Action fileSystemAction, [CallerMemberName]string test = GitCommandsTests.UnknownTestName) From 9f8944e1686bb880c7e50c6d0e3eb15453f4a025 Mon Sep 17 00:00:00 2001 From: Kevin Willford Date: Wed, 12 Sep 2018 09:06:46 -0600 Subject: [PATCH 157/272] Simplify checking the newly created files and removing from modified paths --- GVFS/GVFS.Virtualization/FileSystemCallbacks.cs | 15 ++++----------- 1 file changed, 4 insertions(+), 11 deletions(-) diff --git a/GVFS/GVFS.Virtualization/FileSystemCallbacks.cs b/GVFS/GVFS.Virtualization/FileSystemCallbacks.cs index c640d41a0..b41a2c634 100644 --- a/GVFS/GVFS.Virtualization/FileSystemCallbacks.cs +++ b/GVFS/GVFS.Virtualization/FileSystemCallbacks.cs @@ -691,14 +691,10 @@ private FileSystemTaskResult ExecuteBackgroundOperation(FileSystemTask gitUpdate result = this.TryAddModifiedPath(gitUpdate.VirtualPath, isFolder: true); if (result == FileSystemTaskResult.Success) { - if (!this.newlyCreatedFileAndFolderPaths.Contains(gitUpdate.VirtualPath)) - { - this.newlyCreatedFileAndFolderPaths.Add(gitUpdate.VirtualPath); - } - + this.newlyCreatedFileAndFolderPaths.Add(gitUpdate.VirtualPath); if (this.newlyCreatedFileAndFolderPaths.Contains(gitUpdate.OldVirtualPath)) { - this.TryRemoveModifiedPath(gitUpdate.OldVirtualPath, isFolder: true); + result = this.TryRemoveModifiedPath(gitUpdate.OldVirtualPath, isFolder: true); } Queue relativeFolderPaths = new Queue(); @@ -716,14 +712,11 @@ private FileSystemTaskResult ExecuteBackgroundOperation(FileSystemTask gitUpdate { string itemVirtualPath = Path.Combine(folderPath, itemInfo.Name); string oldItemVirtualPath = gitUpdate.OldVirtualPath + itemVirtualPath.Substring(gitUpdate.VirtualPath.Length); - if (!this.newlyCreatedFileAndFolderPaths.Contains(itemVirtualPath)) - { - this.newlyCreatedFileAndFolderPaths.Add(itemVirtualPath); - } + this.newlyCreatedFileAndFolderPaths.Add(itemVirtualPath); if (this.newlyCreatedFileAndFolderPaths.Contains(oldItemVirtualPath)) { - this.TryRemoveModifiedPath(oldItemVirtualPath, isFolder: itemInfo.IsDirectory); + result = this.TryRemoveModifiedPath(oldItemVirtualPath, isFolder: itemInfo.IsDirectory); } if (itemInfo.IsDirectory) From f7e268a3fedbbdadf0de1f634459794a0d194def Mon Sep 17 00:00:00 2001 From: Kevin Willford Date: Thu, 20 Sep 2018 10:46:06 -0400 Subject: [PATCH 158/272] Fix broken tests for cleaning modified paths --- .../EnlistmentPerTestCase/ModifiedPathsTests.cs | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/GVFS/GVFS.FunctionalTests/Tests/EnlistmentPerTestCase/ModifiedPathsTests.cs b/GVFS/GVFS.FunctionalTests/Tests/EnlistmentPerTestCase/ModifiedPathsTests.cs index 14fe1a5be..49981b86a 100644 --- a/GVFS/GVFS.FunctionalTests/Tests/EnlistmentPerTestCase/ModifiedPathsTests.cs +++ b/GVFS/GVFS.FunctionalTests/Tests/EnlistmentPerTestCase/ModifiedPathsTests.cs @@ -34,7 +34,6 @@ public class ModifiedPathsTests : TestsWithEnlistmentPerTestCase $"A {GVFSHelpers.ConvertPathToGitFormat(FileToRename)}", $"A {GVFSHelpers.ConvertPathToGitFormat(RenameFileTarget)}", $"A {FolderToCreate}/", - $"A {FolderToRename}/", $"A {RenameFolderTarget}/", $"A {RenameNewDotGitFileTarget}", $"A {FileToCreateOutsideRepo}", @@ -162,11 +161,12 @@ public void ModifiedPathsSavedAfterRemount(FileSystemRunner fileSystem) [TestCaseSource(typeof(FileSystemRunner), FileSystemRunner.TestRunners)] public void ModifiedPathsCorrectAfterHardLinking(FileSystemRunner fileSystem) { - const string ExpectedModifiedFilesContentsAfterHardlinks = -@"A .gitattributes -A LinkToReadme.md -A LinkToFileOutsideSrc.txt -"; + string[] expectedModifiedFilesContentsAfterHardlinks = + { + "A .gitattributes", + "A LinkToReadme.md", + "A LinkToFileOutsideSrc.txt", + }; // Create a link from src\LinkToReadme.md to src\Readme.md string existingFileInRepoPath = this.Enlistment.GetVirtualPathTo("Readme.md"); @@ -198,7 +198,8 @@ A LinkToFileOutsideSrc.txt modifiedPathsDatabase.ShouldBeAFile(fileSystem); using (StreamReader reader = new StreamReader(File.Open(modifiedPathsDatabase, FileMode.Open, FileAccess.Read, FileShare.ReadWrite))) { - reader.ReadToEnd().ShouldEqual(ExpectedModifiedFilesContentsAfterHardlinks); + reader.ReadToEnd().Split(new char[] { '\r', '\n' }, StringSplitOptions.RemoveEmptyEntries).OrderBy(x => x) + .ShouldMatchInOrder(expectedModifiedFilesContentsAfterHardlinks.OrderBy(x => x)); } } From a0023aa659997d6abc9cb697d7730ba6fa770101 Mon Sep 17 00:00:00 2001 From: Kevin Willford Date: Fri, 21 Sep 2018 13:42:23 -0600 Subject: [PATCH 159/272] Remove extra return statments and adding to new paths during rename --- GVFS/GVFS.Common/ModifiedPathsDatabase.cs | 4 ---- GVFS/GVFS.Virtualization/FileSystemCallbacks.cs | 1 - 2 files changed, 5 deletions(-) diff --git a/GVFS/GVFS.Common/ModifiedPathsDatabase.cs b/GVFS/GVFS.Common/ModifiedPathsDatabase.cs index 945613115..ffde4f692 100644 --- a/GVFS/GVFS.Common/ModifiedPathsDatabase.cs +++ b/GVFS/GVFS.Common/ModifiedPathsDatabase.cs @@ -84,8 +84,6 @@ public bool TryAdd(string path, bool isFolder, out bool isRetryable) isRetryable = false; return false; } - - return true; } return true; @@ -113,8 +111,6 @@ public bool TryRemove(string path, bool isFolder, out bool isRetryable) isRetryable = false; return false; } - - return true; } return true; diff --git a/GVFS/GVFS.Virtualization/FileSystemCallbacks.cs b/GVFS/GVFS.Virtualization/FileSystemCallbacks.cs index b41a2c634..4931aa1b3 100644 --- a/GVFS/GVFS.Virtualization/FileSystemCallbacks.cs +++ b/GVFS/GVFS.Virtualization/FileSystemCallbacks.cs @@ -425,7 +425,6 @@ public void OnFileConvertedToFull(string relativePath) public virtual void OnFileRenamed(string oldRelativePath, string newRelativePath) { - this.newlyCreatedFileAndFolderPaths.Add(newRelativePath); this.backgroundFileSystemTaskRunner.Enqueue(FileSystemTask.OnFileRenamed(oldRelativePath, newRelativePath)); } From dcf925fe61d618caea768fa7d8f7d1527f54e0eb Mon Sep 17 00:00:00 2001 From: Kevin Willford Date: Mon, 24 Sep 2018 15:36:55 -0600 Subject: [PATCH 160/272] Add PreDelete handler in BG operation to handle Mac --- .../MacFileSystemVirtualizer.cs | 2 +- .../WindowsFileSystemVirtualizer.cs | 2 +- .../Background/FileSystemTask.cs | 14 ++++- .../FileSystem/FileSystemVirtualizer.cs | 20 ++++++- .../FileSystemCallbacks.cs | 59 ++++++++++++++++--- 5 files changed, 84 insertions(+), 13 deletions(-) diff --git a/GVFS/GVFS.Platform.Mac/MacFileSystemVirtualizer.cs b/GVFS/GVFS.Platform.Mac/MacFileSystemVirtualizer.cs index 3d22f7f34..127173d0e 100644 --- a/GVFS/GVFS.Platform.Mac/MacFileSystemVirtualizer.cs +++ b/GVFS/GVFS.Platform.Mac/MacFileSystemVirtualizer.cs @@ -352,7 +352,7 @@ private Result OnPreDelete(string relativePath, bool isDirectory) } else { - this.OnWorkingDirectoryFileOrFolderDeleted(relativePath, isDirectory); + this.OnWorkingDirectoryFileOrFolderDeleteNotification(relativePath, isDirectory, isPreDelete: true); } } catch (Exception e) diff --git a/GVFS/GVFS.Platform.Windows/WindowsFileSystemVirtualizer.cs b/GVFS/GVFS.Platform.Windows/WindowsFileSystemVirtualizer.cs index 42c1f109a..de1963b99 100644 --- a/GVFS/GVFS.Platform.Windows/WindowsFileSystemVirtualizer.cs +++ b/GVFS/GVFS.Platform.Windows/WindowsFileSystemVirtualizer.cs @@ -1296,7 +1296,7 @@ private void NotifyFileHandleClosedFileModifiedOrDeletedHandler( } else { - this.OnWorkingDirectoryFileOrFolderDeleted(virtualPath, isDirectory); + this.OnWorkingDirectoryFileOrFolderDeleteNotification(virtualPath, isDirectory, isPreDelete: false); } } } diff --git a/GVFS/GVFS.Virtualization/Background/FileSystemTask.cs b/GVFS/GVFS.Virtualization/Background/FileSystemTask.cs index b5644bf86..eaab198a4 100644 --- a/GVFS/GVFS.Virtualization/Background/FileSystemTask.cs +++ b/GVFS/GVFS.Virtualization/Background/FileSystemTask.cs @@ -29,7 +29,9 @@ public enum OperationType OnFolderFirstWrite, OnIndexWriteWithoutProjectionChange, OnPlaceholderCreationsBlockedForGit, - OnFileHardLinkCreated + OnFileHardLinkCreated, + OnFilePreDelete, + OnFolderPreDelete, } public OperationType Operation { get; } @@ -57,6 +59,11 @@ public static FileSystemTask OnFileDeleted(string virtualPath) return new FileSystemTask(OperationType.OnFileDeleted, virtualPath, oldVirtualPath: null); } + public static FileSystemTask OnFilePreDelete(string virtualPath) + { + return new FileSystemTask(OperationType.OnFilePreDelete, virtualPath, oldVirtualPath: null); + } + public static FileSystemTask OnFileOverwritten(string virtualPath) { return new FileSystemTask(OperationType.OnFileOverwritten, virtualPath, oldVirtualPath: null); @@ -97,6 +104,11 @@ public static FileSystemTask OnFolderDeleted(string virtualPath) return new FileSystemTask(OperationType.OnFolderDeleted, virtualPath, oldVirtualPath: null); } + public static FileSystemTask OnFolderPreDelete(string virtualPath) + { + return new FileSystemTask(OperationType.OnFolderPreDelete, virtualPath, oldVirtualPath: null); + } + public static FileSystemTask OnIndexWriteWithoutProjectionChange() { return new FileSystemTask(OperationType.OnIndexWriteWithoutProjectionChange, virtualPath: null, oldVirtualPath: null); diff --git a/GVFS/GVFS.Virtualization/FileSystem/FileSystemVirtualizer.cs b/GVFS/GVFS.Virtualization/FileSystem/FileSystemVirtualizer.cs index c77e94669..70cac75d1 100644 --- a/GVFS/GVFS.Virtualization/FileSystem/FileSystemVirtualizer.cs +++ b/GVFS/GVFS.Virtualization/FileSystem/FileSystemVirtualizer.cs @@ -185,7 +185,7 @@ protected void OnDotGitFileOrFolderDeleted(string relativePath) } } - protected void OnWorkingDirectoryFileOrFolderDeleted(string relativePath, bool isDirectory) + protected void OnWorkingDirectoryFileOrFolderDeleteNotification(string relativePath, bool isDirectory, bool isPreDelete) { if (isDirectory) { @@ -193,12 +193,26 @@ protected void OnWorkingDirectoryFileOrFolderDeleted(string relativePath, bool i GitCommandLineParser gitCommand = new GitCommandLineParser(this.Context.Repository.GVFSLock.GetLockedGitCommand()); if (!gitCommand.IsValidGitCommand) { - this.FileSystemCallbacks.OnFolderDeleted(relativePath); + if (isPreDelete) + { + this.FileSystemCallbacks.OnFolderPreDelete(relativePath); + } + else + { + this.FileSystemCallbacks.OnFolderDeleted(relativePath); + } } } else { - this.FileSystemCallbacks.OnFileDeleted(relativePath); + if (isPreDelete) + { + this.FileSystemCallbacks.OnFilePreDelete(relativePath); + } + else + { + this.FileSystemCallbacks.OnFileDeleted(relativePath); + } } this.FileSystemCallbacks.InvalidateGitStatusCache(); diff --git a/GVFS/GVFS.Virtualization/FileSystemCallbacks.cs b/GVFS/GVFS.Virtualization/FileSystemCallbacks.cs index 4931aa1b3..44a0aaf62 100644 --- a/GVFS/GVFS.Virtualization/FileSystemCallbacks.cs +++ b/GVFS/GVFS.Virtualization/FileSystemCallbacks.cs @@ -438,6 +438,11 @@ public void OnFileDeleted(string relativePath) this.backgroundFileSystemTaskRunner.Enqueue(FileSystemTask.OnFileDeleted(relativePath)); } + public void OnFilePreDelete(string relativePath) + { + this.backgroundFileSystemTaskRunner.Enqueue(FileSystemTask.OnFilePreDelete(relativePath)); + } + public void OnFolderCreated(string relativePath) { this.newlyCreatedFileAndFolderPaths.Add(relativePath); @@ -454,6 +459,11 @@ public void OnFolderDeleted(string relativePath) this.backgroundFileSystemTaskRunner.Enqueue(FileSystemTask.OnFolderDeleted(relativePath)); } + public void OnFolderPreDelete(string relativePath) + { + this.backgroundFileSystemTaskRunner.Enqueue(FileSystemTask.OnFolderPreDelete(relativePath)); + } + public void OnPlaceholderFileCreated(string relativePath, string sha, string triggeringProcessImageFileName) { this.GitIndexProjection.OnPlaceholderFileCreated(relativePath, sha); @@ -650,6 +660,27 @@ private FileSystemTaskResult ExecuteBackgroundOperation(FileSystemTask gitUpdate break; + case FileSystemTask.OperationType.OnFilePreDelete: + metadata.Add("virtualPath", gitUpdate.VirtualPath); + if (this.newlyCreatedFileAndFolderPaths.Contains(gitUpdate.VirtualPath)) + { + string fullPathToFolder = Path.Combine(this.context.Enlistment.WorkingDirectoryRoot, gitUpdate.VirtualPath); + if (!this.context.FileSystem.FileExists(fullPathToFolder)) + { + result = this.TryRemoveModifiedPath(gitUpdate.VirtualPath, isFolder: false); + } + else + { + result = FileSystemTaskResult.Success; + } + } + else + { + result = this.AddModifiedPathAndRemoveFromPlaceholderList(gitUpdate.VirtualPath); + } + + break; + case FileSystemTask.OperationType.OnFileDeleted: metadata.Add("virtualPath", gitUpdate.VirtualPath); if (this.newlyCreatedFileAndFolderPaths.Contains(gitUpdate.VirtualPath)) @@ -764,6 +795,27 @@ private FileSystemTaskResult ExecuteBackgroundOperation(FileSystemTask gitUpdate break; + case FileSystemTask.OperationType.OnFolderPreDelete: + metadata.Add("virtualPath", gitUpdate.VirtualPath); + if (this.newlyCreatedFileAndFolderPaths.Contains(gitUpdate.VirtualPath)) + { + string fullPathToFolder = Path.Combine(this.context.Enlistment.WorkingDirectoryRoot, gitUpdate.VirtualPath); + if (!this.context.FileSystem.DirectoryExists(fullPathToFolder)) + { + result = this.TryRemoveModifiedPath(gitUpdate.VirtualPath, isFolder: true); + } + else + { + result = FileSystemTaskResult.Success; + } + } + else + { + result = this.TryAddModifiedPath(gitUpdate.VirtualPath, isFolder: true); + } + + break; + case FileSystemTask.OperationType.OnFolderDeleted: metadata.Add("virtualPath", gitUpdate.VirtualPath); if (this.newlyCreatedFileAndFolderPaths.Contains(gitUpdate.VirtualPath)) @@ -808,13 +860,6 @@ private FileSystemTaskResult ExecuteBackgroundOperation(FileSystemTask gitUpdate private FileSystemTaskResult TryRemoveModifiedPath(string virtualPath, bool isFolder) { - string fullPathToItem = Path.Combine(this.context.Enlistment.WorkingDirectoryRoot, virtualPath); - if ((isFolder && this.context.FileSystem.DirectoryExists(fullPathToItem)) || - (!isFolder && this.context.FileSystem.FileExists(fullPathToItem))) - { - return FileSystemTaskResult.Success; - } - if (!this.modifiedPaths.TryRemove(virtualPath, isFolder, out bool isRetryable)) { return isRetryable ? FileSystemTaskResult.RetryableError : FileSystemTaskResult.FatalError; From 45159dbf5e41c2352e660605ea62ee03b333a2b2 Mon Sep 17 00:00:00 2001 From: Kevin Willford Date: Tue, 25 Sep 2018 09:14:37 -0600 Subject: [PATCH 161/272] Add comment explaining only one of predelete or delete should be queued --- GVFS/GVFS.Virtualization/FileSystemCallbacks.cs | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/GVFS/GVFS.Virtualization/FileSystemCallbacks.cs b/GVFS/GVFS.Virtualization/FileSystemCallbacks.cs index 44a0aaf62..95618e10e 100644 --- a/GVFS/GVFS.Virtualization/FileSystemCallbacks.cs +++ b/GVFS/GVFS.Virtualization/FileSystemCallbacks.cs @@ -661,6 +661,9 @@ private FileSystemTaskResult ExecuteBackgroundOperation(FileSystemTask gitUpdate break; case FileSystemTask.OperationType.OnFilePreDelete: + // This code assumes that the current implementations of FileSystemVirtualizer will call either + // the PreDelete or the Delete not both so if a new implementation starts calling both + // this will need to be cleaned up to not duplicate the work that is being done. metadata.Add("virtualPath", gitUpdate.VirtualPath); if (this.newlyCreatedFileAndFolderPaths.Contains(gitUpdate.VirtualPath)) { @@ -682,6 +685,9 @@ private FileSystemTaskResult ExecuteBackgroundOperation(FileSystemTask gitUpdate break; case FileSystemTask.OperationType.OnFileDeleted: + // This code assumes that the current implementations of FileSystemVirtualizer will call either + // the PreDelete or the Delete not both so if a new implementation starts calling both + // this will need to be cleaned up to not duplicate the work that is being done. metadata.Add("virtualPath", gitUpdate.VirtualPath); if (this.newlyCreatedFileAndFolderPaths.Contains(gitUpdate.VirtualPath)) { @@ -796,6 +802,9 @@ private FileSystemTaskResult ExecuteBackgroundOperation(FileSystemTask gitUpdate break; case FileSystemTask.OperationType.OnFolderPreDelete: + // This code assumes that the current implementations of FileSystemVirtualizer will call either + // the PreDelete or the Delete not both so if a new implementation starts calling both + // this will need to be cleaned up to not duplicate the work that is being done. metadata.Add("virtualPath", gitUpdate.VirtualPath); if (this.newlyCreatedFileAndFolderPaths.Contains(gitUpdate.VirtualPath)) { @@ -817,6 +826,9 @@ private FileSystemTaskResult ExecuteBackgroundOperation(FileSystemTask gitUpdate break; case FileSystemTask.OperationType.OnFolderDeleted: + // This code assumes that the current implementations of FileSystemVirtualizer will call either + // the PreDelete or the Delete not both so if a new implementation starts calling both + // this will need to be cleaned up to not duplicate the work that is being done. metadata.Add("virtualPath", gitUpdate.VirtualPath); if (this.newlyCreatedFileAndFolderPaths.Contains(gitUpdate.VirtualPath)) { From 472904c45ef03f8757bdf151924652802e73d7f0 Mon Sep 17 00:00:00 2001 From: Kevin Willford Date: Tue, 25 Sep 2018 12:21:17 -0600 Subject: [PATCH 162/272] Fix case only folder rename test --- .../EnlistmentPerFixture/GitFilesTests.cs | 20 ++++++++++++++----- .../FileSystemCallbacks.cs | 12 ++++++----- 2 files changed, 22 insertions(+), 10 deletions(-) diff --git a/GVFS/GVFS.FunctionalTests/Tests/EnlistmentPerFixture/GitFilesTests.cs b/GVFS/GVFS.FunctionalTests/Tests/EnlistmentPerFixture/GitFilesTests.cs index 88dbf1709..bfbdcc457 100644 --- a/GVFS/GVFS.FunctionalTests/Tests/EnlistmentPerFixture/GitFilesTests.cs +++ b/GVFS/GVFS.FunctionalTests/Tests/EnlistmentPerFixture/GitFilesTests.cs @@ -132,22 +132,32 @@ public void RenameFolderTest() [Category(Categories.MacTODO.M2)] public void CaseOnlyRenameOfNewFolderKeepsModifiedPathsEntries() { - string[] expectedModifiedPathsEntries = + if (this.fileSystem is PowerShellRunner) { - "Folder/", - "Folder/testfile", + Assert.Ignore("Powershell does not support case only renames."); + } + + string[] expectedModifiedPathsEntriesAfterCreate = + { + "A Folder/", + "A Folder/testfile", + }; + + string[] expectedModifiedPathsEntriesAfterRename = + { + "A folder/", }; this.fileSystem.CreateDirectory(Path.Combine(this.Enlistment.RepoRoot, "Folder")); this.fileSystem.CreateEmptyFile(Path.Combine(this.Enlistment.RepoRoot, "Folder", "testfile")); this.Enlistment.WaitForBackgroundOperations().ShouldEqual(true, "Background operations failed to complete."); - GVFSHelpers.ModifiedPathsShouldContain(this.fileSystem, this.Enlistment.DotGVFSRoot, expectedModifiedPathsEntries); + GVFSHelpers.ModifiedPathsShouldContain(this.fileSystem, this.Enlistment.DotGVFSRoot, expectedModifiedPathsEntriesAfterCreate); this.fileSystem.RenameDirectory(this.Enlistment.RepoRoot, "Folder", "folder"); this.Enlistment.WaitForBackgroundOperations().ShouldEqual(true, "Background operations failed to complete."); - GVFSHelpers.ModifiedPathsShouldContain(this.fileSystem, this.Enlistment.DotGVFSRoot, expectedModifiedPathsEntries); + GVFSHelpers.ModifiedPathsShouldContain(this.fileSystem, this.Enlistment.DotGVFSRoot, expectedModifiedPathsEntriesAfterRename); } [TestCase, Order(7)] diff --git a/GVFS/GVFS.Virtualization/FileSystemCallbacks.cs b/GVFS/GVFS.Virtualization/FileSystemCallbacks.cs index 95618e10e..d7c0d57b4 100644 --- a/GVFS/GVFS.Virtualization/FileSystemCallbacks.cs +++ b/GVFS/GVFS.Virtualization/FileSystemCallbacks.cs @@ -718,20 +718,22 @@ private FileSystemTaskResult ExecuteBackgroundOperation(FileSystemTask gitUpdate metadata.Add("oldVirtualPath", gitUpdate.OldVirtualPath); metadata.Add("virtualPath", gitUpdate.VirtualPath); + if (!string.IsNullOrEmpty(gitUpdate.OldVirtualPath) && + this.newlyCreatedFileAndFolderPaths.Contains(gitUpdate.OldVirtualPath)) + { + result = this.TryRemoveModifiedPath(gitUpdate.OldVirtualPath, isFolder: true); + } + // An empty destination path means the folder was renamed to somewhere outside of the repo // Note that only full folders can be moved\renamed, and so there will already be a recursive // sparse-checkout entry for the virtualPath of the folder being moved (meaning that no // additional work is needed for any files\folders inside the folder being moved) - if (!string.IsNullOrEmpty(gitUpdate.VirtualPath)) + if (result == FileSystemTaskResult.Success && !string.IsNullOrEmpty(gitUpdate.VirtualPath)) { result = this.TryAddModifiedPath(gitUpdate.VirtualPath, isFolder: true); if (result == FileSystemTaskResult.Success) { this.newlyCreatedFileAndFolderPaths.Add(gitUpdate.VirtualPath); - if (this.newlyCreatedFileAndFolderPaths.Contains(gitUpdate.OldVirtualPath)) - { - result = this.TryRemoveModifiedPath(gitUpdate.OldVirtualPath, isFolder: true); - } Queue relativeFolderPaths = new Queue(); relativeFolderPaths.Enqueue(gitUpdate.VirtualPath); From 243f6214d7334a69df31fee1ffa3563e5e45f3af Mon Sep 17 00:00:00 2001 From: Kevin Willford Date: Tue, 25 Sep 2018 16:03:35 -0600 Subject: [PATCH 163/272] Fix test that moved new folder outside the repo --- .../Tests/EnlistmentPerTestCase/ModifiedPathsTests.cs | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/GVFS/GVFS.FunctionalTests/Tests/EnlistmentPerTestCase/ModifiedPathsTests.cs b/GVFS/GVFS.FunctionalTests/Tests/EnlistmentPerTestCase/ModifiedPathsTests.cs index 49981b86a..e3e9de6a5 100644 --- a/GVFS/GVFS.FunctionalTests/Tests/EnlistmentPerTestCase/ModifiedPathsTests.cs +++ b/GVFS/GVFS.FunctionalTests/Tests/EnlistmentPerTestCase/ModifiedPathsTests.cs @@ -34,7 +34,6 @@ public class ModifiedPathsTests : TestsWithEnlistmentPerTestCase $"A {GVFSHelpers.ConvertPathToGitFormat(FileToRename)}", $"A {GVFSHelpers.ConvertPathToGitFormat(RenameFileTarget)}", $"A {FolderToCreate}/", - $"A {RenameFolderTarget}/", $"A {RenameNewDotGitFileTarget}", $"A {FileToCreateOutsideRepo}", $"A {FolderToCreateOutsideRepo}/", @@ -108,7 +107,7 @@ public void ModifiedPathsSavedAfterRemount(FileSystemRunner fileSystem) string folderToRenameTarget = this.Enlistment.GetVirtualPathTo(RenameFolderTarget); fileSystem.MoveDirectory(folderToRename, folderToRenameTarget); - // Moving the new folder out of the repo should not change the modified paths file + // Moving the new folder out of the repo will remove it from the modified paths file string folderTargetOutsideSrc = Path.Combine(this.Enlistment.EnlistmentRoot, RenameFolderTarget); folderTargetOutsideSrc.ShouldNotExistOnDisk(fileSystem); fileSystem.MoveDirectory(folderToRenameTarget, folderTargetOutsideSrc); From 355fd062cdc290ea12211df99e70b0a71e662ed5 Mon Sep 17 00:00:00 2001 From: William Baker Date: Tue, 25 Sep 2018 13:36:04 -0700 Subject: [PATCH 164/272] Add new unit and functional tests --- .../FileSystemRunners/BashRunner.cs | 56 ++++-- .../EnlistmentPerFixture/SymbolicLinkTests.cs | 163 ++++++++++++++++++ .../NamedPipeStreamReaderWriterTests.cs | 2 + .../Mock/Mac/MockVirtualizationInstance.cs | 22 +++ .../MacFileSystemVirtualizerTests.cs | 105 ++++++++++- 5 files changed, 331 insertions(+), 17 deletions(-) create mode 100644 GVFS/GVFS.FunctionalTests/Tests/EnlistmentPerFixture/SymbolicLinkTests.cs diff --git a/GVFS/GVFS.FunctionalTests/FileSystemRunners/BashRunner.cs b/GVFS/GVFS.FunctionalTests/FileSystemRunners/BashRunner.cs index cd1a79337..af35b8737 100644 --- a/GVFS/GVFS.FunctionalTests/FileSystemRunners/BashRunner.cs +++ b/GVFS/GVFS.FunctionalTests/FileSystemRunners/BashRunner.cs @@ -53,6 +53,14 @@ public BashRunner() } } + private enum FileType + { + Invalid, + File, + Directory, + SymLink + } + protected override string FileName { get @@ -86,15 +94,22 @@ public static void DeleteDirectoryWithUnlimitedRetries(string path) } } - public override bool FileExists(string path) + public bool IsSymbolicLink(string path) { - string bashPath = this.ConvertWinPathToBashPath(path); + return this.FileExistsOnDisk(path, FileType.SymLink); + } - string command = string.Format("-c \"[ -f {0} ] && echo {1} || echo {2}\"", bashPath, ShellRunner.SuccessOutput, ShellRunner.FailureOutput); + public void CreateSymbolicLink(string newLinkFilePath, string existingFilePath) + { + string existingFileBashPath = this.ConvertWinPathToBashPath(existingFilePath); + string newLinkBashPath = this.ConvertWinPathToBashPath(newLinkFilePath); - string output = this.RunProcess(command).Trim(); + this.RunProcess(string.Format("-c \"ln -s -F {0} {1}\"", existingFileBashPath, newLinkBashPath)); + } - return output.Equals(ShellRunner.SuccessOutput, StringComparison.InvariantCulture); + public override bool FileExists(string path) + { + return this.FileExistsOnDisk(path, FileType.File); } public override string MoveFile(string sourcePath, string targetPath) @@ -187,11 +202,7 @@ public override void WriteAllTextShouldFail(string path, string c public override bool DirectoryExists(string path) { - string bashPath = this.ConvertWinPathToBashPath(path); - - string output = this.RunProcess(string.Format("-c \"[ -d {0} ] && echo {1} || echo {2}\"", bashPath, ShellRunner.SuccessOutput, ShellRunner.FailureOutput)).Trim(); - - return output.Equals(ShellRunner.SuccessOutput, StringComparison.InvariantCulture); + return this.FileExistsOnDisk(path, FileType.Directory); } public override void MoveDirectory(string sourcePath, string targetPath) @@ -267,6 +278,31 @@ public override void DeleteDirectory_ShouldBeBlockedByProcess(string path) Assert.Fail("Unlike the other runners, bash.exe does not check folder handle before recusively deleting"); } + private bool FileExistsOnDisk(string path, FileType type) + { + string checkArgument = string.Empty; + switch (type) + { + case FileType.File: + checkArgument = "-f"; + break; + case FileType.Directory: + checkArgument = "-d"; + break; + case FileType.SymLink: + checkArgument = "-h"; + break; + default: + Assert.Fail($"{nameof(FileExistsOnDisk)} does not support {nameof(FileType)} {type}"); + break; + } + + string bashPath = this.ConvertWinPathToBashPath(path); + string command = $"-c \"[ {checkArgument} {bashPath} ] && echo {ShellRunner.SuccessOutput} || echo {ShellRunner.FailureOutput}\""; + string output = this.RunProcess(command).Trim(); + return output.Equals(ShellRunner.SuccessOutput, StringComparison.InvariantCulture); + } + private string ConvertWinPathToBashPath(string winPath) { string bashPath = string.Concat("/", winPath); diff --git a/GVFS/GVFS.FunctionalTests/Tests/EnlistmentPerFixture/SymbolicLinkTests.cs b/GVFS/GVFS.FunctionalTests/Tests/EnlistmentPerFixture/SymbolicLinkTests.cs new file mode 100644 index 000000000..04179c671 --- /dev/null +++ b/GVFS/GVFS.FunctionalTests/Tests/EnlistmentPerFixture/SymbolicLinkTests.cs @@ -0,0 +1,163 @@ +using System.IO; +using GVFS.FunctionalTests.FileSystemRunners; +using GVFS.FunctionalTests.Should; +using GVFS.FunctionalTests.Tools; +using GVFS.Tests.Should; +using NUnit.Framework; + +namespace GVFS.FunctionalTests.Tests.EnlistmentPerFixture +{ + // MacOnly until issue #297 (add SymLink support for Windows) is complete + [Category(Categories.MacOnly)] + [TestFixture] + public class SymbolicLinkTests : TestsWithEnlistmentPerFixture + { + private const string TestFolderName = "Test_EPF_SymbolicLinks"; + + // FunctionalTests/20180925_SymLinksPart1 files + private const string TestFileName = "TestFile.txt"; + private const string TestFileContents = "This is a real file"; + private const string TestFile2Name = "TestFile2.txt"; + private const string TestFile2Contents = "This is the second real file"; + private const string ChildFolderName = "ChildDir"; + private const string ChildLinkName = "LinkToFileInFolder"; + private const string GrandChildLinkName = "LinkToFileInParentFolder"; + + // FunctionalTests/20180925_SymLinksPart2 files + // Note: In this branch ChildLinkName has been changed to point to TestFile2Name + private const string GrandChildFileName = "TestFile3.txt"; + private const string GrandChildFileContents = "This is the third file"; + private const string GrandChildLinkNowAFileContents = "This was a link but is now a file"; + + private BashRunner bashRunner; + public SymbolicLinkTests() + { + this.bashRunner = new BashRunner(); + } + + [TestCase, Order(1)] + public void CheckoutBranchWithSymLinks() + { + GitHelpers.InvokeGitAgainstGVFSRepo(this.Enlistment.RepoRoot, "checkout FunctionalTests/20180925_SymLinksPart1"); + GitHelpers.CheckGitCommandAgainstGVFSRepo( + this.Enlistment.RepoRoot, + "status", + "On branch FunctionalTests/20180925_SymLinksPart1", + "nothing to commit, working tree clean"); + + string testFilePath = this.Enlistment.GetVirtualPathTo(Path.Combine(TestFolderName, TestFileName)); + testFilePath.ShouldBeAFile(this.bashRunner).WithContents(TestFileContents); + this.bashRunner.IsSymbolicLink(testFilePath).ShouldBeFalse($"{testFilePath} should not be a symlink"); + + string testFile2Path = this.Enlistment.GetVirtualPathTo(Path.Combine(TestFolderName, TestFile2Name)); + testFile2Path.ShouldBeAFile(this.bashRunner).WithContents(TestFile2Contents); + this.bashRunner.IsSymbolicLink(testFile2Path).ShouldBeFalse($"{testFile2Path} should not be a symlink"); + + string childLinkPath = this.Enlistment.GetVirtualPathTo(Path.Combine(TestFolderName, ChildLinkName)); + this.bashRunner.IsSymbolicLink(childLinkPath).ShouldBeTrue($"{childLinkPath} should be a symlink"); + childLinkPath.ShouldBeAFile(this.bashRunner).WithContents(TestFileContents); + + string grandChildLinkPath = this.Enlistment.GetVirtualPathTo(Path.Combine(TestFolderName, ChildFolderName, GrandChildLinkName)); + this.bashRunner.IsSymbolicLink(grandChildLinkPath).ShouldBeTrue($"{grandChildLinkPath} should be a symlink"); + grandChildLinkPath.ShouldBeAFile(this.bashRunner).WithContents(TestFile2Contents); + } + + [TestCase, Order(2)] + public void CheckoutBranchWhereSymLinksChangeContentsAndTransitionToFile() + { + GitHelpers.InvokeGitAgainstGVFSRepo(this.Enlistment.RepoRoot, "checkout FunctionalTests/20180925_SymLinksPart2"); + GitHelpers.CheckGitCommandAgainstGVFSRepo( + this.Enlistment.RepoRoot, + "status", + "On branch FunctionalTests/20180925_SymLinksPart2", + "nothing to commit, working tree clean"); + + // testFilePath and testFile2Path are unchanged from FunctionalTests/20180925_SymLinksPart2 + string testFilePath = this.Enlistment.GetVirtualPathTo(Path.Combine(TestFolderName, TestFileName)); + testFilePath.ShouldBeAFile(this.bashRunner).WithContents(TestFileContents); + this.bashRunner.IsSymbolicLink(testFilePath).ShouldBeFalse($"{testFilePath} should not be a symlink"); + + string testFile2Path = this.Enlistment.GetVirtualPathTo(Path.Combine(TestFolderName, TestFile2Name)); + testFile2Path.ShouldBeAFile(this.bashRunner).WithContents(TestFile2Contents); + this.bashRunner.IsSymbolicLink(testFile2Path).ShouldBeFalse($"{testFile2Path} should not be a symlink"); + + // In this branch childLinkPath has been changed to point to testFile2Path + string childLinkPath = this.Enlistment.GetVirtualPathTo(Path.Combine(TestFolderName, ChildLinkName)); + this.bashRunner.IsSymbolicLink(childLinkPath).ShouldBeTrue($"{childLinkPath} should be a symlink"); + childLinkPath.ShouldBeAFile(this.bashRunner).WithContents(TestFile2Contents); + + // grandChildLinkPath should now be a file + string grandChildLinkPath = this.Enlistment.GetVirtualPathTo(Path.Combine(TestFolderName, ChildFolderName, GrandChildLinkName)); + this.bashRunner.IsSymbolicLink(grandChildLinkPath).ShouldBeFalse($"{grandChildLinkPath} should not be a symlink"); + grandChildLinkPath.ShouldBeAFile(this.bashRunner).WithContents(GrandChildLinkNowAFileContents); + + // There should also be a new file in the child folder + string newGrandChildFilePath = this.Enlistment.GetVirtualPathTo(Path.Combine(TestFolderName, ChildFolderName, GrandChildFileName)); + newGrandChildFilePath.ShouldBeAFile(this.bashRunner).WithContents(GrandChildFileContents); + this.bashRunner.IsSymbolicLink(newGrandChildFilePath).ShouldBeFalse($"{newGrandChildFilePath} should not be a symlink"); + } + + [TestCase, Order(3)] + public void CheckoutBranchWhereFilesTransitionToSymLinks() + { + GitHelpers.InvokeGitAgainstGVFSRepo(this.Enlistment.RepoRoot, "checkout FunctionalTests/20180925_SymLinksPart3"); + GitHelpers.CheckGitCommandAgainstGVFSRepo( + this.Enlistment.RepoRoot, + "status", + "On branch FunctionalTests/20180925_SymLinksPart3", + "nothing to commit, working tree clean"); + + // In this branch testFilePath has been changed to point to newGrandChildFilePath + string testFilePath = this.Enlistment.GetVirtualPathTo(Path.Combine(TestFolderName, TestFileName)); + testFilePath.ShouldBeAFile(this.bashRunner).WithContents(GrandChildFileContents); + this.bashRunner.IsSymbolicLink(testFilePath).ShouldBeTrue($"{testFilePath} should be a symlink"); + + // The rest of the files are unchanged from FunctionalTests/20180925_SymLinksPart2 + string testFile2Path = this.Enlistment.GetVirtualPathTo(Path.Combine(TestFolderName, TestFile2Name)); + testFile2Path.ShouldBeAFile(this.bashRunner).WithContents(TestFile2Contents); + this.bashRunner.IsSymbolicLink(testFile2Path).ShouldBeFalse($"{testFile2Path} should not be a symlink"); + + string childLinkPath = this.Enlistment.GetVirtualPathTo(Path.Combine(TestFolderName, ChildLinkName)); + this.bashRunner.IsSymbolicLink(childLinkPath).ShouldBeTrue($"{childLinkPath} should be a symlink"); + childLinkPath.ShouldBeAFile(this.bashRunner).WithContents(TestFile2Contents); + + string grandChildLinkPath = this.Enlistment.GetVirtualPathTo(Path.Combine(TestFolderName, ChildFolderName, GrandChildLinkName)); + this.bashRunner.IsSymbolicLink(grandChildLinkPath).ShouldBeFalse($"{grandChildLinkPath} should not be a symlink"); + grandChildLinkPath.ShouldBeAFile(this.bashRunner).WithContents(GrandChildLinkNowAFileContents); + + string newGrandChildFilePath = this.Enlistment.GetVirtualPathTo(Path.Combine(TestFolderName, ChildFolderName, GrandChildFileName)); + newGrandChildFilePath.ShouldBeAFile(this.bashRunner).WithContents(GrandChildFileContents); + this.bashRunner.IsSymbolicLink(newGrandChildFilePath).ShouldBeFalse($"{newGrandChildFilePath} should not be a symlink"); + } + + [TestCase, Order(4)] + public void GitStatusReportsSymLinkChanges() + { + GitHelpers.CheckGitCommandAgainstGVFSRepo( + this.Enlistment.RepoRoot, + "status", + "On branch FunctionalTests/20180925_SymLinksPart3", + "nothing to commit, working tree clean"); + + string testFilePath = this.Enlistment.GetVirtualPathTo(Path.Combine(TestFolderName, TestFileName)); + testFilePath.ShouldBeAFile(this.bashRunner).WithContents(GrandChildFileContents); + this.bashRunner.IsSymbolicLink(testFilePath).ShouldBeTrue($"{testFilePath} should be a symlink"); + + string testFile2Path = this.Enlistment.GetVirtualPathTo(Path.Combine(TestFolderName, TestFile2Name)); + testFile2Path.ShouldBeAFile(this.bashRunner).WithContents(TestFile2Contents); + this.bashRunner.IsSymbolicLink(testFile2Path).ShouldBeFalse($"{testFile2Path} should not be a symlink"); + + // Update testFilePath's symlink to point to testFile2Path + this.bashRunner.CreateSymbolicLink(testFilePath, testFile2Path); + + testFilePath.ShouldBeAFile(this.bashRunner).WithContents(TestFile2Contents); + this.bashRunner.IsSymbolicLink(testFilePath).ShouldBeTrue($"{testFilePath} should be a symlink"); + + GitHelpers.CheckGitCommandAgainstGVFSRepo( + this.Enlistment.RepoRoot, + "status", + "On branch FunctionalTests/20180925_SymLinksPart3", + $"modified: {TestFolderName}/{TestFileName}"); + } + } +} diff --git a/GVFS/GVFS.UnitTests/Common/NamedPipeStreamReaderWriterTests.cs b/GVFS/GVFS.UnitTests/Common/NamedPipeStreamReaderWriterTests.cs index 003ca048f..7e630fea6 100644 --- a/GVFS/GVFS.UnitTests/Common/NamedPipeStreamReaderWriterTests.cs +++ b/GVFS/GVFS.UnitTests/Common/NamedPipeStreamReaderWriterTests.cs @@ -1,5 +1,6 @@ using GVFS.Common.NamedPipes; using GVFS.Tests.Should; +using GVFS.UnitTests.Category; using NUnit.Framework; using System.IO; @@ -58,6 +59,7 @@ public void CanSendBufferSizedMessage() [Test] [Description("Verify that the expected exception is thrown if message is not terminated with expected byte.")] + [Category(CategoryConstants.ExceptionExpected)] public void ReadingPartialMessgeThrows() { byte[] bytes = System.Text.Encoding.ASCII.GetBytes("This is a partial message"); diff --git a/GVFS/GVFS.UnitTests/Mock/Mac/MockVirtualizationInstance.cs b/GVFS/GVFS.UnitTests/Mock/Mac/MockVirtualizationInstance.cs index 57ae1b720..082565d22 100644 --- a/GVFS/GVFS.UnitTests/Mock/Mac/MockVirtualizationInstance.cs +++ b/GVFS/GVFS.UnitTests/Mock/Mac/MockVirtualizationInstance.cs @@ -15,6 +15,7 @@ public MockVirtualizationInstance() { this.commandCompleted = new AutoResetEvent(false); this.CreatedPlaceholders = new ConcurrentDictionary(); + this.CreatedSymLinks = new ConcurrentHashSet(); this.WriteFileReturnResult = Result.Success; } @@ -28,6 +29,8 @@ public MockVirtualizationInstance() public ConcurrentDictionary CreatedPlaceholders { get; private set; } + public ConcurrentHashSet CreatedSymLinks { get; } + public override EnumerateDirectoryCallback OnEnumerateDirectory { get; set; } public override GetFileStreamCallback OnGetFileStream { get; set; } @@ -79,6 +82,14 @@ public override Result WritePlaceholderFile( return Result.Success; } + public override Result WriteSymLink( + string relativePath, + string symLinkContents) + { + this.CreatedSymLinks.Add(relativePath); + return Result.Success; + } + public override Result UpdatePlaceholderIfNeeded( string relativePath, byte[] providerId, @@ -92,6 +103,17 @@ public override Result UpdatePlaceholderIfNeeded( return this.UpdatePlaceholderIfNeededResult; } + public override Result ReplacePlaceholderFileWithSymLink( + string relativePath, + string symLinkContents, + UpdateType updateFlags, + out UpdateFailureCause failureCause) + { + this.CreatedSymLinks.Add(relativePath); + failureCause = this.UpdatePlaceholderIfNeededFailureCause; + return this.UpdatePlaceholderIfNeededResult; + } + public override Result CompleteCommand( ulong commandId, Result result) diff --git a/GVFS/GVFS.UnitTests/Platform.Mac/MacFileSystemVirtualizerTests.cs b/GVFS/GVFS.UnitTests/Platform.Mac/MacFileSystemVirtualizerTests.cs index 6d4d19d6d..9be03ae2c 100644 --- a/GVFS/GVFS.UnitTests/Platform.Mac/MacFileSystemVirtualizerTests.cs +++ b/GVFS/GVFS.UnitTests/Platform.Mac/MacFileSystemVirtualizerTests.cs @@ -10,6 +10,7 @@ using GVFS.UnitTests.Virtual; using GVFS.Virtualization; using GVFS.Virtualization.FileSystem; +using GVFS.Virtualization.Projection; using NUnit.Framework; using PrjFSLib.Mac; using System; @@ -21,7 +22,6 @@ namespace GVFS.UnitTests.Platform.Mac [TestFixture] public class MacFileSystemVirtualizerTests : TestsWithCommonRepo { - private static readonly ushort RegularFileType = 0x8000; private static readonly ushort FileMode644 = Convert.ToUInt16("644", 8); private static readonly ushort FileMode664 = Convert.ToUInt16("664", 8); private static readonly ushort FileMode755 = Convert.ToUInt16("755", 8); @@ -100,7 +100,7 @@ public void UpdatePlaceholderIfNeeded() virtualizer)) { string filePath = "test" + Path.DirectorySeparatorChar + "test.txt"; - gitIndexProjection.MockFileTypesAndModes.TryAdd(filePath, (ushort)(RegularFileType | FileMode644)); + gitIndexProjection.MockFileTypesAndModes.TryAdd(filePath, (ushort)((ushort)GitIndexProjection.FileType.Regular | FileMode644)); string error; fileSystemCallbacks.TryStart(out error).ShouldEqual(true); @@ -163,6 +163,97 @@ public void UpdatePlaceholderIfNeeded() } } + [TestCase] + public void WritePlaceholderForSymLink() + { + using (MockBackgroundFileSystemTaskRunner backgroundTaskRunner = new MockBackgroundFileSystemTaskRunner()) + using (MockVirtualizationInstance mockVirtualization = new MockVirtualizationInstance()) + using (MockGitIndexProjection gitIndexProjection = new MockGitIndexProjection(new[] { "test.txt" })) + using (MacFileSystemVirtualizer virtualizer = new MacFileSystemVirtualizer(this.Repo.Context, this.Repo.GitObjects, mockVirtualization)) + using (FileSystemCallbacks fileSystemCallbacks = new FileSystemCallbacks( + this.Repo.Context, + this.Repo.GitObjects, + RepoMetadata.Instance, + new MockBlobSizes(), + gitIndexProjection, + backgroundTaskRunner, + virtualizer)) + { + string filePath = "test" + Path.DirectorySeparatorChar + "test.txt"; + gitIndexProjection.MockFileTypesAndModes.TryAdd(filePath, (ushort)GitIndexProjection.FileType.SymLink); + string error; + fileSystemCallbacks.TryStart(out error).ShouldEqual(true); + + virtualizer.WritePlaceholderFile( + filePath, + endOfFile: 0, + sha: string.Empty).ShouldEqual(new FileSystemResult(FSResult.Ok, (int)Result.Success)); + + mockVirtualization.CreatedPlaceholders.ShouldBeEmpty(); + mockVirtualization.CreatedSymLinks.Count.ShouldEqual(1); + mockVirtualization.CreatedSymLinks.ShouldContain(entry => entry.Equals(filePath)); + + // Creating a symlink should schedule a background task + backgroundTaskRunner.Count.ShouldEqual(1); + backgroundTaskRunner.BackgroundTasks[0].Operation.ShouldEqual(GVFS.Virtualization.Background.FileSystemTask.OperationType.OnFileSymLinkCreated); + backgroundTaskRunner.BackgroundTasks[0].VirtualPath.ShouldEqual(filePath); + + fileSystemCallbacks.Stop(); + } + } + + [TestCase] + public void UpdatePlaceholderToSymLink() + { + using (MockBackgroundFileSystemTaskRunner backgroundTaskRunner = new MockBackgroundFileSystemTaskRunner()) + using (MockVirtualizationInstance mockVirtualization = new MockVirtualizationInstance()) + using (MockGitIndexProjection gitIndexProjection = new MockGitIndexProjection(new[] { "test.txt" })) + using (MacFileSystemVirtualizer virtualizer = new MacFileSystemVirtualizer(this.Repo.Context, this.Repo.GitObjects, mockVirtualization)) + using (FileSystemCallbacks fileSystemCallbacks = new FileSystemCallbacks( + this.Repo.Context, + this.Repo.GitObjects, + RepoMetadata.Instance, + new MockBlobSizes(), + gitIndexProjection, + backgroundTaskRunner, + virtualizer)) + { + string filePath = "test" + Path.DirectorySeparatorChar + "test.txt"; + gitIndexProjection.MockFileTypesAndModes.TryAdd(filePath, (ushort)GitIndexProjection.FileType.SymLink); + string error; + fileSystemCallbacks.TryStart(out error).ShouldEqual(true); + + UpdateFailureReason failureReason = UpdateFailureReason.NoFailure; + + mockVirtualization.UpdatePlaceholderIfNeededResult = Result.Success; + mockVirtualization.UpdatePlaceholderIfNeededFailureCause = UpdateFailureCause.NoFailure; + virtualizer + .UpdatePlaceholderIfNeeded( + filePath, + DateTime.Now, + DateTime.Now, + DateTime.Now, + DateTime.Now, + 0, + 15, + string.Empty, + UpdatePlaceholderType.AllowReadOnly, + out failureReason) + .ShouldEqual(new FileSystemResult(FSResult.Ok, (int)mockVirtualization.UpdatePlaceholderIfNeededResult)); + failureReason.ShouldEqual((UpdateFailureReason)mockVirtualization.UpdatePlaceholderIfNeededFailureCause); + + mockVirtualization.CreatedSymLinks.Count.ShouldEqual(1); + mockVirtualization.CreatedSymLinks.ShouldContain(entry => entry.Equals(filePath)); + + // Creating a symlink should schedule a background task + backgroundTaskRunner.Count.ShouldEqual(1); + backgroundTaskRunner.BackgroundTasks[0].Operation.ShouldEqual(GVFS.Virtualization.Background.FileSystemTask.OperationType.OnFileSymLinkCreated); + backgroundTaskRunner.BackgroundTasks[0].VirtualPath.ShouldEqual(filePath); + + fileSystemCallbacks.Stop(); + } + } + [TestCase] public void ClearNegativePathCacheIsNoOp() { @@ -191,7 +282,7 @@ public void OnEnumerateDirectoryReturnsSuccessWhenResultsNotInMemory() backgroundFileSystemTaskRunner: backgroundTaskRunner, fileSystemVirtualizer: virtualizer)) { - gitIndexProjection.MockFileTypesAndModes.TryAdd("test" + Path.DirectorySeparatorChar + "test.txt", (ushort)(RegularFileType | FileMode644)); + gitIndexProjection.MockFileTypesAndModes.TryAdd("test" + Path.DirectorySeparatorChar + "test.txt", (ushort)((ushort)GitIndexProjection.FileType.Regular | FileMode644)); string error; fileSystemCallbacks.TryStart(out error).ShouldEqual(true); @@ -221,7 +312,7 @@ public void OnEnumerateDirectoryReturnsSuccessWhenResultsInMemory() backgroundFileSystemTaskRunner: backgroundTaskRunner, fileSystemVirtualizer: virtualizer)) { - gitIndexProjection.MockFileTypesAndModes.TryAdd("test" + Path.DirectorySeparatorChar + "test.txt", (ushort)(RegularFileType | FileMode644)); + gitIndexProjection.MockFileTypesAndModes.TryAdd("test" + Path.DirectorySeparatorChar + "test.txt", (ushort)((ushort)GitIndexProjection.FileType.Regular | FileMode644)); string error; fileSystemCallbacks.TryStart(out error).ShouldEqual(true); @@ -252,9 +343,9 @@ public void OnEnumerateDirectorySetsFileModes() backgroundFileSystemTaskRunner: backgroundTaskRunner, fileSystemVirtualizer: virtualizer)) { - gitIndexProjection.MockFileTypesAndModes.TryAdd("test" + Path.DirectorySeparatorChar + "test644.txt", (ushort)(RegularFileType | FileMode644)); - gitIndexProjection.MockFileTypesAndModes.TryAdd("test" + Path.DirectorySeparatorChar + "test664.txt", (ushort)(RegularFileType | FileMode664)); - gitIndexProjection.MockFileTypesAndModes.TryAdd("test" + Path.DirectorySeparatorChar + "test755.txt", (ushort)(RegularFileType | FileMode755)); + gitIndexProjection.MockFileTypesAndModes.TryAdd("test" + Path.DirectorySeparatorChar + "test644.txt", (ushort)((ushort)GitIndexProjection.FileType.Regular | FileMode644)); + gitIndexProjection.MockFileTypesAndModes.TryAdd("test" + Path.DirectorySeparatorChar + "test664.txt", (ushort)((ushort)GitIndexProjection.FileType.Regular | FileMode664)); + gitIndexProjection.MockFileTypesAndModes.TryAdd("test" + Path.DirectorySeparatorChar + "test755.txt", (ushort)((ushort)GitIndexProjection.FileType.Regular | FileMode755)); string error; fileSystemCallbacks.TryStart(out error).ShouldEqual(true); From 42a399121a47bb07862afcbd6acdbe1a58621ef4 Mon Sep 17 00:00:00 2001 From: Derrick Stolee Date: Wed, 26 Sep 2018 12:55:14 -0400 Subject: [PATCH 165/272] Fix a flaky test involving the background prefetch thread --- .../Tests/EnlistmentPerFixture/PrefetchVerbTests.cs | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/GVFS/GVFS.FunctionalTests/Tests/EnlistmentPerFixture/PrefetchVerbTests.cs b/GVFS/GVFS.FunctionalTests/Tests/EnlistmentPerFixture/PrefetchVerbTests.cs index cb0a2a4a6..bcfa9cbd3 100644 --- a/GVFS/GVFS.FunctionalTests/Tests/EnlistmentPerFixture/PrefetchVerbTests.cs +++ b/GVFS/GVFS.FunctionalTests/Tests/EnlistmentPerFixture/PrefetchVerbTests.cs @@ -131,10 +131,13 @@ private void PostFetchJobShouldComplete() string objectDir = this.Enlistment.GetObjectRoot(this.fileSystem); string postFetchLock = Path.Combine(objectDir, "post-fetch.lock"); - while (this.fileSystem.FileExists(postFetchLock)) + // Wait first, to hopefully ensure the background thread has + // started before we check for the lock file. + do { Thread.Sleep(500); } + while (this.fileSystem.FileExists(postFetchLock)); ProcessResult midxResult = GitProcess.InvokeProcess(this.Enlistment.RepoRoot, "midx --read --pack-dir=\"" + objectDir + "/pack\""); midxResult.ExitCode.ShouldEqual(0); From 6c38d6c1d702f231f937d73a95c206da4448c6ce Mon Sep 17 00:00:00 2001 From: Kevin Willford Date: Thu, 27 Sep 2018 14:07:23 -0600 Subject: [PATCH 166/272] Add wait for background operations to complete in modified paths test --- .../Tests/EnlistmentPerTestCase/ModifiedPathsTests.cs | 2 ++ 1 file changed, 2 insertions(+) diff --git a/GVFS/GVFS.FunctionalTests/Tests/EnlistmentPerTestCase/ModifiedPathsTests.cs b/GVFS/GVFS.FunctionalTests/Tests/EnlistmentPerTestCase/ModifiedPathsTests.cs index e3e9de6a5..ee4c1e060 100644 --- a/GVFS/GVFS.FunctionalTests/Tests/EnlistmentPerTestCase/ModifiedPathsTests.cs +++ b/GVFS/GVFS.FunctionalTests/Tests/EnlistmentPerTestCase/ModifiedPathsTests.cs @@ -193,6 +193,8 @@ public void ModifiedPathsCorrectAfterHardLinking(FileSystemRunner fileSystem) fileSystem.CreateHardLink(hardLinkOutsideRepoToFileInRepoPath, secondFileInRepoPath); hardLinkOutsideRepoToFileInRepoPath.ShouldBeAFile(fileSystem).WithContents(contents); + this.Enlistment.WaitForBackgroundOperations().ShouldEqual(true, "Background operations failed to complete."); + string modifiedPathsDatabase = Path.Combine(this.Enlistment.DotGVFSRoot, TestConstants.Databases.ModifiedPaths); modifiedPathsDatabase.ShouldBeAFile(fileSystem); using (StreamReader reader = new StreamReader(File.Open(modifiedPathsDatabase, FileMode.Open, FileAccess.Read, FileShare.ReadWrite))) From 25ac9c543b449632a52593c65fccedbea8a30685 Mon Sep 17 00:00:00 2001 From: William Baker Date: Thu, 27 Sep 2018 09:14:47 -0700 Subject: [PATCH 167/272] PR Feedback: Cleanup and flag link as in root --- .../FileSystemRunners/BashRunner.cs | 2 +- .../MacFileSystemVirtualizer.cs | 99 ++++++++++++------- .../Mock/Mac/MockVirtualizationInstance.cs | 4 +- .../Projection/MockGitIndexProjection.cs | 10 +- .../MacFileSystemVirtualizerTests.cs | 16 +-- .../Projection/FileTypeAndMode.cs | 78 +++++++++++++++ .../GitIndexProjection.GitIndexEntry.cs | 2 +- .../GitIndexProjection.GitIndexParser.cs | 31 +++--- .../Projection/GitIndexProjection.cs | 35 ++----- .../PrjFSLib.Mac.Managed/Interop/PrjFSLib.cs | 4 +- .../VirtualizationInstance.cs | 8 +- ProjFS.Mac/PrjFSLib/PrjFSLib.cpp | 22 +++-- ProjFS.Mac/PrjFSLib/PrjFSLib.h | 4 +- 13 files changed, 202 insertions(+), 113 deletions(-) create mode 100644 GVFS/GVFS.Virtualization/Projection/FileTypeAndMode.cs diff --git a/GVFS/GVFS.FunctionalTests/FileSystemRunners/BashRunner.cs b/GVFS/GVFS.FunctionalTests/FileSystemRunners/BashRunner.cs index af35b8737..747345cd5 100644 --- a/GVFS/GVFS.FunctionalTests/FileSystemRunners/BashRunner.cs +++ b/GVFS/GVFS.FunctionalTests/FileSystemRunners/BashRunner.cs @@ -58,7 +58,7 @@ private enum FileType Invalid, File, Directory, - SymLink + SymLink, } protected override string FileName diff --git a/GVFS/GVFS.Platform.Mac/MacFileSystemVirtualizer.cs b/GVFS/GVFS.Platform.Mac/MacFileSystemVirtualizer.cs index c1fa1f340..e351f4f75 100644 --- a/GVFS/GVFS.Platform.Mac/MacFileSystemVirtualizer.cs +++ b/GVFS/GVFS.Platform.Mac/MacFileSystemVirtualizer.cs @@ -17,6 +17,8 @@ public class MacFileSystemVirtualizer : FileSystemVirtualizer { public static readonly byte[] PlaceholderVersionId = ToVersionIdByteArray(new byte[] { PlaceholderVersion }); + private const int SymLinkTargetBufferSize = 4096; + private const string ClassName = nameof(MacFileSystemVirtualizer); private VirtualizationInstance virtualizationInstance; @@ -81,26 +83,25 @@ public override FileSystemResult WritePlaceholderFile( string sha) { // TODO(Mac): Add functional tests that validate file mode is set correctly - ushort fileTypeAndMode = this.FileSystemCallbacks.GitIndexProjection.GetFileTypeAndMode(relativePath); - GitIndexProjection.FileType fileType = (GitIndexProjection.FileType)(fileTypeAndMode & GitIndexProjection.FileTypeMask); + FileTypeAndMode fileTypeAndMode = this.FileSystemCallbacks.GitIndexProjection.GetFileTypeAndMode(relativePath); - if (fileType == GitIndexProjection.FileType.Regular) + if (fileTypeAndMode.Type == FileTypeAndMode.FileType.Regular) { Result result = this.virtualizationInstance.WritePlaceholderFile( relativePath, PlaceholderVersionId, ToVersionIdByteArray(FileSystemVirtualizer.ConvertShaToContentId(sha)), (ulong)endOfFile, - (ushort)(fileTypeAndMode & GitIndexProjection.FileModeMask)); + fileTypeAndMode.Mode); return new FileSystemResult(ResultToFSResult(result), unchecked((int)result)); } - else if (fileType == GitIndexProjection.FileType.SymLink) + else if (fileTypeAndMode.Type == FileTypeAndMode.FileType.SymLink) { - string symLinkContents; - if (this.TryGetSymLinkObjectContents(sha, out symLinkContents)) + string symLinkTarget; + if (this.TryGetSymLinkTarget(sha, out symLinkTarget)) { - Result result = this.virtualizationInstance.WriteSymLink(relativePath, symLinkContents); + Result result = this.virtualizationInstance.WriteSymLink(relativePath, symLinkTarget); this.FileSystemCallbacks.OnFileSymLinkCreated(relativePath); @@ -115,7 +116,7 @@ public override FileSystemResult WritePlaceholderFile( else { EventMetadata metadata = this.CreateEventMetadata(relativePath); - metadata.Add(nameof(fileType), fileType); + metadata.Add("FileType", fileTypeAndMode.Type); this.Context.Tracer.RelatedError(metadata, $"{nameof(this.WritePlaceholderFile)}: Unsupported fileType"); return new FileSystemResult(FSResult.IOError, 0); } @@ -144,31 +145,30 @@ public override FileSystemResult UpdatePlaceholderIfNeeded( // TODO(Mac): Add functional tests that include: // - Mode + content changes between commits // - Mode only changes (without any change to content, see issue #223) - ushort fileTypeAndMode = this.FileSystemCallbacks.GitIndexProjection.GetFileTypeAndMode(relativePath); - GitIndexProjection.FileType fileType = (GitIndexProjection.FileType)(fileTypeAndMode & GitIndexProjection.FileTypeMask); + FileTypeAndMode fileTypeAndMode = this.FileSystemCallbacks.GitIndexProjection.GetFileTypeAndMode(relativePath); - if (fileType == GitIndexProjection.FileType.Regular) + if (fileTypeAndMode.Type == FileTypeAndMode.FileType.Regular) { Result result = this.virtualizationInstance.UpdatePlaceholderIfNeeded( relativePath, PlaceholderVersionId, ToVersionIdByteArray(ConvertShaToContentId(shaContentId)), (ulong)endOfFile, - (ushort)(fileTypeAndMode & GitIndexProjection.FileModeMask), + fileTypeAndMode.Mode, (UpdateType)updateFlags, out failureCause); failureReason = (UpdateFailureReason)failureCause; return new FileSystemResult(ResultToFSResult(result), unchecked((int)result)); } - else if (fileType == GitIndexProjection.FileType.SymLink) + else if (fileTypeAndMode.Type == FileTypeAndMode.FileType.SymLink) { - string symLinkContents; - if (this.TryGetSymLinkObjectContents(shaContentId, out symLinkContents)) + string symLinkTarget; + if (this.TryGetSymLinkTarget(shaContentId, out symLinkTarget)) { Result result = this.virtualizationInstance.ReplacePlaceholderFileWithSymLink( relativePath, - symLinkContents, + symLinkTarget, (UpdateType)updateFlags, out failureCause); @@ -187,8 +187,8 @@ public override FileSystemResult UpdatePlaceholderIfNeeded( else { EventMetadata metadata = this.CreateEventMetadata(relativePath); - metadata.Add(nameof(fileType), fileType); - metadata.Add(nameof(fileTypeAndMode), fileTypeAndMode); + metadata.Add("FileType", fileTypeAndMode.Type); + metadata.Add("FileMode", fileTypeAndMode.Mode); this.Context.Tracer.RelatedError(metadata, $"{nameof(this.UpdatePlaceholderIfNeeded)}: Unsupported fileType"); failureReason = UpdateFailureReason.NoFailure; return new FileSystemResult(FSResult.IOError, 0); @@ -232,11 +232,16 @@ private static byte[] ToVersionIdByteArray(byte[] version) return bytes; } - private bool TryGetSymLinkObjectContents(string sha, out string contents) + ///

+ /// Gets the target of the symbolic link. + /// + /// SHA of the loose object containing the target path of the symbolic link + /// Target path of the symbolic link + private bool TryGetSymLinkTarget(string sha, out string symLinkTarget) { - contents = null; - StringBuilder objectContents = new StringBuilder(); + symLinkTarget = null; + string symLinkBlobContents = null; try { if (!this.GitObjects.TryCopyBlobContentStream( @@ -245,9 +250,10 @@ private bool TryGetSymLinkObjectContents(string sha, out string contents) GVFSGitObjects.RequestSource.SymLinkCreation, (stream, blobLength) => { - // TODO(Mac): Find a better solution than reading from the stream one byte at at time - byte[] buffer = new byte[4096]; + byte[] buffer = new byte[SymLinkTargetBufferSize]; uint bufferIndex = 0; + + // TODO(Mac): Find a better solution than reading from the stream one byte at at time int nextByte = stream.ReadByte(); while (nextByte != -1) { @@ -258,38 +264,51 @@ private bool TryGetSymLinkObjectContents(string sha, out string contents) ++bufferIndex; } - while (bufferIndex < buffer.Length) + if (bufferIndex < buffer.Length) { buffer[bufferIndex] = 0; - ++bufferIndex; - } - - objectContents.Append(System.Text.Encoding.ASCII.GetString(buffer)); - - if (bufferIndex == buffer.Length) + symLinkBlobContents = Encoding.UTF8.GetString(buffer); + } + else { - bufferIndex = 0; + buffer[bufferIndex - 1] = 0; + + EventMetadata metadata = this.CreateEventMetadata(); + metadata.Add(nameof(sha), sha); + metadata.Add("bufferContents", Encoding.UTF8.GetString(buffer)); + this.Context.Tracer.RelatedError(metadata, $"{nameof(this.TryGetSymLinkTarget)}: SymLink target exceeds buffer size"); + + throw new GetSymLinkTargetException("SymLink target exceeds buffer size");; } } })) { EventMetadata metadata = this.CreateEventMetadata(); metadata.Add(nameof(sha), sha); - this.Context.Tracer.RelatedError(metadata, $"{nameof(this.TryGetSymLinkObjectContents)}: TryCopyBlobContentStream failed"); + this.Context.Tracer.RelatedError(metadata, $"{nameof(this.TryGetSymLinkTarget)}: TryCopyBlobContentStream failed"); return false; } } - catch (GetFileStreamException e) + catch (GetSymLinkTargetException e) { EventMetadata metadata = this.CreateEventMetadata(relativePath: null, exception: e); metadata.Add(nameof(sha), sha); - this.Context.Tracer.RelatedError(metadata, $"{nameof(this.TryGetSymLinkObjectContents)}: TryCopyBlobContentStream caught GetFileStreamException"); + this.Context.Tracer.RelatedError(metadata, $"{nameof(this.TryGetSymLinkTarget)}: TryCopyBlobContentStream caught GetSymLinkTargetException"); return false; } + catch (DecoderFallbackException e) + { + EventMetadata metadata = this.CreateEventMetadata(relativePath: null, exception: e); + metadata.Add(nameof(sha), sha); + this.Context.Tracer.RelatedError(metadata, $"{nameof(this.TryGetSymLinkTarget)}: TryCopyBlobContentStream caught DecoderFallbackException"); + + return false; + } + + symLinkTarget = symLinkBlobContents; - contents = objectContents.ToString(); return true; } @@ -664,5 +683,13 @@ public GetFileStreamException(string message, Result result) public Result Result { get; } } + + private class GetSymLinkTargetException : Exception + { + public GetSymLinkTargetException(string message) + : base(message) + { + } + } } } diff --git a/GVFS/GVFS.UnitTests/Mock/Mac/MockVirtualizationInstance.cs b/GVFS/GVFS.UnitTests/Mock/Mac/MockVirtualizationInstance.cs index 082565d22..156ae8382 100644 --- a/GVFS/GVFS.UnitTests/Mock/Mac/MockVirtualizationInstance.cs +++ b/GVFS/GVFS.UnitTests/Mock/Mac/MockVirtualizationInstance.cs @@ -84,7 +84,7 @@ public override Result WritePlaceholderFile( public override Result WriteSymLink( string relativePath, - string symLinkContents) + string symLinkTarget) { this.CreatedSymLinks.Add(relativePath); return Result.Success; @@ -105,7 +105,7 @@ public override Result UpdatePlaceholderIfNeeded( public override Result ReplacePlaceholderFileWithSymLink( string relativePath, - string symLinkContents, + string symLinkTarget, UpdateType updateFlags, out UpdateFailureCause failureCause) { diff --git a/GVFS/GVFS.UnitTests/Mock/Virtualization/Projection/MockGitIndexProjection.cs b/GVFS/GVFS.UnitTests/Mock/Virtualization/Projection/MockGitIndexProjection.cs index 9bd7c5f5a..ca877baa8 100644 --- a/GVFS/GVFS.UnitTests/Mock/Virtualization/Projection/MockGitIndexProjection.cs +++ b/GVFS/GVFS.UnitTests/Mock/Virtualization/Projection/MockGitIndexProjection.cs @@ -36,7 +36,7 @@ public MockGitIndexProjection(IEnumerable projectedFiles) this.PlaceholdersCreated = new ConcurrentHashSet(); this.ExpandedFolders = new ConcurrentHashSet(); - this.MockFileTypesAndModes = new ConcurrentDictionary(); + this.MockFileTypesAndModes = new ConcurrentDictionary(); this.unblockGetProjectedItems = new ManualResetEvent(true); this.waitForGetProjectedItems = new ManualResetEvent(true); @@ -56,7 +56,7 @@ public MockGitIndexProjection(IEnumerable projectedFiles) public ConcurrentHashSet ExpandedFolders { get; } - public ConcurrentDictionary MockFileTypesAndModes { get; } + public ConcurrentDictionary MockFileTypesAndModes { get; } public bool ThrowOperationCanceledExceptionOnProjectionRequest { get; set; } @@ -161,15 +161,15 @@ public override bool TryGetProjectedItemsFromMemory(string folderPath, out List< return false; } - public override ushort GetFileTypeAndMode(string path) + public override FileTypeAndMode GetFileTypeAndMode(string path) { - ushort result; + FileTypeAndMode result; if (this.MockFileTypesAndModes.TryGetValue(path, out result)) { return result; } - return 0; + return new FileTypeAndMode(0); } public override List GetProjectedItems( diff --git a/GVFS/GVFS.UnitTests/Platform.Mac/MacFileSystemVirtualizerTests.cs b/GVFS/GVFS.UnitTests/Platform.Mac/MacFileSystemVirtualizerTests.cs index 9be03ae2c..81c52e43f 100644 --- a/GVFS/GVFS.UnitTests/Platform.Mac/MacFileSystemVirtualizerTests.cs +++ b/GVFS/GVFS.UnitTests/Platform.Mac/MacFileSystemVirtualizerTests.cs @@ -100,7 +100,7 @@ public void UpdatePlaceholderIfNeeded() virtualizer)) { string filePath = "test" + Path.DirectorySeparatorChar + "test.txt"; - gitIndexProjection.MockFileTypesAndModes.TryAdd(filePath, (ushort)((ushort)GitIndexProjection.FileType.Regular | FileMode644)); + gitIndexProjection.MockFileTypesAndModes.TryAdd(filePath, FileTypeAndMode.Regular644File); string error; fileSystemCallbacks.TryStart(out error).ShouldEqual(true); @@ -180,7 +180,7 @@ public void WritePlaceholderForSymLink() virtualizer)) { string filePath = "test" + Path.DirectorySeparatorChar + "test.txt"; - gitIndexProjection.MockFileTypesAndModes.TryAdd(filePath, (ushort)GitIndexProjection.FileType.SymLink); + gitIndexProjection.MockFileTypesAndModes.TryAdd(filePath, new FileTypeAndMode((ushort)FileTypeAndMode.FileType.SymLink)); string error; fileSystemCallbacks.TryStart(out error).ShouldEqual(true); @@ -219,7 +219,7 @@ public void UpdatePlaceholderToSymLink() virtualizer)) { string filePath = "test" + Path.DirectorySeparatorChar + "test.txt"; - gitIndexProjection.MockFileTypesAndModes.TryAdd(filePath, (ushort)GitIndexProjection.FileType.SymLink); + gitIndexProjection.MockFileTypesAndModes.TryAdd(filePath, new FileTypeAndMode((ushort)FileTypeAndMode.FileType.SymLink)); string error; fileSystemCallbacks.TryStart(out error).ShouldEqual(true); @@ -282,7 +282,7 @@ public void OnEnumerateDirectoryReturnsSuccessWhenResultsNotInMemory() backgroundFileSystemTaskRunner: backgroundTaskRunner, fileSystemVirtualizer: virtualizer)) { - gitIndexProjection.MockFileTypesAndModes.TryAdd("test" + Path.DirectorySeparatorChar + "test.txt", (ushort)((ushort)GitIndexProjection.FileType.Regular | FileMode644)); + gitIndexProjection.MockFileTypesAndModes.TryAdd("test" + Path.DirectorySeparatorChar + "test.txt", FileTypeAndMode.Regular644File); string error; fileSystemCallbacks.TryStart(out error).ShouldEqual(true); @@ -312,7 +312,7 @@ public void OnEnumerateDirectoryReturnsSuccessWhenResultsInMemory() backgroundFileSystemTaskRunner: backgroundTaskRunner, fileSystemVirtualizer: virtualizer)) { - gitIndexProjection.MockFileTypesAndModes.TryAdd("test" + Path.DirectorySeparatorChar + "test.txt", (ushort)((ushort)GitIndexProjection.FileType.Regular | FileMode644)); + gitIndexProjection.MockFileTypesAndModes.TryAdd("test" + Path.DirectorySeparatorChar + "test.txt", FileTypeAndMode.Regular644File); string error; fileSystemCallbacks.TryStart(out error).ShouldEqual(true); @@ -343,9 +343,9 @@ public void OnEnumerateDirectorySetsFileModes() backgroundFileSystemTaskRunner: backgroundTaskRunner, fileSystemVirtualizer: virtualizer)) { - gitIndexProjection.MockFileTypesAndModes.TryAdd("test" + Path.DirectorySeparatorChar + "test644.txt", (ushort)((ushort)GitIndexProjection.FileType.Regular | FileMode644)); - gitIndexProjection.MockFileTypesAndModes.TryAdd("test" + Path.DirectorySeparatorChar + "test664.txt", (ushort)((ushort)GitIndexProjection.FileType.Regular | FileMode664)); - gitIndexProjection.MockFileTypesAndModes.TryAdd("test" + Path.DirectorySeparatorChar + "test755.txt", (ushort)((ushort)GitIndexProjection.FileType.Regular | FileMode755)); + gitIndexProjection.MockFileTypesAndModes.TryAdd("test" + Path.DirectorySeparatorChar + "test644.txt", FileTypeAndMode.Regular644File); + gitIndexProjection.MockFileTypesAndModes.TryAdd("test" + Path.DirectorySeparatorChar + "test664.txt", new FileTypeAndMode((ushort)((ushort)FileTypeAndMode.FileType.Regular | FileMode664))); + gitIndexProjection.MockFileTypesAndModes.TryAdd("test" + Path.DirectorySeparatorChar + "test755.txt", new FileTypeAndMode((ushort)((ushort)FileTypeAndMode.FileType.Regular | FileMode755))); string error; fileSystemCallbacks.TryStart(out error).ShouldEqual(true); diff --git a/GVFS/GVFS.Virtualization/Projection/FileTypeAndMode.cs b/GVFS/GVFS.Virtualization/Projection/FileTypeAndMode.cs new file mode 100644 index 000000000..0afab8cc2 --- /dev/null +++ b/GVFS/GVFS.Virtualization/Projection/FileTypeAndMode.cs @@ -0,0 +1,78 @@ +using System; +namespace GVFS.Virtualization.Projection +{ + public struct FileTypeAndMode + { + public static readonly ushort FileMode755; + public static readonly ushort FileMode664; + public static readonly ushort FileMode644; + public static readonly FileTypeAndMode Regular644File; + + // Bitmasks for extracting file type and mode from the ushort stored in the index + private const ushort FileTypeMask = 0xF000; + private const ushort FileModeMask = 0x1FF; + + private readonly ushort fileTypeAndMode; + + static FileTypeAndMode() + { + FileMode755 = Convert.ToUInt16("755", 8); + FileMode664 = Convert.ToUInt16("664", 8); + FileMode644 = Convert.ToUInt16("644", 8); + + Regular644File = new FileTypeAndMode((ushort)((ushort)FileType.Regular | FileMode644)); + } + + public FileTypeAndMode(ushort typeAndModeInIndexFormat) + { + this.fileTypeAndMode = typeAndModeInIndexFormat; + } + + public enum FileType : ushort + { + Invalid = 0, + + Regular = 0x8000, + SymLink = 0xA000, + GitLink = 0xE000, + } + + public FileType Type => (FileType)(this.fileTypeAndMode & FileTypeMask); + public ushort Mode => (ushort)(this.fileTypeAndMode & FileModeMask); + + public static bool operator ==(FileTypeAndMode lhs, FileTypeAndMode rhs) + { + return lhs.Equals(rhs); + } + + public static bool operator !=(FileTypeAndMode lhs, FileTypeAndMode rhs) + { + return !lhs.Equals(rhs); + } + + public override bool Equals(object obj) + { + if (obj is FileTypeAndMode) + { + return this.Equals((FileTypeAndMode)obj); + } + + return false; + } + + public override int GetHashCode() + { + return this.fileTypeAndMode; + } + + public bool Equals(FileTypeAndMode otherFileTypeAndMode) + { + return this.fileTypeAndMode == otherFileTypeAndMode.fileTypeAndMode; + } + + public string GetModeAsOctalString() + { + return Convert.ToString(this.Mode, 8); + } + } +} diff --git a/GVFS/GVFS.Virtualization/Projection/GitIndexProjection.GitIndexEntry.cs b/GVFS/GVFS.Virtualization/Projection/GitIndexProjection.GitIndexEntry.cs index 612931242..8cd38e283 100644 --- a/GVFS/GVFS.Virtualization/Projection/GitIndexProjection.GitIndexEntry.cs +++ b/GVFS/GVFS.Virtualization/Projection/GitIndexProjection.GitIndexEntry.cs @@ -27,7 +27,7 @@ public GitIndexEntry() public byte[] Sha { get; } = new byte[20]; public bool SkipWorktree { get; set; } - public ushort FileTypeAndMode { get; set; } + public FileTypeAndMode TypeAndMode { get; set; } public GitIndexParser.MergeStage MergeState { get; set; } public int ReplaceIndex { get; set; } diff --git a/GVFS/GVFS.Virtualization/Projection/GitIndexProjection.GitIndexParser.cs b/GVFS/GVFS.Virtualization/Projection/GitIndexProjection.GitIndexParser.cs index ca5f88dad..3753abb7c 100644 --- a/GVFS/GVFS.Virtualization/Projection/GitIndexProjection.GitIndexParser.cs +++ b/GVFS/GVFS.Virtualization/Projection/GitIndexProjection.GitIndexParser.cs @@ -16,10 +16,6 @@ internal partial class GitIndexParser private const ushort ExtendedBit = 0x4000; private const ushort SkipWorktreeBit = 0x4000; - private static readonly ushort FileMode755 = Convert.ToUInt16("755", 8); - private static readonly ushort FileMode664 = Convert.ToUInt16("644", 8); - private static readonly ushort FileMode644 = Convert.ToUInt16("644", 8); - private Stream indexStream; private byte[] page; private int nextByteIndex; @@ -185,35 +181,36 @@ private FileSystemTaskResult ParseIndex(ITracer tracer, Stream indexStream, Func // 3-bit unused // 9-bit unix permission. Only 0755 and 0644 are valid for regular files. (Legacy repos can also contain 664) // Symbolic links and gitlinks have value 0 in this field. - ushort typeAndMode = this.ReadUInt16(); + ushort indexFormatTypeAndMode = this.ReadUInt16(); - ushort type = (ushort)(typeAndMode & FileTypeMask); - ushort mode = (ushort)(typeAndMode & FileModeMask); + FileTypeAndMode typeAndMode = new FileTypeAndMode(indexFormatTypeAndMode); - switch ((FileType)type) + switch (typeAndMode.Type) { - case FileType.Regular: - if (mode != FileMode755 && mode != FileMode644 && mode != FileMode664) + case FileTypeAndMode.FileType.Regular: + if (typeAndMode.Mode != FileTypeAndMode.FileMode755 && + typeAndMode.Mode != FileTypeAndMode.FileMode644 && + typeAndMode.Mode != FileTypeAndMode.FileMode664) { - throw new InvalidDataException($"Invalid file mode {Convert.ToString(mode, 8)} found for regular file in index"); + throw new InvalidDataException($"Invalid file mode {typeAndMode.GetModeAsOctalString()} found for regular file in index"); } break; - case FileType.SymLink: - case FileType.GitLink: - if (mode != 0) + case FileTypeAndMode.FileType.SymLink: + case FileTypeAndMode.FileType.GitLink: + if (typeAndMode.Mode != 0) { - throw new InvalidDataException($"Invalid file mode {Convert.ToString(mode, 8)} found for link file({type:X}) in index"); + throw new InvalidDataException($"Invalid file mode {typeAndMode.GetModeAsOctalString()} found for link file({typeAndMode.Type:X}) in index"); } break; default: - throw new InvalidDataException($"Invalid file type {type:X} found in index"); + throw new InvalidDataException($"Invalid file type {typeAndMode.Type:X} found in index"); } - this.resuableParsedIndexEntry.FileTypeAndMode = typeAndMode; + this.resuableParsedIndexEntry.TypeAndMode = typeAndMode; this.Skip(12); } diff --git a/GVFS/GVFS.Virtualization/Projection/GitIndexProjection.cs b/GVFS/GVFS.Virtualization/Projection/GitIndexProjection.cs index 8c1dbe924..deaa26d1e 100644 --- a/GVFS/GVFS.Virtualization/Projection/GitIndexProjection.cs +++ b/GVFS/GVFS.Virtualization/Projection/GitIndexProjection.cs @@ -22,12 +22,6 @@ public partial class GitIndexProjection : IDisposable, IProfilerOnlyIndexProject { public const string ProjectionIndexBackupName = "GVFS_projection"; - // Bitmasks for extracting file type and mode from the ushort stored in the index - public const ushort FileTypeMask = 0xF000; - public const ushort FileModeMask = 0x1FF; - - protected static readonly ushort Regular644File = (ushort)((ushort)FileType.Regular | Convert.ToUInt16("644", 8)); - private const int IndexFileStreamBufferSize = 512 * 1024; private const UpdatePlaceholderType FolderPlaceholderDeleteFlags = @@ -56,9 +50,9 @@ public partial class GitIndexProjection : IDisposable, IProfilerOnlyIndexProject // Cache of folder paths (in Windows format) to folder data private ConcurrentDictionary projectionFolderCache = new ConcurrentDictionary(StringComparer.OrdinalIgnoreCase); - // nonDefaultFileModes is only populated when the platform supports file mode - // On platforms that support file modes, file paths that are not in nonDefaultFileModes have mode 644 - private Dictionary nonDefaultFileTypesAndModes = new Dictionary(StringComparer.OrdinalIgnoreCase); + // nonDefaultFileTypesAndModes is only populated when the platform supports file mode + // On platforms that support file modes, file paths that are not in nonDefaultFileTypesAndModes are regular files with mode 644 + private Dictionary nonDefaultFileTypesAndModes = new Dictionary(StringComparer.OrdinalIgnoreCase); private BlobSizes blobSizes; private PlaceholderListDatabase placeholderList; @@ -122,15 +116,6 @@ protected GitIndexProjection() { } - public enum FileType : ushort - { - Invalid = 0, - - Regular = 0x8000, - SymLink = 0xA000, - GitLink = 0xE000, - } - public int EstimatedPlaceholderCount { get @@ -369,7 +354,7 @@ public virtual bool TryGetProjectedItemsFromMemory(string folderPath, out List

Date: Tue, 25 Sep 2018 14:35:00 -0400 Subject: [PATCH 168/272] Set config to useHttpPath=true Hostname is no longer sufficent for VSTS authentication. VSTS now requires dev.azure.com/account to determine the tenant. By setting useHttpPath, credential managers will get the path which contains the account as the first parameter. They can then use this information for auth appropriately. --- GVFS/GVFS.Common/Git/GitConfigSetting.cs | 1 + GVFS/GVFS.Common/Git/GitProcess.cs | 2 +- GVFS/GVFS.UnitTests/Git/GitAuthenticationTests.cs | 2 +- GVFS/GVFS/CommandLine/GVFSVerb.cs | 1 + 4 files changed, 4 insertions(+), 2 deletions(-) diff --git a/GVFS/GVFS.Common/Git/GitConfigSetting.cs b/GVFS/GVFS.Common/Git/GitConfigSetting.cs index 12ea0eb23..fc836d234 100644 --- a/GVFS/GVFS.Common/Git/GitConfigSetting.cs +++ b/GVFS/GVFS.Common/Git/GitConfigSetting.cs @@ -6,6 +6,7 @@ public class GitConfigSetting { public const string CoreVirtualizeObjectsName = "core.virtualizeobjects"; public const string CoreVirtualFileSystemName = "core.virtualfilesystem"; + public const string CredentialUseHttpPath = "credential.useHttpPath"; public GitConfigSetting(string name, params string[] values) { diff --git a/GVFS/GVFS.Common/Git/GitProcess.cs b/GVFS/GVFS.Common/Git/GitProcess.cs index 47ad144e8..5c3bba3cd 100644 --- a/GVFS/GVFS.Common/Git/GitProcess.cs +++ b/GVFS/GVFS.Common/Git/GitProcess.cs @@ -112,7 +112,7 @@ public virtual bool TryGetCredentials( using (ITracer activity = tracer.StartActivity("TryGetCredentials", EventLevel.Informational)) { Result gitCredentialOutput = this.InvokeGitAgainstDotGitFolder( - "credential fill", + "-c " + GitConfigSetting.CredentialUseHttpPath + "=true credential fill", stdin => stdin.Write("url=" + repoUrl + "\n\n"), parseStdOutLine: null); diff --git a/GVFS/GVFS.UnitTests/Git/GitAuthenticationTests.cs b/GVFS/GVFS.UnitTests/Git/GitAuthenticationTests.cs index abe2bac1a..bc22fd7c1 100644 --- a/GVFS/GVFS.UnitTests/Git/GitAuthenticationTests.cs +++ b/GVFS/GVFS.UnitTests/Git/GitAuthenticationTests.cs @@ -194,7 +194,7 @@ private MockGitProcess GetGitProcess() int revocations = 0; gitProcess.SetExpectedCommandResult( - "credential fill", + "-c credential.useHttpPath=true credential fill", () => new GitProcess.Result("username=username\r\npassword=password" + revocations + "\r\n", string.Empty, GitProcess.Result.SuccessCode)); gitProcess.SetExpectedCommandResult( diff --git a/GVFS/GVFS/CommandLine/GVFSVerb.cs b/GVFS/GVFS/CommandLine/GVFSVerb.cs index 325e2dcee..d001699c3 100644 --- a/GVFS/GVFS/CommandLine/GVFSVerb.cs +++ b/GVFS/GVFS/CommandLine/GVFSVerb.cs @@ -92,6 +92,7 @@ public static bool TrySetRequiredGitConfigSettings(Enlistment enlistment) { GitConfigSetting.CoreVirtualizeObjectsName, "true" }, { GitConfigSetting.CoreVirtualFileSystemName, Paths.ConvertPathToGitFormat(GVFSConstants.DotGit.Hooks.VirtualFileSystemPath) }, { "core.hookspath", expectedHooksPath }, + { GitConfigSetting.CredentialUseHttpPath, "true" }, { "credential.validate", "false" }, { "diff.autoRefreshIndex", "false" }, { "gc.auto", "0" }, From ae21413746629f3d7fa0902557017ec92e1882f5 Mon Sep 17 00:00:00 2001 From: Kevin Willford Date: Wed, 26 Sep 2018 14:32:28 -0600 Subject: [PATCH 169/272] Check modified paths for parent directories before adding --- GVFS/GVFS.Common/ModifiedPathsDatabase.cs | 20 +++- .../EnlistmentPerFixture/GitFilesTests.cs | 2 - .../Common/ModifiedPathsDatabaseTests.cs | 95 ++++++++++++++----- 3 files changed, 89 insertions(+), 28 deletions(-) diff --git a/GVFS/GVFS.Common/ModifiedPathsDatabase.cs b/GVFS/GVFS.Common/ModifiedPathsDatabase.cs index ffde4f692..338c1f4ce 100644 --- a/GVFS/GVFS.Common/ModifiedPathsDatabase.cs +++ b/GVFS/GVFS.Common/ModifiedPathsDatabase.cs @@ -1,6 +1,8 @@ using System; using System.Collections.Generic; using System.IO; +using System.Linq; +using System.Text; using GVFS.Common.FileSystem; using GVFS.Common.Tracing; @@ -67,7 +69,7 @@ public bool TryAdd(string path, bool isFolder, out bool isRetryable) { isRetryable = true; string entry = this.NormalizeEntryString(path, isFolder); - if (!this.modifiedPaths.Contains(entry)) + if (!this.modifiedPaths.Contains(entry) && !this.ContainsParentDirectory(entry)) { try { @@ -183,6 +185,22 @@ private bool TryParseRemoveLine(string line, out string key, out string error) return true; } + private bool ContainsParentDirectory(string modifiedPath) + { + string[] pathParts = modifiedPath.Split(new char[] { '/' }, StringSplitOptions.RemoveEmptyEntries); + StringBuilder parentFolder = new StringBuilder(); + foreach (string pathPart in pathParts.Take(pathParts.Length - 1)) + { + parentFolder.Append(pathPart + "/"); + if (this.modifiedPaths.Contains(parentFolder.ToString())) + { + return true; + } + } + + return false; + } + private string NormalizeEntryString(string virtualPath, bool isFolder) { // TODO(Mac) This can be optimized if needed diff --git a/GVFS/GVFS.FunctionalTests/Tests/EnlistmentPerFixture/GitFilesTests.cs b/GVFS/GVFS.FunctionalTests/Tests/EnlistmentPerFixture/GitFilesTests.cs index bfbdcc457..8a0a0c3f6 100644 --- a/GVFS/GVFS.FunctionalTests/Tests/EnlistmentPerFixture/GitFilesTests.cs +++ b/GVFS/GVFS.FunctionalTests/Tests/EnlistmentPerFixture/GitFilesTests.cs @@ -77,7 +77,6 @@ public void CreateFileInFolderTest() this.Enlistment.WaitForBackgroundOperations().ShouldEqual(true, "Background operations failed to complete."); GVFSHelpers.ModifiedPathsShouldContain(this.fileSystem, this.Enlistment.DotGVFSRoot, folderName + "/"); - GVFSHelpers.ModifiedPathsShouldContain(this.fileSystem, this.Enlistment.DotGVFSRoot, folderName + "/" + fileName); } [TestCase, Order(4)] @@ -140,7 +139,6 @@ public void CaseOnlyRenameOfNewFolderKeepsModifiedPathsEntries() string[] expectedModifiedPathsEntriesAfterCreate = { "A Folder/", - "A Folder/testfile", }; string[] expectedModifiedPathsEntriesAfterRename = diff --git a/GVFS/GVFS.UnitTests/Common/ModifiedPathsDatabaseTests.cs b/GVFS/GVFS.UnitTests/Common/ModifiedPathsDatabaseTests.cs index 313484200..663a76bb0 100644 --- a/GVFS/GVFS.UnitTests/Common/ModifiedPathsDatabaseTests.cs +++ b/GVFS/GVFS.UnitTests/Common/ModifiedPathsDatabaseTests.cs @@ -22,31 +22,31 @@ A dir1/dir2/file3.txt [TestCase] public void ParsesExistingDataCorrectly() { - ModifiedPathsDatabase mpd = CreateModifiedPathsDatabase(ExistingEntries); - mpd.Count.ShouldEqual(3); - mpd.Contains("file.txt", isFolder: false).ShouldBeTrue(); - mpd.Contains("dir/file2.txt", isFolder: false).ShouldBeTrue(); - mpd.Contains("dir1/dir2/file3.txt", isFolder: false).ShouldBeTrue(); + ModifiedPathsDatabase modifiedPathsDatabase = CreateModifiedPathsDatabase(ExistingEntries); + modifiedPathsDatabase.Count.ShouldEqual(3); + modifiedPathsDatabase.Contains("file.txt", isFolder: false).ShouldBeTrue(); + modifiedPathsDatabase.Contains("dir/file2.txt", isFolder: false).ShouldBeTrue(); + modifiedPathsDatabase.Contains("dir1/dir2/file3.txt", isFolder: false).ShouldBeTrue(); } [TestCase] public void AddsDefaultEntry() { - ModifiedPathsDatabase mpd = CreateModifiedPathsDatabase(initialContents: string.Empty); - mpd.Count.ShouldEqual(1); - mpd.Contains(DefaultEntry, isFolder: false).ShouldBeTrue(); + ModifiedPathsDatabase modifiedPathsDatabase = CreateModifiedPathsDatabase(initialContents: string.Empty); + modifiedPathsDatabase.Count.ShouldEqual(1); + modifiedPathsDatabase.Contains(DefaultEntry, isFolder: false).ShouldBeTrue(); } [TestCase] public void BadDataFailsToLoad() { - ConfigurableFileSystem fs = new ConfigurableFileSystem(); - fs.ExpectedFiles.Add(MockEntryFileName, new ReusableMemoryStream("This is bad data!\r\n")); + ConfigurableFileSystem configurableFileSystem = new ConfigurableFileSystem(); + configurableFileSystem.ExpectedFiles.Add(MockEntryFileName, new ReusableMemoryStream("This is bad data!\r\n")); string error; - ModifiedPathsDatabase mpd; - ModifiedPathsDatabase.TryLoadOrCreate(null, MockEntryFileName, fs, out mpd, out error).ShouldBeFalse(); - mpd.ShouldBeNull(); + ModifiedPathsDatabase modifiedPathsDatabase; + ModifiedPathsDatabase.TryLoadOrCreate(null, MockEntryFileName, configurableFileSystem, out modifiedPathsDatabase, out error).ShouldBeFalse(); + modifiedPathsDatabase.ShouldBeNull(); } [TestCase] @@ -74,6 +74,51 @@ public void DirectorySeparatorAddedForFolder() TestAddingPath(pathToAdd: Path.Combine("dir", "subdir"), pathInList: Path.Combine("dir", "subdir") + Path.DirectorySeparatorChar, isFolder: true); } + [TestCase] + public void EntryNotAddedIfParentDirectoryExists() + { + ModifiedPathsDatabase modifiedPathsDatabase = CreateModifiedPathsDatabase(initialContents: "A dir/\r\n"); + modifiedPathsDatabase.Count.ShouldEqual(1); + modifiedPathsDatabase.Contains("dir", isFolder: true).ShouldBeTrue(); + + // Try adding a file for the directory that is in the modified paths + modifiedPathsDatabase.TryAdd("dir/file.txt", isFolder: false, isRetryable: out _); + modifiedPathsDatabase.Count.ShouldEqual(1); + modifiedPathsDatabase.Contains("dir", isFolder: true).ShouldBeTrue(); + + // Try adding a directory for the directory that is in the modified paths + modifiedPathsDatabase.TryAdd("dir/dir2", isFolder: true, isRetryable: out _); + modifiedPathsDatabase.Count.ShouldEqual(1); + modifiedPathsDatabase.Contains("dir", isFolder: true).ShouldBeTrue(); + + // Try adding a file for a directory that is not in the modified paths + modifiedPathsDatabase.TryAdd("dir2/file.txt", isFolder: false, isRetryable: out _); + modifiedPathsDatabase.Count.ShouldEqual(2); + modifiedPathsDatabase.Contains("dir", isFolder: true).ShouldBeTrue(); + modifiedPathsDatabase.Contains("dir2/file.txt", isFolder: false).ShouldBeTrue(); + + // Try adding a directory for a the directory that is not in the modified paths + modifiedPathsDatabase.TryAdd("dir2/dir", isFolder: true, isRetryable: out _); + modifiedPathsDatabase.Count.ShouldEqual(3); + modifiedPathsDatabase.Contains("dir", isFolder: true).ShouldBeTrue(); + modifiedPathsDatabase.Contains("dir2/file.txt", isFolder: false).ShouldBeTrue(); + modifiedPathsDatabase.Contains("dir2/dir", isFolder: true).ShouldBeTrue(); + + // Try adding a file in a subdirectory that is in the modified paths + modifiedPathsDatabase.TryAdd("dir2/dir/file.txt", isFolder: false, isRetryable: out _); + modifiedPathsDatabase.Count.ShouldEqual(3); + modifiedPathsDatabase.Contains("dir", isFolder: true).ShouldBeTrue(); + modifiedPathsDatabase.Contains("dir2/file.txt", isFolder: false).ShouldBeTrue(); + modifiedPathsDatabase.Contains("dir2/dir", isFolder: true).ShouldBeTrue(); + + // Try adding a directory for a subdirectory that is in the modified paths + modifiedPathsDatabase.TryAdd("dir2/dir/dir3", isFolder: true, isRetryable: out _); + modifiedPathsDatabase.Count.ShouldEqual(3); + modifiedPathsDatabase.Contains("dir", isFolder: true).ShouldBeTrue(); + modifiedPathsDatabase.Contains("dir2/file.txt", isFolder: false).ShouldBeTrue(); + modifiedPathsDatabase.Contains("dir2/dir", isFolder: true).ShouldBeTrue(); + } + private static void TestAddingPath(string path, bool isFolder = false) { TestAddingPath(path, path, isFolder); @@ -81,13 +126,13 @@ private static void TestAddingPath(string path, bool isFolder = false) private static void TestAddingPath(string pathToAdd, string pathInList, bool isFolder = false) { - ModifiedPathsDatabase mpd = CreateModifiedPathsDatabase(initialContents: $"A {DefaultEntry}\r\n"); + ModifiedPathsDatabase modifiedPathsDatabase = CreateModifiedPathsDatabase(initialContents: $"A {DefaultEntry}\r\n"); bool isRetryable; - mpd.TryAdd(pathToAdd, isFolder, out isRetryable); - mpd.Count.ShouldEqual(2); - mpd.Contains(pathInList, isFolder).ShouldBeTrue(); - mpd.Contains(ToGitPathSeparators(pathInList), isFolder).ShouldBeTrue(); - mpd.GetAllModifiedPaths().ShouldContainSingle(x => string.Compare(x, ToGitPathSeparators(pathInList), StringComparison.OrdinalIgnoreCase) == 0); + modifiedPathsDatabase.TryAdd(pathToAdd, isFolder, out isRetryable); + modifiedPathsDatabase.Count.ShouldEqual(2); + modifiedPathsDatabase.Contains(pathInList, isFolder).ShouldBeTrue(); + modifiedPathsDatabase.Contains(ToGitPathSeparators(pathInList), isFolder).ShouldBeTrue(); + modifiedPathsDatabase.GetAllModifiedPaths().ShouldContainSingle(x => string.Compare(x, ToGitPathSeparators(pathInList), StringComparison.OrdinalIgnoreCase) == 0); } private static string ToGitPathSeparators(string path) @@ -102,14 +147,14 @@ private static string ToPathSeparators(string path) private static ModifiedPathsDatabase CreateModifiedPathsDatabase(string initialContents) { - ConfigurableFileSystem fs = new ConfigurableFileSystem(); - fs.ExpectedFiles.Add(MockEntryFileName, new ReusableMemoryStream(initialContents)); + ConfigurableFileSystem configurableFileSystem = new ConfigurableFileSystem(); + configurableFileSystem.ExpectedFiles.Add(MockEntryFileName, new ReusableMemoryStream(initialContents)); string error; - ModifiedPathsDatabase mpd; - ModifiedPathsDatabase.TryLoadOrCreate(null, MockEntryFileName, fs, out mpd, out error).ShouldBeTrue(); - mpd.ShouldNotBeNull(); - return mpd; + ModifiedPathsDatabase modifiedPathsDatabase; + ModifiedPathsDatabase.TryLoadOrCreate(null, MockEntryFileName, configurableFileSystem, out modifiedPathsDatabase, out error).ShouldBeTrue(); + modifiedPathsDatabase.ShouldNotBeNull(); + return modifiedPathsDatabase; } } } From 8c8c06f9b7f198d986d20c29481e0066260e14e3 Mon Sep 17 00:00:00 2001 From: Kevin Willford Date: Fri, 28 Sep 2018 11:59:51 -0600 Subject: [PATCH 170/272] Fix test the will no longer have child entries of a folder in the modified paths --- .../Windows/Tests/DiskLayoutUpgradeTests.cs | 2 -- 1 file changed, 2 deletions(-) diff --git a/GVFS/GVFS.FunctionalTests.Windows/Windows/Tests/DiskLayoutUpgradeTests.cs b/GVFS/GVFS.FunctionalTests.Windows/Windows/Tests/DiskLayoutUpgradeTests.cs index eb1ce473d..1f39f0285 100644 --- a/GVFS/GVFS.FunctionalTests.Windows/Windows/Tests/DiskLayoutUpgradeTests.cs +++ b/GVFS/GVFS.FunctionalTests.Windows/Windows/Tests/DiskLayoutUpgradeTests.cs @@ -328,8 +328,6 @@ public void MountCreatesModifiedPathsDatabase() { "A .gitattributes", "A developer/me/", - "A developer/me/JLANGE9._prerazzle", - "A developer/me/StateSwitch.Save", "A tools/x86/remote.exe", "A tools/x86/runelevated.exe", "A tools/amd64/remote.exe", From d4ae8e113ab56b77028a9125c4f5c3b17b65d751 Mon Sep 17 00:00:00 2001 From: William Baker Date: Fri, 28 Sep 2018 11:40:39 -0700 Subject: [PATCH 171/272] PR Feedback: Code cleanup --- .../PrjFSKext/PrjFSKext/KauthHandler.cpp | 18 ++++++++++++------ ProjFS.Mac/PrjFSLib/PrjFSLib.cpp | 1 - 2 files changed, 12 insertions(+), 7 deletions(-) diff --git a/ProjFS.Mac/PrjFSKext/PrjFSKext/KauthHandler.cpp b/ProjFS.Mac/PrjFSKext/PrjFSKext/KauthHandler.cpp index 13e4cacb5..dbf425d53 100644 --- a/ProjFS.Mac/PrjFSKext/PrjFSKext/KauthHandler.cpp +++ b/ProjFS.Mac/PrjFSKext/PrjFSKext/KauthHandler.cpp @@ -266,6 +266,8 @@ static int HandleVnodeOperation( uint32_t currentVnodeFileFlags; int pid; char procname[MAXCOMLEN + 1]; + bool isDeleteAction = false; + bool isDirectory = false; // TODO(Mac): Issue #271 - Reduce reliance on vn_getpath // Call vn_getpath first when the cache is hottest to increase the chances @@ -289,11 +291,16 @@ static int HandleVnodeOperation( goto CleanupAndReturn; } - if (ActionBitIsSet(action, KAUTH_VNODE_DELETE)) + isDeleteAction = ActionBitIsSet(action, KAUTH_VNODE_DELETE); + isDirectory = VDIR == vnodeType; + + if (isDeleteAction) { if (!TrySendRequestAndWaitForResponse( root, - VDIR == vnodeType ? MessageType_KtoU_NotifyDirectoryPreDelete : MessageType_KtoU_NotifyFilePreDelete, + isDirectory ? + MessageType_KtoU_NotifyDirectoryPreDelete : + MessageType_KtoU_NotifyFilePreDelete, currentVnode, vnodePath, pid, @@ -305,7 +312,7 @@ static int HandleVnodeOperation( } } - if (VDIR == vnodeType) + if (isDirectory) { if (ActionBitIsSet( action, @@ -316,13 +323,12 @@ static int HandleVnodeOperation( KAUTH_VNODE_READ_EXTATTRIBUTES | KAUTH_VNODE_DELETE)) { - bool deleteAction = ActionBitIsSet(action, KAUTH_VNODE_DELETE); // Recursively expand directory on delete to ensure child placeholders are created before rename operations - if (deleteAction || FileFlagsBitIsSet(currentVnodeFileFlags, FileFlags_IsEmpty)) + if (isDeleteAction || FileFlagsBitIsSet(currentVnodeFileFlags, FileFlags_IsEmpty)) { if (!TrySendRequestAndWaitForResponse( root, - deleteAction ? + isDeleteAction ? MessageType_KtoU_RecursivelyEnumerateDirectory : MessageType_KtoU_EnumerateDirectory, currentVnode, diff --git a/ProjFS.Mac/PrjFSLib/PrjFSLib.cpp b/ProjFS.Mac/PrjFSLib/PrjFSLib.cpp index eb67b38dc..62f0175fd 100644 --- a/ProjFS.Mac/PrjFSLib/PrjFSLib.cpp +++ b/ProjFS.Mac/PrjFSLib/PrjFSLib.cpp @@ -8,7 +8,6 @@ #include #include #include -#include #include #include #include From 63bd3e0b55dfd449694106aba5f3fdaf1f8a602e Mon Sep 17 00:00:00 2001 From: William Baker Date: Fri, 28 Sep 2018 11:55:57 -0700 Subject: [PATCH 172/272] PR Feedback: Remove explicit std namespace usage --- ProjFS.Mac/PrjFSLib/PrjFSLib.cpp | 82 +++++++++++++++++++------------- 1 file changed, 49 insertions(+), 33 deletions(-) diff --git a/ProjFS.Mac/PrjFSLib/PrjFSLib.cpp b/ProjFS.Mac/PrjFSLib/PrjFSLib.cpp index 62f0175fd..be777c7a3 100644 --- a/ProjFS.Mac/PrjFSLib/PrjFSLib.cpp +++ b/ProjFS.Mac/PrjFSLib/PrjFSLib.cpp @@ -26,10 +26,25 @@ #define STRINGIFY(s) #s -using std::endl; using std::cerr; -using std::unordered_map; using std::set; using std::string; +using std::cerr; +using std::cout; +using std::dec; +using std::endl; +using std::extent; +using std::hex; +using std::is_pod; +using std::lock_guard; +using std::make_pair; +using std::move; using std::mutex; -typedef std::lock_guard mutex_lock; +using std::oct; +using std::pair; +using std::queue; +using std::set; +using std::string; +using std::unordered_map; + +typedef lock_guard mutex_lock; // Structs struct _PrjFS_FileHandle @@ -74,14 +89,14 @@ static const char* NotificationTypeToString(PrjFS_NotificationType notificationT // State static io_connect_t s_kernelServiceConnection = IO_OBJECT_NULL; -static std::string s_virtualizationRootFullPath; +static string s_virtualizationRootFullPath; static PrjFS_Callbacks s_callbacks; static dispatch_queue_t s_messageQueueDispatchQueue; static dispatch_queue_t s_kernelRequestHandlingConcurrentQueue; // Map of relative path -> set of pending message IDs for that path, plus mutex to protect it. static unordered_map> s_PendingRequestMessageIDs; -static std::mutex s_PendingRequestMessageMutex; +static mutex s_PendingRequestMessageMutex; // The full API is defined in the header, but only the minimal set of functions needed @@ -96,13 +111,13 @@ PrjFS_Result PrjFS_StartVirtualizationInstance( _In_ unsigned int poolThreadCount) { #ifdef DEBUG - std::cout + cout << "PrjFS_StartVirtualizationInstance(" << virtualizationRootFullPath << ", " << callbacks.EnumerateDirectory << ", " << callbacks.GetFileStream << ", " << callbacks.NotifyOperation << ", " - << poolThreadCount << ")" << std::endl; + << poolThreadCount << ")" << endl; #endif if (nullptr == virtualizationRootFullPath || @@ -174,7 +189,7 @@ PrjFS_Result PrjFS_StartVirtualizationInstance( IOReturn result = IODataQueueDequeue(dataQueue.queueMemory, messageMemory, &dequeuedSize); if (kIOReturnSuccess != result || dequeuedSize != messageSize) { - cerr << "Unexpected result dequeueing message - result 0x" << std::hex << result << " dequeued " << dequeuedSize << "/" << messageSize << " bytes\n"; + cerr << "Unexpected result dequeueing message - result 0x" << hex << result << " dequeued " << dequeuedSize << "/" << messageSize << " bytes\n"; abort(); } @@ -191,8 +206,8 @@ PrjFS_Result PrjFS_StartVirtualizationInstance( if (file_messages_found == s_PendingRequestMessageIDs.end()) { // Not handling this file/dir yet - std::pair inserted = - s_PendingRequestMessageIDs.insert(std::make_pair(string(message.path), set{ message.messageHeader->messageId })); + pair inserted = + s_PendingRequestMessageIDs.insert(make_pair(string(message.path), set{ message.messageHeader->messageId })); assert(inserted.second); } else @@ -220,7 +235,7 @@ PrjFS_Result PrjFS_ConvertDirectoryToVirtualizationRoot( _In_ const char* virtualizationRootFullPath) { #ifdef DEBUG - std::cout << "PrjFS_ConvertDirectoryToVirtualizationRoot(" << virtualizationRootFullPath << ")" << std::endl; + cout << "PrjFS_ConvertDirectoryToVirtualizationRoot(" << virtualizationRootFullPath << ")" << endl; #endif if (nullptr == virtualizationRootFullPath) @@ -253,7 +268,7 @@ PrjFS_Result PrjFS_WritePlaceholderDirectory( _In_ const char* relativePath) { #ifdef DEBUG - std::cout << "PrjFS_WritePlaceholderDirectory(" << relativePath << ")" << std::endl; + cout << "PrjFS_WritePlaceholderDirectory(" << relativePath << ")" << endl; #endif if (nullptr == relativePath) @@ -289,13 +304,13 @@ PrjFS_Result PrjFS_WritePlaceholderFile( _In_ uint16_t fileMode) { #ifdef DEBUG - std::cout + cout << "PrjFS_WritePlaceholderFile(" << relativePath << ", " << (int)providerId[0] << ", " << (int)contentId[0] << ", " << fileSize << ", " - << std::oct << fileMode << std::dec << ")" << std::endl; + << oct << fileMode << dec << ")" << endl; #endif if (nullptr == relativePath) @@ -368,14 +383,14 @@ PrjFS_Result PrjFS_UpdatePlaceholderFileIfNeeded( _Out_ PrjFS_UpdateFailureCause* failureCause) { #ifdef DEBUG - std::cout + cout << "PrjFS_UpdatePlaceholderFileIfNeeded(" << relativePath << ", " << (int)providerId[0] << ", " << (int)contentId[0] << ", " << fileSize << ", " - << std::oct << fileMode << std::dec << ", " - << std::hex << updateFlags << std::dec << ")" << std::endl; + << oct << fileMode << dec << ", " + << hex << updateFlags << dec << ")" << endl; #endif // TODO(Mac): Check if the contentId or fileMode have changed before proceeding @@ -396,10 +411,10 @@ PrjFS_Result PrjFS_DeleteFile( _Out_ PrjFS_UpdateFailureCause* failureCause) { #ifdef DEBUG - std::cout + cout << "PrjFS_DeleteFile(" << relativePath << ", " - << std::hex << updateFlags << std::dec << ")" << std::endl; + << hex << updateFlags << dec << ")" << endl; #endif // TODO(Mac): Populate failure cause appropriately @@ -435,13 +450,13 @@ PrjFS_Result PrjFS_WriteFileContents( _In_ unsigned int byteCount) { #ifdef DEBUG - std::cout + cout << "PrjFS_WriteFile(" << fileHandle->file << ", " << (int)((char*)bytes)[0] << ", " << (int)((char*)bytes)[1] << ", " << (int)((char*)bytes)[2] << ", " - << byteCount << ")" << std::endl; + << byteCount << ")" << endl; #endif if (nullptr == fileHandle->file || @@ -550,13 +565,13 @@ static void HandleKernelRequest(Message request, void* messageMemory) ? MessageType_Response_Success : MessageType_Response_Fail; - std::set messageIDs; + set messageIDs; { mutex_lock lock(s_PendingRequestMessageMutex); unordered_map>::iterator fileMessageIDsFound = s_PendingRequestMessageIDs.find(request.path); assert(fileMessageIDsFound != s_PendingRequestMessageIDs.end()); - messageIDs = std::move(fileMessageIDsFound->second); + messageIDs = move(fileMessageIDsFound->second); s_PendingRequestMessageIDs.erase(fileMessageIDsFound); } @@ -572,7 +587,7 @@ static void HandleKernelRequest(Message request, void* messageMemory) static PrjFS_Result HandleEnumerateDirectoryRequest(const MessageHeader* request, const char* path) { #ifdef DEBUG - std::cout << "PrjFSLib.HandleEnumerateDirectoryRequest: " << path << std::endl; + cout << "PrjFSLib.HandleEnumerateDirectoryRequest: " << path << endl; #endif PrjFS_Result callbackResult = s_callbacks.EnumerateDirectory( @@ -600,12 +615,12 @@ static PrjFS_Result HandleEnumerateDirectoryRequest(const MessageHeader* request static PrjFS_Result HandleRecursivelyEnumerateDirectoryRequest(const MessageHeader* request, const char* path) { #ifdef DEBUG - std::cout << "PrjFSLib.HandleRecursivelyEnumerateDirectoryRequest: " << path << std::endl; + cout << "PrjFSLib.HandleRecursivelyEnumerateDirectoryRequest: " << path << endl; #endif DIR* directory = nullptr; PrjFS_Result result = PrjFS_Result_Success; - std::queue directoryRelativePaths; + queue directoryRelativePaths; directoryRelativePaths.push(path); // Walk each directory, expanding those that are found to be empty @@ -660,7 +675,7 @@ static PrjFS_Result HandleRecursivelyEnumerateDirectoryRequest(const MessageHead static PrjFS_Result HandleHydrateFileRequest(const MessageHeader* request, const char* path) { #ifdef DEBUG - std::cout << "PrjFSLib.HandleHydrateFileRequest: " << path << std::endl; + cout << "PrjFSLib.HandleHydrateFileRequest: " << path << endl; #endif char fullPath[PrjFSMaxPath]; @@ -734,9 +749,10 @@ static PrjFS_Result HandleFileNotification( PrjFS_NotificationType notificationType) { #ifdef DEBUG - std::cout << "PrjFSLib.HandleFileNotification: " << path - << " notificationType: " << NotificationTypeToString(notificationType) - << " isDirectory: " << isDirectory << std::endl; + cout + << "PrjFSLib.HandleFileNotification: " << path + << " notificationType: " << NotificationTypeToString(notificationType) + << " isDirectory: " << isDirectory << endl; #endif char fullPath[PrjFSMaxPath]; @@ -772,7 +788,7 @@ static bool InitializeEmptyPlaceholder(const char* fullPath, TPlaceholder* data, data->header.magicNumber = PlaceholderMagicNumber; data->header.formatVersion = PlaceholderFormatVersion; - static_assert(std::is_pod(), "TPlaceholder must be a POD struct"); + static_assert(is_pod(), "TPlaceholder must be a POD struct"); if (AddXAttr(fullPath, xattrName, data, sizeof(TPlaceholder))) { return true; @@ -898,8 +914,8 @@ static errno_t SendKernelMessageResponse(uint64_t messageId, MessageType respons IOReturn callResult = IOConnectCallScalarMethod( s_kernelServiceConnection, ProviderSelector_KernelMessageResponse, - inputs, std::extent::value, // scalar inputs - nullptr, nullptr); // no outputs + inputs, extent::value, // scalar inputs + nullptr, nullptr); // no outputs return callResult == kIOReturnSuccess ? 0 : EBADMSG; } From 7302b1f7194ede68b530ab5f669d4cdd46406b99 Mon Sep 17 00:00:00 2001 From: William Baker Date: Fri, 28 Sep 2018 13:56:31 -0700 Subject: [PATCH 173/272] Clean up std namespace usage --- ProjFS.Mac/PrjFSLib/PrjFSLib.cpp | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/ProjFS.Mac/PrjFSLib/PrjFSLib.cpp b/ProjFS.Mac/PrjFSLib/PrjFSLib.cpp index 0f95354ae..82e273fb4 100644 --- a/ProjFS.Mac/PrjFSLib/PrjFSLib.cpp +++ b/ProjFS.Mac/PrjFSLib/PrjFSLib.cpp @@ -378,10 +378,10 @@ PrjFS_Result PrjFS_WriteSymLink( _In_ const char* symLinkTarget) { #ifdef DEBUG - std::cout + cout << "PrjFS_WriteSymLink(" << relativePath << ", " - << symLinkTarget << ")" << std::endl; + << symLinkTarget << ")" << endl; #endif if (nullptr == relativePath || nullptr == symLinkTarget) @@ -446,11 +446,11 @@ PrjFS_Result PrjFS_ReplacePlaceholderFileWithSymLink( _Out_ PrjFS_UpdateFailureCause* failureCause) { #ifdef DEBUG - std::cout + cout << "PrjFS_ReplacePlaceholderFileWithSymLink(" << relativePath << ", " << symLinkTarget << ", " - << std::hex << updateFlags << std::dec << ")" << std::endl; + << hex << updateFlags << dec << ")" << endl; #endif PrjFS_Result result = PrjFS_DeleteFile(relativePath, updateFlags, failureCause); From f97a47cd7f054554ca502c5e9bd0dc2d0e9585fc Mon Sep 17 00:00:00 2001 From: John Briggs Date: Mon, 1 Oct 2018 09:30:08 -0400 Subject: [PATCH 174/272] Fix large repo and mac build badges Links were broken when a project was renamed in Azure DevOps. --- Readme.md | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/Readme.md b/Readme.md index e049af181..133b93754 100644 --- a/Readme.md +++ b/Readme.md @@ -4,14 +4,14 @@ |Branch|Unit Tests|Functional Tests|Large Repo Perf|Large Repo Build| |:--:|:--:|:--:|:--:|:--:| -|**master**|[![Build status](https://dev.azure.com/gvfs/ci/_apis/build/status/CI%20-%20Windows?branchName=master)](https://dev.azure.com/gvfs/ci/_build/latest?definitionId=7&branchName=master)|[![Build status](https://dev.azure.com/gvfs/ci/_apis/build/status/CI%20-%20Windows%20-%20Full%20Functional%20Tests?branchName=master)](https://dev.azure.com/gvfs/ci/_build/latest?definitionId=6&branchName=master)|[![Build status](https://dev.azure.com/mseng/VSOnline/_apis/build/status/GVFS/GitHub%20VFSForGit%20Large%20Repo%20Perf%20Tests?branchName=master)](https://dev.azure.com/mseng/VSOnline/_build/latest?definitionId=7179&branchName=master)|[![Build status](https://dev.azure.com/mseng/VSOnline/_apis/build/status/GVFS/GitHub%20VFSForGit%20Large%20Repo%20Build?branchName=master)](https://dev.azure.com/mseng/VSOnline/_build/latest?definitionId=7180&branchName=master)| -|**shipped**|[![Build status](https://dev.azure.com/gvfs/ci/_apis/build/status/CI%20-%20Windows?branchName=releases%2Fshipped)](https://dev.azure.com/gvfs/ci/_build/latest?definitionId=7&branchName=releases%2Fshipped)|[![Build status](https://dev.azure.com/gvfs/ci/_apis/build/status/CI%20-%20Windows%20-%20Full%20Functional%20Tests?branchName=releases%2Fshipped)](https://dev.azure.com/gvfs/ci/_build/latest?definitionId=6&branchName=releases%2Fshipped)|[![Build status](https://dev.azure.com/mseng/VSOnline/_apis/build/status/GVFS/GitHub%20VFSForGit%20Large%20Repo%20Perf%20Tests?branchName=releases%2Fshipped)](https://dev.azure.com/mseng/VSOnline/_build/latest?definitionId=7179&branchName=releases%2Fshipped)|[![Build status](https://dev.azure.com/mseng/VSOnline/_apis/build/status/GVFS/GitHub%20VFSForGit%20Large%20Repo%20Build?branchName=releases%2Fshipped)](https://dev.azure.com/mseng/VSOnline/_build/latest?definitionId=7180&branchName=releases%2Fshipped)| +|**master**|[![Build status](https://dev.azure.com/gvfs/ci/_apis/build/status/CI%20-%20Windows?branchName=master)](https://dev.azure.com/gvfs/ci/_build/latest?definitionId=7&branchName=master)|[![Build status](https://dev.azure.com/gvfs/ci/_apis/build/status/CI%20-%20Windows%20-%20Full%20Functional%20Tests?branchName=master)](https://dev.azure.com/gvfs/ci/_build/latest?definitionId=6&branchName=master)|[![Build status](https://dev.azure.com/mseng/AzureDevOps/_apis/build/status/GVFS/GitHub%20VFSForGit%20Large%20Repo%20Perf%20Tests?branchName=master)](https://dev.azure.com/mseng/AzureDevOps/_build/latest?definitionId=7179&branchName=master)|[![Build status](https://dev.azure.com/mseng/AzureDevOps/_apis/build/status/GVFS/GitHub%20VFSForGit%20Large%20Repo%20Build?branchName=master)](https://dev.azure.com/mseng/AzureDevOps/_build/latest?definitionId=7180&branchName=master)| +|**shipped**|[![Build status](https://dev.azure.com/gvfs/ci/_apis/build/status/CI%20-%20Windows?branchName=releases%2Fshipped)](https://dev.azure.com/gvfs/ci/_build/latest?definitionId=7&branchName=releases%2Fshipped)|[![Build status](https://dev.azure.com/gvfs/ci/_apis/build/status/CI%20-%20Windows%20-%20Full%20Functional%20Tests?branchName=releases%2Fshipped)](https://dev.azure.com/gvfs/ci/_build/latest?definitionId=6&branchName=releases%2Fshipped)|[![Build status](https://dev.azure.com/mseng/AzureDevOps/_apis/build/status/GVFS/GitHub%20VFSForGit%20Large%20Repo%20Perf%20Tests?branchName=releases%2Fshipped)](https://dev.azure.com/mseng/AzureDevOps/_build/latest?definitionId=7179&branchName=releases%2Fshipped)|[![Build status](https://dev.azure.com/mseng/AzureDevOps/_apis/build/status/GVFS/GitHub%20VFSForGit%20Large%20Repo%20Build?branchName=releases%2Fshipped)](https://dev.azure.com/mseng/AzureDevOps/_build/latest?definitionId=7180&branchName=releases%2Fshipped)| ## Mac |Branch|Unit Tests|Functional Tests| |:--:|:--:|:--:| -|**master**|[![Build status](https://dev.azure.com/gvfs/ci/_apis/build/status/CI%20-%20Mac?branchName=master)](https://dev.azure.com/gvfs/ci/_build/latest?definitionId=15&branchName=master)|[![Build status](https://dev.azure.com/mseng/VSOnline/_apis/build/status/GVFS/CI%20-%20Mac%20-%20Functional%20Tests?branchName=master)](https://dev.azure.com/mseng/VSOnline/_build/latest?definitionId=7376&branchName=master)| -|**shipped**|[![Build status](https://dev.azure.com/gvfs/ci/_apis/build/status/CI%20-%20Mac?branchName=releases%2Fshipped)](https://dev.azure.com/gvfs/ci/_build/latest?definitionId=15&branchName=releases%2Fshipped)|[![Build status](https://dev.azure.com/mseng/VSOnline/_apis/build/status/GVFS/CI%20-%20Mac%20-%20Functional%20Tests?branchName=releases%2Fshipped)](https://dev.azure.com/mseng/VSOnline/_build/latest?definitionId=7376&branchName=releases%2Fshipped)| +|**master**|[![Build status](https://dev.azure.com/gvfs/ci/_apis/build/status/CI%20-%20Mac?branchName=master)](https://dev.azure.com/gvfs/ci/_build/latest?definitionId=15&branchName=master)|[![Build status](https://dev.azure.com/mseng/AzureDevOps/_apis/build/status/GVFS/CI%20-%20Mac%20-%20Functional%20Tests?branchName=master)](https://dev.azure.com/mseng/AzureDevOps/_build/latest?definitionId=7376&branchName=master)| +|**shipped**|[![Build status](https://dev.azure.com/gvfs/ci/_apis/build/status/CI%20-%20Mac?branchName=releases%2Fshipped)](https://dev.azure.com/gvfs/ci/_build/latest?definitionId=15&branchName=releases%2Fshipped)|[![Build status](https://dev.azure.com/mseng/AzureDevOps/_apis/build/status/GVFS/CI%20-%20Mac%20-%20Functional%20Tests?branchName=releases%2Fshipped)](https://dev.azure.com/mseng/AzureDevOps/_build/latest?definitionId=7376&branchName=releases%2Fshipped)| ## What is VFS for Git? From c04e0ff1892a4314d3a388650e6097a7398bc490 Mon Sep 17 00:00:00 2001 From: William Baker Date: Mon, 1 Oct 2018 11:24:13 -0700 Subject: [PATCH 175/272] PR Feedback: Eliminate masking and simplify FileTypeAndMode --- .../MacFileSystemVirtualizerTests.cs | 31 ++++--- .../Projection/FileTypeAndMode.cs | 81 ++++++++----------- .../Projection/GitIndexProjection.cs | 11 ++- 3 files changed, 54 insertions(+), 69 deletions(-) diff --git a/GVFS/GVFS.UnitTests/Platform.Mac/MacFileSystemVirtualizerTests.cs b/GVFS/GVFS.UnitTests/Platform.Mac/MacFileSystemVirtualizerTests.cs index 81c52e43f..8835cbe33 100644 --- a/GVFS/GVFS.UnitTests/Platform.Mac/MacFileSystemVirtualizerTests.cs +++ b/GVFS/GVFS.UnitTests/Platform.Mac/MacFileSystemVirtualizerTests.cs @@ -22,10 +22,7 @@ namespace GVFS.UnitTests.Platform.Mac [TestFixture] public class MacFileSystemVirtualizerTests : TestsWithCommonRepo { - private static readonly ushort FileMode644 = Convert.ToUInt16("644", 8); - private static readonly ushort FileMode664 = Convert.ToUInt16("664", 8); - private static readonly ushort FileMode755 = Convert.ToUInt16("755", 8); - + private static readonly FileTypeAndMode Regular644FileTypeAndMode = new FileTypeAndMode(FileTypeAndMode.FileType.Regular, FileTypeAndMode.FileMode644); private static readonly Dictionary MappedResults = new Dictionary() { { Result.Success, FSResult.Ok }, @@ -100,7 +97,7 @@ public void UpdatePlaceholderIfNeeded() virtualizer)) { string filePath = "test" + Path.DirectorySeparatorChar + "test.txt"; - gitIndexProjection.MockFileTypesAndModes.TryAdd(filePath, FileTypeAndMode.Regular644File); + gitIndexProjection.MockFileTypesAndModes.TryAdd(filePath, Regular644FileTypeAndMode); string error; fileSystemCallbacks.TryStart(out error).ShouldEqual(true); @@ -180,7 +177,7 @@ public void WritePlaceholderForSymLink() virtualizer)) { string filePath = "test" + Path.DirectorySeparatorChar + "test.txt"; - gitIndexProjection.MockFileTypesAndModes.TryAdd(filePath, new FileTypeAndMode((ushort)FileTypeAndMode.FileType.SymLink)); + gitIndexProjection.MockFileTypesAndModes.TryAdd(filePath, new FileTypeAndMode(FileTypeAndMode.FileType.SymLink, mode: 0)); string error; fileSystemCallbacks.TryStart(out error).ShouldEqual(true); @@ -219,7 +216,7 @@ public void UpdatePlaceholderToSymLink() virtualizer)) { string filePath = "test" + Path.DirectorySeparatorChar + "test.txt"; - gitIndexProjection.MockFileTypesAndModes.TryAdd(filePath, new FileTypeAndMode((ushort)FileTypeAndMode.FileType.SymLink)); + gitIndexProjection.MockFileTypesAndModes.TryAdd(filePath, new FileTypeAndMode(FileTypeAndMode.FileType.SymLink, mode: 0)); string error; fileSystemCallbacks.TryStart(out error).ShouldEqual(true); @@ -282,7 +279,7 @@ public void OnEnumerateDirectoryReturnsSuccessWhenResultsNotInMemory() backgroundFileSystemTaskRunner: backgroundTaskRunner, fileSystemVirtualizer: virtualizer)) { - gitIndexProjection.MockFileTypesAndModes.TryAdd("test" + Path.DirectorySeparatorChar + "test.txt", FileTypeAndMode.Regular644File); + gitIndexProjection.MockFileTypesAndModes.TryAdd("test" + Path.DirectorySeparatorChar + "test.txt", Regular644FileTypeAndMode); string error; fileSystemCallbacks.TryStart(out error).ShouldEqual(true); @@ -291,7 +288,7 @@ public void OnEnumerateDirectoryReturnsSuccessWhenResultsNotInMemory() gitIndexProjection.EnumerationInMemory = false; mockVirtualization.OnEnumerateDirectory(1, "test", triggeringProcessId: 1, triggeringProcessName: "UnitTests").ShouldEqual(Result.Success); mockVirtualization.CreatedPlaceholders.ShouldContain( - kvp => kvp.Key.Equals(Path.Combine("test", "test.txt"), StringComparison.OrdinalIgnoreCase) && kvp.Value == FileMode644); + kvp => kvp.Key.Equals(Path.Combine("test", "test.txt"), StringComparison.OrdinalIgnoreCase) && kvp.Value == FileTypeAndMode.FileMode644); fileSystemCallbacks.Stop(); } } @@ -312,7 +309,7 @@ public void OnEnumerateDirectoryReturnsSuccessWhenResultsInMemory() backgroundFileSystemTaskRunner: backgroundTaskRunner, fileSystemVirtualizer: virtualizer)) { - gitIndexProjection.MockFileTypesAndModes.TryAdd("test" + Path.DirectorySeparatorChar + "test.txt", FileTypeAndMode.Regular644File); + gitIndexProjection.MockFileTypesAndModes.TryAdd("test" + Path.DirectorySeparatorChar + "test.txt", Regular644FileTypeAndMode); string error; fileSystemCallbacks.TryStart(out error).ShouldEqual(true); @@ -321,7 +318,7 @@ public void OnEnumerateDirectoryReturnsSuccessWhenResultsInMemory() gitIndexProjection.EnumerationInMemory = true; mockVirtualization.OnEnumerateDirectory(1, "test", triggeringProcessId: 1, triggeringProcessName: "UnitTests").ShouldEqual(Result.Success); mockVirtualization.CreatedPlaceholders.ShouldContain( - kvp => kvp.Key.Equals(Path.Combine("test", "test.txt"), StringComparison.OrdinalIgnoreCase) && kvp.Value == FileMode644); + kvp => kvp.Key.Equals(Path.Combine("test", "test.txt"), StringComparison.OrdinalIgnoreCase) && kvp.Value == FileTypeAndMode.FileMode644); gitIndexProjection.ExpandedFolders.ShouldMatchInOrder("test"); fileSystemCallbacks.Stop(); } @@ -343,9 +340,9 @@ public void OnEnumerateDirectorySetsFileModes() backgroundFileSystemTaskRunner: backgroundTaskRunner, fileSystemVirtualizer: virtualizer)) { - gitIndexProjection.MockFileTypesAndModes.TryAdd("test" + Path.DirectorySeparatorChar + "test644.txt", FileTypeAndMode.Regular644File); - gitIndexProjection.MockFileTypesAndModes.TryAdd("test" + Path.DirectorySeparatorChar + "test664.txt", new FileTypeAndMode((ushort)((ushort)FileTypeAndMode.FileType.Regular | FileMode664))); - gitIndexProjection.MockFileTypesAndModes.TryAdd("test" + Path.DirectorySeparatorChar + "test755.txt", new FileTypeAndMode((ushort)((ushort)FileTypeAndMode.FileType.Regular | FileMode755))); + gitIndexProjection.MockFileTypesAndModes.TryAdd("test" + Path.DirectorySeparatorChar + "test644.txt", Regular644FileTypeAndMode); + gitIndexProjection.MockFileTypesAndModes.TryAdd("test" + Path.DirectorySeparatorChar + "test664.txt", new FileTypeAndMode(FileTypeAndMode.FileType.Regular, FileTypeAndMode.FileMode664)); + gitIndexProjection.MockFileTypesAndModes.TryAdd("test" + Path.DirectorySeparatorChar + "test755.txt", new FileTypeAndMode(FileTypeAndMode.FileType.Regular, FileTypeAndMode.FileMode755)); string error; fileSystemCallbacks.TryStart(out error).ShouldEqual(true); @@ -354,11 +351,11 @@ public void OnEnumerateDirectorySetsFileModes() gitIndexProjection.EnumerationInMemory = true; mockVirtualization.OnEnumerateDirectory(1, "test", triggeringProcessId: 1, triggeringProcessName: "UnitTests").ShouldEqual(Result.Success); mockVirtualization.CreatedPlaceholders.ShouldContain( - kvp => kvp.Key.Equals(Path.Combine("test", "test644.txt"), StringComparison.OrdinalIgnoreCase) && kvp.Value == FileMode644); + kvp => kvp.Key.Equals(Path.Combine("test", "test644.txt"), StringComparison.OrdinalIgnoreCase) && kvp.Value == FileTypeAndMode.FileMode644); mockVirtualization.CreatedPlaceholders.ShouldContain( - kvp => kvp.Key.Equals(Path.Combine("test", "test664.txt"), StringComparison.OrdinalIgnoreCase) && kvp.Value == FileMode664); + kvp => kvp.Key.Equals(Path.Combine("test", "test664.txt"), StringComparison.OrdinalIgnoreCase) && kvp.Value == FileTypeAndMode.FileMode664); mockVirtualization.CreatedPlaceholders.ShouldContain( - kvp => kvp.Key.Equals(Path.Combine("test", "test755.txt"), StringComparison.OrdinalIgnoreCase) && kvp.Value == FileMode755); + kvp => kvp.Key.Equals(Path.Combine("test", "test755.txt"), StringComparison.OrdinalIgnoreCase) && kvp.Value == FileTypeAndMode.FileMode755); fileSystemCallbacks.Stop(); } } diff --git a/GVFS/GVFS.Virtualization/Projection/FileTypeAndMode.cs b/GVFS/GVFS.Virtualization/Projection/FileTypeAndMode.cs index 0afab8cc2..eaea06fd7 100644 --- a/GVFS/GVFS.Virtualization/Projection/FileTypeAndMode.cs +++ b/GVFS/GVFS.Virtualization/Projection/FileTypeAndMode.cs @@ -3,73 +3,58 @@ namespace GVFS.Virtualization.Projection { public struct FileTypeAndMode { - public static readonly ushort FileMode755; - public static readonly ushort FileMode664; - public static readonly ushort FileMode644; - public static readonly FileTypeAndMode Regular644File; + public static readonly ushort FileMode755 = Convert.ToUInt16("755", 8); + public static readonly ushort FileMode664 = Convert.ToUInt16("664", 8); + public static readonly ushort FileMode644 = Convert.ToUInt16("644", 8); // Bitmasks for extracting file type and mode from the ushort stored in the index private const ushort FileTypeMask = 0xF000; private const ushort FileModeMask = 0x1FF; - private readonly ushort fileTypeAndMode; - - static FileTypeAndMode() - { - FileMode755 = Convert.ToUInt16("755", 8); - FileMode664 = Convert.ToUInt16("664", 8); - FileMode644 = Convert.ToUInt16("644", 8); - - Regular644File = new FileTypeAndMode((ushort)((ushort)FileType.Regular | FileMode644)); - } + // Values used in the index file to indicate the type of the file + private const ushort RegularFileIndexEntry = 0x8000; + private const ushort SymLinkFileIndexEntry = 0xA000; + private const ushort GitLinkFileIndexEntry = 0xE000; public FileTypeAndMode(ushort typeAndModeInIndexFormat) { - this.fileTypeAndMode = typeAndModeInIndexFormat; - } - - public enum FileType : ushort - { - Invalid = 0, - - Regular = 0x8000, - SymLink = 0xA000, - GitLink = 0xE000, - } - - public FileType Type => (FileType)(this.fileTypeAndMode & FileTypeMask); - public ushort Mode => (ushort)(this.fileTypeAndMode & FileModeMask); - - public static bool operator ==(FileTypeAndMode lhs, FileTypeAndMode rhs) - { - return lhs.Equals(rhs); - } - - public static bool operator !=(FileTypeAndMode lhs, FileTypeAndMode rhs) - { - return !lhs.Equals(rhs); - } - - public override bool Equals(object obj) - { - if (obj is FileTypeAndMode) + switch (typeAndModeInIndexFormat & FileTypeMask) { - return this.Equals((FileTypeAndMode)obj); + case RegularFileIndexEntry: + this.Type = FileType.Regular; + break; + case SymLinkFileIndexEntry: + this.Type = FileType.SymLink; + break; + case GitLinkFileIndexEntry: + this.Type = FileType.GitLink; + break; + default: + this.Type = FileType.Invalid; + break; } - return false; + this.Mode = (ushort)(typeAndModeInIndexFormat & FileModeMask); } - public override int GetHashCode() + public FileTypeAndMode(FileType type, ushort mode) { - return this.fileTypeAndMode; + this.Type = type; + this.Mode = mode; } - public bool Equals(FileTypeAndMode otherFileTypeAndMode) + public enum FileType : short { - return this.fileTypeAndMode == otherFileTypeAndMode.fileTypeAndMode; + Invalid, + + Regular, + SymLink, + GitLink, } + public FileType Type { get; } + public ushort Mode { get; } + public string GetModeAsOctalString() { return Convert.ToString(this.Mode, 8); diff --git a/GVFS/GVFS.Virtualization/Projection/GitIndexProjection.cs b/GVFS/GVFS.Virtualization/Projection/GitIndexProjection.cs index deaa26d1e..211c2ce5b 100644 --- a/GVFS/GVFS.Virtualization/Projection/GitIndexProjection.cs +++ b/GVFS/GVFS.Virtualization/Projection/GitIndexProjection.cs @@ -37,6 +37,8 @@ public partial class GitIndexProjection : IDisposable, IProfilerOnlyIndexProject private const int ExternalLockReleaseTimeoutMs = 50; + private static readonly FileTypeAndMode Regular644FileTypeAndMode = new FileTypeAndMode(FileTypeAndMode.FileType.Regular, FileTypeAndMode.FileMode644); + private char[] gitPathSeparatorCharArray = new char[] { GVFSConstants.GitPathSeparator }; private GVFSContext context; @@ -371,7 +373,7 @@ public virtual FileTypeAndMode GetFileTypeAndMode(string filePath) return fileTypeAndMode; } - return FileTypeAndMode.Regular644File; + return Regular644FileTypeAndMode; } finally { @@ -663,9 +665,10 @@ private void AddItemFromIndexEntry(GitIndexEntry indexEntry) if (GVFSPlatform.Instance.FileSystem.SupportsFileMode) { - // TODO(Mac): Test if performance could be improved by reduce this to a single check - // (e.g. by defaulting FileMode to 644 and eliminating the SupportsFileMode check) - if (indexEntry.TypeAndMode != FileTypeAndMode.Regular644File) + // TODO(Mac): Test if performance could be improved by eliminating the SupportsFileMode check + // (e.g. by defaulting FileMode to Regular 644 and eliminating the SupportsFileMode check) + if (indexEntry.TypeAndMode.Type != FileTypeAndMode.FileType.Regular || + indexEntry.TypeAndMode.Mode != FileTypeAndMode.FileMode644) { // TODO(Mac): The line below causes a conversion from LazyUTF8String to .NET string. // Measure the perf and memory overhead of performing this conversion, and determine if we need From fd1cfc6d1552a6e2588639ba2903c45e79757a11 Mon Sep 17 00:00:00 2001 From: Derrick Stolee Date: Fri, 28 Sep 2018 10:09:46 -0400 Subject: [PATCH 176/272] CloneVerb: Force download of commit and trees on failed initial checkout --- GVFS/GVFS/CommandLine/CloneVerb.cs | 24 ++++++++++++++++++++++++ GVFS/GVFS/CommandLine/GVFSVerb.cs | 27 ++++++++++++++------------- 2 files changed, 38 insertions(+), 13 deletions(-) diff --git a/GVFS/GVFS/CommandLine/CloneVerb.cs b/GVFS/GVFS/CommandLine/CloneVerb.cs index 5d86b909f..9078e688c 100644 --- a/GVFS/GVFS/CommandLine/CloneVerb.cs +++ b/GVFS/GVFS/CommandLine/CloneVerb.cs @@ -567,6 +567,30 @@ private Result CreateClone( } GitProcess.Result forceCheckoutResult = git.ForceCheckout(branch); + if (forceCheckoutResult.HasErrors && forceCheckoutResult.Errors.IndexOf("unable to read tree") > 0) + { + // It is possible to have the above TryDownloadCommit() fail because we + // already have the commit and root tree we intend to check out, but + // don't have a tree further down the working directory. If we fail + // checkout here, it may be due to not having these trees and the + // read-object hook is not available yet. Force downloading the commit + // again and retry the checkout. + + if (!this.TryDownloadCommit( + refs.GetTipCommitId(branch), + enlistment, + objectRequestor, + gitObjects, + gitRepo, + out errorMessage, + ignoreIfRootExists: false)) + { + return new Result(errorMessage); + } + + forceCheckoutResult = git.ForceCheckout(branch); + } + if (forceCheckoutResult.HasErrors) { string[] errorLines = forceCheckoutResult.Errors.Split('\n'); diff --git a/GVFS/GVFS/CommandLine/GVFSVerb.cs b/GVFS/GVFS/CommandLine/GVFSVerb.cs index 325e2dcee..38ee3e1e6 100644 --- a/GVFS/GVFS/CommandLine/GVFSVerb.cs +++ b/GVFS/GVFS/CommandLine/GVFSVerb.cs @@ -176,15 +176,15 @@ protected bool ShowStatusWhileRunning( } protected bool ShowStatusWhileRunning( - Func action, - string message, + Func action, + string message, bool suppressGvfsLogMessage = false) { string gvfsLogEnlistmentRoot = null; if (!suppressGvfsLogMessage) { string errorMessage; - GVFSPlatform.Instance.TryGetGVFSEnlistmentRoot(this.EnlistmentRootPathParameter, out gvfsLogEnlistmentRoot, out errorMessage); + GVFSPlatform.Instance.TryGetGVFSEnlistmentRoot(this.EnlistmentRootPathParameter, out gvfsLogEnlistmentRoot, out errorMessage); } return this.ShowStatusWhileRunning(action, message, gvfsLogEnlistmentRoot); @@ -261,7 +261,7 @@ protected GVFSConfig QueryGVFSConfig(ITracer tracer, GVFSEnlistment enlistment, } return gvfsConfig; - } + } protected void ValidateClientVersions(ITracer tracer, GVFSEnlistment enlistment, GVFSConfig gvfsConfig, bool showWarnings) { @@ -401,9 +401,10 @@ protected bool TryDownloadCommit( GitObjectsHttpRequestor objectRequestor, GVFSGitObjects gitObjects, GitRepo repo, - out string error) + out string error, + bool ignoreIfRootExists = true) { - if (!repo.CommitAndRootTreeExists(commitId)) + if (!ignoreIfRootExists || !repo.CommitAndRootTreeExists(commitId)) { if (!gitObjects.TryDownloadCommit(commitId)) { @@ -740,7 +741,7 @@ public ForExistingEnlistment(bool validateOrigin = true) : base(validateOrigin) MetaName = "Enlistment Root Path", HelpText = "Full or relative path to the GVFS enlistment root")] public override string EnlistmentRootPathParameter { get; set; } - + public sealed override void Execute() { this.ValidatePathParameter(this.EnlistmentRootPathParameter); @@ -855,7 +856,7 @@ private void EnsureLocalCacheIsHealthy( if (File.Exists(alternatesFilePath)) { try - { + { using (Stream stream = fileSystem.OpenFileStream( alternatesFilePath, FileMode.Open, @@ -922,7 +923,7 @@ private void EnsureLocalCacheIsHealthy( gvfsConfig = this.QueryGVFSConfig(tracer, enlistment, retryConfig); } - + string localCacheKey; LocalCacheResolver localCacheResolver = new LocalCacheResolver(enlistment); if (!localCacheResolver.TryGetLocalCacheKeyFromLocalConfigOrRemoteCacheServers( @@ -1009,7 +1010,7 @@ private GVFSEnlistment CreateEnlistment(string enlistmentRootPath) if (GVFSPlatform.Instance.IsUnderConstruction) { hooksPath = "hooksUnderConstruction"; - } + } else { hooksPath = ProcessHelper.WhereDirectory(GVFSPlatform.Instance.Constants.GVFSHooksExecutableName); @@ -1017,8 +1018,8 @@ private GVFSEnlistment CreateEnlistment(string enlistmentRootPath) { this.ReportErrorAndExit("Could not find " + GVFSPlatform.Instance.Constants.GVFSHooksExecutableName); } - } - + } + GVFSEnlistment enlistment = null; try { @@ -1030,7 +1031,7 @@ private GVFSEnlistment CreateEnlistment(string enlistmentRootPath) { enlistment = GVFSEnlistment.CreateWithoutRepoUrlFromDirectory(enlistmentRootPath, gitBinPath, hooksPath); } - + if (enlistment == null) { this.ReportErrorAndExit( From 6f79da131a0ff87c681a8d790b5304268bfa15b0 Mon Sep 17 00:00:00 2001 From: William Baker Date: Mon, 1 Oct 2018 13:10:03 -0700 Subject: [PATCH 177/272] Enhance SymbolicLinkTests and fix bug in ModifiedPathsShouldContain --- .../EnlistmentPerFixture/SymbolicLinkTests.cs | 47 ++++++++++++++++++- .../GVFS.FunctionalTests/Tools/GVFSHelpers.cs | 11 +++-- 2 files changed, 53 insertions(+), 5 deletions(-) diff --git a/GVFS/GVFS.FunctionalTests/Tests/EnlistmentPerFixture/SymbolicLinkTests.cs b/GVFS/GVFS.FunctionalTests/Tests/EnlistmentPerFixture/SymbolicLinkTests.cs index 04179c671..3b32e037b 100644 --- a/GVFS/GVFS.FunctionalTests/Tests/EnlistmentPerFixture/SymbolicLinkTests.cs +++ b/GVFS/GVFS.FunctionalTests/Tests/EnlistmentPerFixture/SymbolicLinkTests.cs @@ -29,6 +29,12 @@ public class SymbolicLinkTests : TestsWithEnlistmentPerFixture private const string GrandChildFileContents = "This is the third file"; private const string GrandChildLinkNowAFileContents = "This was a link but is now a file"; + // FunctionalTests/20180925_SymLinksPart3 files + private const string ChildFolder2Name = "ChildDir2"; + + // FunctionalTests/20180925_SymLinksPart4 files + // Note: In this branch ChildLinkName has been changed to a directory and ChildFolder2Name has been changed to a link to ChildFolderName + private BashRunner bashRunner; public SymbolicLinkTests() { @@ -48,18 +54,22 @@ public void CheckoutBranchWithSymLinks() string testFilePath = this.Enlistment.GetVirtualPathTo(Path.Combine(TestFolderName, TestFileName)); testFilePath.ShouldBeAFile(this.bashRunner).WithContents(TestFileContents); this.bashRunner.IsSymbolicLink(testFilePath).ShouldBeFalse($"{testFilePath} should not be a symlink"); + GVFSHelpers.ModifiedPathsShouldNotContain(this.bashRunner, this.Enlistment.DotGVFSRoot, TestFolderName + "/" + TestFileName); string testFile2Path = this.Enlistment.GetVirtualPathTo(Path.Combine(TestFolderName, TestFile2Name)); testFile2Path.ShouldBeAFile(this.bashRunner).WithContents(TestFile2Contents); this.bashRunner.IsSymbolicLink(testFile2Path).ShouldBeFalse($"{testFile2Path} should not be a symlink"); + GVFSHelpers.ModifiedPathsShouldNotContain(this.bashRunner, this.Enlistment.DotGVFSRoot, TestFolderName + "/" + TestFile2Name); string childLinkPath = this.Enlistment.GetVirtualPathTo(Path.Combine(TestFolderName, ChildLinkName)); this.bashRunner.IsSymbolicLink(childLinkPath).ShouldBeTrue($"{childLinkPath} should be a symlink"); childLinkPath.ShouldBeAFile(this.bashRunner).WithContents(TestFileContents); + GVFSHelpers.ModifiedPathsShouldContain(this.bashRunner, this.Enlistment.DotGVFSRoot, TestFolderName + "/" + ChildLinkName); string grandChildLinkPath = this.Enlistment.GetVirtualPathTo(Path.Combine(TestFolderName, ChildFolderName, GrandChildLinkName)); this.bashRunner.IsSymbolicLink(grandChildLinkPath).ShouldBeTrue($"{grandChildLinkPath} should be a symlink"); grandChildLinkPath.ShouldBeAFile(this.bashRunner).WithContents(TestFile2Contents); + GVFSHelpers.ModifiedPathsShouldContain(this.bashRunner, this.Enlistment.DotGVFSRoot, TestFolderName + "/" + ChildFolderName + "/" + GrandChildLinkName); } [TestCase, Order(2)] @@ -76,15 +86,18 @@ public void CheckoutBranchWhereSymLinksChangeContentsAndTransitionToFile() string testFilePath = this.Enlistment.GetVirtualPathTo(Path.Combine(TestFolderName, TestFileName)); testFilePath.ShouldBeAFile(this.bashRunner).WithContents(TestFileContents); this.bashRunner.IsSymbolicLink(testFilePath).ShouldBeFalse($"{testFilePath} should not be a symlink"); + GVFSHelpers.ModifiedPathsShouldNotContain(this.bashRunner, this.Enlistment.DotGVFSRoot, TestFolderName + "/" + TestFileName); string testFile2Path = this.Enlistment.GetVirtualPathTo(Path.Combine(TestFolderName, TestFile2Name)); testFile2Path.ShouldBeAFile(this.bashRunner).WithContents(TestFile2Contents); this.bashRunner.IsSymbolicLink(testFile2Path).ShouldBeFalse($"{testFile2Path} should not be a symlink"); + GVFSHelpers.ModifiedPathsShouldNotContain(this.bashRunner, this.Enlistment.DotGVFSRoot, TestFolderName + "/" + TestFile2Name); // In this branch childLinkPath has been changed to point to testFile2Path string childLinkPath = this.Enlistment.GetVirtualPathTo(Path.Combine(TestFolderName, ChildLinkName)); this.bashRunner.IsSymbolicLink(childLinkPath).ShouldBeTrue($"{childLinkPath} should be a symlink"); childLinkPath.ShouldBeAFile(this.bashRunner).WithContents(TestFile2Contents); + GVFSHelpers.ModifiedPathsShouldContain(this.bashRunner, this.Enlistment.DotGVFSRoot, TestFolderName + "/" + ChildLinkName); // grandChildLinkPath should now be a file string grandChildLinkPath = this.Enlistment.GetVirtualPathTo(Path.Combine(TestFolderName, ChildFolderName, GrandChildLinkName)); @@ -95,6 +108,7 @@ public void CheckoutBranchWhereSymLinksChangeContentsAndTransitionToFile() string newGrandChildFilePath = this.Enlistment.GetVirtualPathTo(Path.Combine(TestFolderName, ChildFolderName, GrandChildFileName)); newGrandChildFilePath.ShouldBeAFile(this.bashRunner).WithContents(GrandChildFileContents); this.bashRunner.IsSymbolicLink(newGrandChildFilePath).ShouldBeFalse($"{newGrandChildFilePath} should not be a symlink"); + GVFSHelpers.ModifiedPathsShouldNotContain(this.bashRunner, this.Enlistment.DotGVFSRoot, TestFolderName + "/" + ChildFolderName + "/" + GrandChildFileName); } [TestCase, Order(3)] @@ -111,6 +125,13 @@ public void CheckoutBranchWhereFilesTransitionToSymLinks() string testFilePath = this.Enlistment.GetVirtualPathTo(Path.Combine(TestFolderName, TestFileName)); testFilePath.ShouldBeAFile(this.bashRunner).WithContents(GrandChildFileContents); this.bashRunner.IsSymbolicLink(testFilePath).ShouldBeTrue($"{testFilePath} should be a symlink"); + GVFSHelpers.ModifiedPathsShouldContain(this.bashRunner, this.Enlistment.DotGVFSRoot, TestFolderName + "/" + TestFileName); + + // There should be a new ChildFolder2Name directory + string childFolder2Path = this.Enlistment.GetVirtualPathTo(Path.Combine(TestFolderName, ChildFolder2Name)); + this.bashRunner.IsSymbolicLink(childFolder2Path).ShouldBeFalse($"{childFolder2Path} should not be a symlink"); + childFolder2Path.ShouldBeADirectory(this.bashRunner); + GVFSHelpers.ModifiedPathsShouldNotContain(this.bashRunner, this.Enlistment.DotGVFSRoot, TestFolderName + "/" + ChildFolder2Name); // The rest of the files are unchanged from FunctionalTests/20180925_SymLinksPart2 string testFile2Path = this.Enlistment.GetVirtualPathTo(Path.Combine(TestFolderName, TestFile2Name)); @@ -120,6 +141,7 @@ public void CheckoutBranchWhereFilesTransitionToSymLinks() string childLinkPath = this.Enlistment.GetVirtualPathTo(Path.Combine(TestFolderName, ChildLinkName)); this.bashRunner.IsSymbolicLink(childLinkPath).ShouldBeTrue($"{childLinkPath} should be a symlink"); childLinkPath.ShouldBeAFile(this.bashRunner).WithContents(TestFile2Contents); + GVFSHelpers.ModifiedPathsShouldContain(this.bashRunner, this.Enlistment.DotGVFSRoot, TestFolderName + "/" + ChildLinkName); string grandChildLinkPath = this.Enlistment.GetVirtualPathTo(Path.Combine(TestFolderName, ChildFolderName, GrandChildLinkName)); this.bashRunner.IsSymbolicLink(grandChildLinkPath).ShouldBeFalse($"{grandChildLinkPath} should not be a symlink"); @@ -131,12 +153,33 @@ public void CheckoutBranchWhereFilesTransitionToSymLinks() } [TestCase, Order(4)] + public void CheckoutBranchWhereSymLinkTransistionsToFolderAndFolderTransitionsToSymlink() + { + GitHelpers.InvokeGitAgainstGVFSRepo(this.Enlistment.RepoRoot, "checkout FunctionalTests/20180925_SymLinksPart4"); + GitHelpers.CheckGitCommandAgainstGVFSRepo( + this.Enlistment.RepoRoot, + "status", + "On branch FunctionalTests/20180925_SymLinksPart4", + "nothing to commit, working tree clean"); + + // In this branch ChildLinkName has been changed to a directory and ChildFolder2Name has been changed to a link to ChildFolderName + string linkNowADirectoryPath = this.Enlistment.GetVirtualPathTo(Path.Combine(TestFolderName, ChildLinkName)); + this.bashRunner.IsSymbolicLink(linkNowADirectoryPath).ShouldBeFalse($"{linkNowADirectoryPath} should not be a symlink"); + linkNowADirectoryPath.ShouldBeADirectory(this.bashRunner); + GVFSHelpers.ModifiedPathsShouldContain(this.bashRunner, this.Enlistment.DotGVFSRoot, TestFolderName + "/" + ChildLinkName); + + string directoryNowALinkPath = this.Enlistment.GetVirtualPathTo(Path.Combine(TestFolderName, ChildFolder2Name)); + this.bashRunner.IsSymbolicLink(directoryNowALinkPath).ShouldBeTrue($"{directoryNowALinkPath} should be a symlink"); + GVFSHelpers.ModifiedPathsShouldContain(this.bashRunner, this.Enlistment.DotGVFSRoot, TestFolderName + "/" + ChildFolder2Name); + } + + [TestCase, Order(5)] public void GitStatusReportsSymLinkChanges() { GitHelpers.CheckGitCommandAgainstGVFSRepo( this.Enlistment.RepoRoot, "status", - "On branch FunctionalTests/20180925_SymLinksPart3", + "On branch FunctionalTests/20180925_SymLinksPart4", "nothing to commit, working tree clean"); string testFilePath = this.Enlistment.GetVirtualPathTo(Path.Combine(TestFolderName, TestFileName)); @@ -156,7 +199,7 @@ public void GitStatusReportsSymLinkChanges() GitHelpers.CheckGitCommandAgainstGVFSRepo( this.Enlistment.RepoRoot, "status", - "On branch FunctionalTests/20180925_SymLinksPart3", + "On branch FunctionalTests/20180925_SymLinksPart4", $"modified: {TestFolderName}/{TestFileName}"); } } diff --git a/GVFS/GVFS.FunctionalTests/Tools/GVFSHelpers.cs b/GVFS/GVFS.FunctionalTests/Tools/GVFSHelpers.cs index 95e988c18..2d41a0db0 100644 --- a/GVFS/GVFS.FunctionalTests/Tools/GVFSHelpers.cs +++ b/GVFS/GVFS.FunctionalTests/Tools/GVFSHelpers.cs @@ -7,7 +7,6 @@ using System.Collections.Generic; using System.IO; using System.Linq; -using System.Reflection; namespace GVFS.FunctionalTests.Tools { @@ -19,6 +18,8 @@ public static class GVFSHelpers public static readonly string PlaceholderListFile = Path.Combine("databases", "PlaceholderList.dat"); public static readonly string RepoMetadataName = Path.Combine("databases", "RepoMetadata.dat"); + private const string ModifedPathsLinePrefix = "A "; + private const string DiskLayoutMajorVersionKey = "DiskLayoutVersion"; private const string DiskLayoutMinorVersionKey = "DiskLayoutMinorVersion"; private const string LocalCacheRootKey = "LocalCacheRoot"; @@ -111,8 +112,12 @@ public static void ModifiedPathsShouldContain(FileSystemRunner fileSystem, strin { string modifiedPathsDatabase = Path.Combine(dotGVFSRoot, TestConstants.Databases.ModifiedPaths); modifiedPathsDatabase.ShouldBeAFile(fileSystem); - GVFSHelpers.ReadAllTextFromWriteLockedFile(modifiedPathsDatabase).ShouldContain( - gitPaths.Select(path => path + ModifiedPathsNewLine).ToArray()); + string modifedPathsContents = GVFSHelpers.ReadAllTextFromWriteLockedFile(modifiedPathsDatabase); + string[] modifedPathLines = modifedPathsContents.Split(ModifiedPathsNewLine); + foreach (string gitPath in gitPaths) + { + modifedPathLines.ShouldContain(path => path.Equals(ModifedPathsLinePrefix + gitPath)); + } } public static void ModifiedPathsShouldNotContain(FileSystemRunner fileSystem, string dotGVFSRoot, params string[] gitPaths) From 413f29ee37d033ba2ffcf550705644f89561f8cf Mon Sep 17 00:00:00 2001 From: William Baker Date: Mon, 1 Oct 2018 13:20:41 -0700 Subject: [PATCH 178/272] Fix .NET Framework compilation issue --- GVFS/GVFS.FunctionalTests/Tools/GVFSHelpers.cs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/GVFS/GVFS.FunctionalTests/Tools/GVFSHelpers.cs b/GVFS/GVFS.FunctionalTests/Tools/GVFSHelpers.cs index 2d41a0db0..ba7dc5ad5 100644 --- a/GVFS/GVFS.FunctionalTests/Tools/GVFSHelpers.cs +++ b/GVFS/GVFS.FunctionalTests/Tools/GVFSHelpers.cs @@ -4,9 +4,9 @@ using Microsoft.Data.Sqlite; using Newtonsoft.Json; using NUnit.Framework; +using System; using System.Collections.Generic; using System.IO; -using System.Linq; namespace GVFS.FunctionalTests.Tools { @@ -113,7 +113,7 @@ public static void ModifiedPathsShouldContain(FileSystemRunner fileSystem, strin string modifiedPathsDatabase = Path.Combine(dotGVFSRoot, TestConstants.Databases.ModifiedPaths); modifiedPathsDatabase.ShouldBeAFile(fileSystem); string modifedPathsContents = GVFSHelpers.ReadAllTextFromWriteLockedFile(modifiedPathsDatabase); - string[] modifedPathLines = modifedPathsContents.Split(ModifiedPathsNewLine); + string[] modifedPathLines = modifedPathsContents.Split(new[] { ModifiedPathsNewLine }, StringSplitOptions.None); foreach (string gitPath in gitPaths) { modifedPathLines.ShouldContain(path => path.Equals(ModifedPathsLinePrefix + gitPath)); From 8eddf8b19522f60722a0d0ce67778f687fb8d6cb Mon Sep 17 00:00:00 2001 From: Kevin Willford Date: Mon, 1 Oct 2018 10:34:35 -0600 Subject: [PATCH 179/272] Cleanup ContainsParentDirectory and add checks in tests for unexpected paths --- GVFS/GVFS.Common/ModifiedPathsDatabase.cs | 12 ++++---- .../EnlistmentPerFixture/GitFilesTests.cs | 29 +++++++++++-------- 2 files changed, 22 insertions(+), 19 deletions(-) diff --git a/GVFS/GVFS.Common/ModifiedPathsDatabase.cs b/GVFS/GVFS.Common/ModifiedPathsDatabase.cs index 338c1f4ce..5a7e34c43 100644 --- a/GVFS/GVFS.Common/ModifiedPathsDatabase.cs +++ b/GVFS/GVFS.Common/ModifiedPathsDatabase.cs @@ -1,8 +1,6 @@ using System; using System.Collections.Generic; using System.IO; -using System.Linq; -using System.Text; using GVFS.Common.FileSystem; using GVFS.Common.Tracing; @@ -187,12 +185,12 @@ private bool TryParseRemoveLine(string line, out string key, out string error) private bool ContainsParentDirectory(string modifiedPath) { - string[] pathParts = modifiedPath.Split(new char[] { '/' }, StringSplitOptions.RemoveEmptyEntries); - StringBuilder parentFolder = new StringBuilder(); - foreach (string pathPart in pathParts.Take(pathParts.Length - 1)) + string[] pathParts = modifiedPath.Split(new char[] { GVFSConstants.GitPathSeparator }, StringSplitOptions.RemoveEmptyEntries); + string parentFolder = string.Empty; + for (int i = 0; i < pathParts.Length - 1; i++) { - parentFolder.Append(pathPart + "/"); - if (this.modifiedPaths.Contains(parentFolder.ToString())) + parentFolder += pathParts[i] + GVFSConstants.GitPathSeparatorString; + if (this.modifiedPaths.Contains(parentFolder)) { return true; } diff --git a/GVFS/GVFS.FunctionalTests/Tests/EnlistmentPerFixture/GitFilesTests.cs b/GVFS/GVFS.FunctionalTests/Tests/EnlistmentPerFixture/GitFilesTests.cs index 8a0a0c3f6..5352f7fc8 100644 --- a/GVFS/GVFS.FunctionalTests/Tests/EnlistmentPerFixture/GitFilesTests.cs +++ b/GVFS/GVFS.FunctionalTests/Tests/EnlistmentPerFixture/GitFilesTests.cs @@ -77,6 +77,7 @@ public void CreateFileInFolderTest() this.Enlistment.WaitForBackgroundOperations().ShouldEqual(true, "Background operations failed to complete."); GVFSHelpers.ModifiedPathsShouldContain(this.fileSystem, this.Enlistment.DotGVFSRoot, folderName + "/"); + GVFSHelpers.ModifiedPathsShouldNotContain(this.fileSystem, this.Enlistment.DotGVFSRoot, folderName + "/" + fileName); } [TestCase, Order(4)] @@ -97,6 +98,7 @@ public void RenameEmptyFolderTest() this.Enlistment.WaitForBackgroundOperations().ShouldEqual(true, "Background operations failed to complete."); GVFSHelpers.ModifiedPathsShouldContain(this.fileSystem, this.Enlistment.DotGVFSRoot, expectedModifiedEntries); + GVFSHelpers.ModifiedPathsShouldNotContain(this.fileSystem, this.Enlistment.DotGVFSRoot, folderName + "/"); } [TestCase, Order(5)] @@ -111,6 +113,17 @@ public void RenameFolderTest() renamedFolderName + "/", }; + string[] unexpectedModifiedEntries = + { + renamedFolderName + "/" + fileNames[0], + renamedFolderName + "/" + fileNames[1], + renamedFolderName + "/" + fileNames[2], + folderName + "/", + folderName + "/" + fileNames[0], + folderName + "/" + fileNames[1], + folderName + "/" + fileNames[2], + }; + this.Enlistment.GetVirtualPathTo(folderName).ShouldNotExistOnDisk(this.fileSystem); this.fileSystem.CreateDirectory(this.Enlistment.GetVirtualPathTo(folderName)); foreach (string fileName in fileNames) @@ -125,6 +138,7 @@ public void RenameFolderTest() this.Enlistment.WaitForBackgroundOperations().ShouldEqual(true, "Background operations failed to complete."); GVFSHelpers.ModifiedPathsShouldContain(this.fileSystem, this.Enlistment.DotGVFSRoot, expectedModifiedEntries); + GVFSHelpers.ModifiedPathsShouldNotContain(this.fileSystem, this.Enlistment.DotGVFSRoot, unexpectedModifiedEntries); } [TestCase, Order(6)] @@ -136,26 +150,17 @@ public void CaseOnlyRenameOfNewFolderKeepsModifiedPathsEntries() Assert.Ignore("Powershell does not support case only renames."); } - string[] expectedModifiedPathsEntriesAfterCreate = - { - "A Folder/", - }; - - string[] expectedModifiedPathsEntriesAfterRename = - { - "A folder/", - }; - this.fileSystem.CreateDirectory(Path.Combine(this.Enlistment.RepoRoot, "Folder")); this.fileSystem.CreateEmptyFile(Path.Combine(this.Enlistment.RepoRoot, "Folder", "testfile")); this.Enlistment.WaitForBackgroundOperations().ShouldEqual(true, "Background operations failed to complete."); - GVFSHelpers.ModifiedPathsShouldContain(this.fileSystem, this.Enlistment.DotGVFSRoot, expectedModifiedPathsEntriesAfterCreate); + GVFSHelpers.ModifiedPathsShouldContain(this.fileSystem, this.Enlistment.DotGVFSRoot, "A Folder/"); this.fileSystem.RenameDirectory(this.Enlistment.RepoRoot, "Folder", "folder"); this.Enlistment.WaitForBackgroundOperations().ShouldEqual(true, "Background operations failed to complete."); - GVFSHelpers.ModifiedPathsShouldContain(this.fileSystem, this.Enlistment.DotGVFSRoot, expectedModifiedPathsEntriesAfterRename); + GVFSHelpers.ModifiedPathsShouldContain(this.fileSystem, this.Enlistment.DotGVFSRoot, "A folder/"); + GVFSHelpers.ModifiedPathsShouldNotContain(this.fileSystem, this.Enlistment.DotGVFSRoot, "A folder/testfile"); } [TestCase, Order(7)] From 24cf13c29d77d1bb64e9d1ffcb43ddcb05ccddf4 Mon Sep 17 00:00:00 2001 From: William Baker Date: Mon, 1 Oct 2018 14:41:05 -0700 Subject: [PATCH 180/272] Fix functional test failures due to change in ModifiedPathsShouldContain behavior --- .../Tests/EnlistmentPerFixture/GitFilesTests.cs | 6 +++--- .../Tests/GitCommands/GitCommandsTests.cs | 6 ++++-- 2 files changed, 7 insertions(+), 5 deletions(-) diff --git a/GVFS/GVFS.FunctionalTests/Tests/EnlistmentPerFixture/GitFilesTests.cs b/GVFS/GVFS.FunctionalTests/Tests/EnlistmentPerFixture/GitFilesTests.cs index bfbdcc457..1d444c857 100644 --- a/GVFS/GVFS.FunctionalTests/Tests/EnlistmentPerFixture/GitFilesTests.cs +++ b/GVFS/GVFS.FunctionalTests/Tests/EnlistmentPerFixture/GitFilesTests.cs @@ -139,13 +139,13 @@ public void CaseOnlyRenameOfNewFolderKeepsModifiedPathsEntries() string[] expectedModifiedPathsEntriesAfterCreate = { - "A Folder/", - "A Folder/testfile", + "Folder/", + "Folder/testfile", }; string[] expectedModifiedPathsEntriesAfterRename = { - "A folder/", + "folder/", }; this.fileSystem.CreateDirectory(Path.Combine(this.Enlistment.RepoRoot, "Folder")); diff --git a/GVFS/GVFS.FunctionalTests/Tests/GitCommands/GitCommandsTests.cs b/GVFS/GVFS.FunctionalTests/Tests/GitCommands/GitCommandsTests.cs index c623f5ea7..31ac5cb08 100644 --- a/GVFS/GVFS.FunctionalTests/Tests/GitCommands/GitCommandsTests.cs +++ b/GVFS/GVFS.FunctionalTests/Tests/GitCommands/GitCommandsTests.cs @@ -980,15 +980,17 @@ public void EditFileNeedingUtf8Encoding() { this.ValidateGitCommand("checkout -b tests/functional/EditFileNeedingUtf8Encoding"); this.ValidateGitCommand("status"); + string virtualFile = Path.Combine(this.Enlistment.RepoRoot, EncodingFileFolder, EncodingFilename); string controlFile = Path.Combine(this.ControlGitRepo.RootPath, EncodingFileFolder, EncodingFilename); + string relativeGitPath = EncodingFileFolder + "/" + EncodingFilename; string contents = virtualFile.ShouldBeAFile(this.FileSystem).WithContents(); string expectedContents = controlFile.ShouldBeAFile(this.FileSystem).WithContents(); contents.ShouldEqual(expectedContents); // Confirm that the entry is not in the the modified paths database - GVFSHelpers.ModifiedPathsShouldNotContain(this.FileSystem, this.Enlistment.DotGVFSRoot, EncodingFilename); + GVFSHelpers.ModifiedPathsShouldNotContain(this.FileSystem, this.Enlistment.DotGVFSRoot, relativeGitPath); this.ValidateGitCommand("status"); this.AppendAllText(ContentWhenEditingFile, virtualFile); @@ -997,7 +999,7 @@ public void EditFileNeedingUtf8Encoding() this.ValidateGitCommand("status"); // Confirm that the entry was added to the modified paths database - GVFSHelpers.ModifiedPathsShouldContain(this.FileSystem, this.Enlistment.DotGVFSRoot, EncodingFilename); + GVFSHelpers.ModifiedPathsShouldContain(this.FileSystem, this.Enlistment.DotGVFSRoot, relativeGitPath); } [TestCase] From 169deabab412bf8935a81def9d9e6ea8b81d65d9 Mon Sep 17 00:00:00 2001 From: William Baker Date: Mon, 1 Oct 2018 15:19:11 -0700 Subject: [PATCH 181/272] Clean up MacFileSystemVirtualizerTests --- .../Mock/Mac/MockVirtualizationInstance.cs | 7 ++ .../MacFileSystemVirtualizerTests.cs | 114 +++++++++++------- 2 files changed, 78 insertions(+), 43 deletions(-) diff --git a/GVFS/GVFS.UnitTests/Mock/Mac/MockVirtualizationInstance.cs b/GVFS/GVFS.UnitTests/Mock/Mac/MockVirtualizationInstance.cs index 156ae8382..532bbcbbf 100644 --- a/GVFS/GVFS.UnitTests/Mock/Mac/MockVirtualizationInstance.cs +++ b/GVFS/GVFS.UnitTests/Mock/Mac/MockVirtualizationInstance.cs @@ -15,6 +15,7 @@ public MockVirtualizationInstance() { this.commandCompleted = new AutoResetEvent(false); this.CreatedPlaceholders = new ConcurrentDictionary(); + this.UpdatedPlaceholders = new ConcurrentDictionary(); this.CreatedSymLinks = new ConcurrentHashSet(); this.WriteFileReturnResult = Result.Success; } @@ -28,6 +29,7 @@ public MockVirtualizationInstance() public UpdateFailureCause DeleteFileUpdateFailureCause { get; set; } public ConcurrentDictionary CreatedPlaceholders { get; private set; } + public ConcurrentDictionary UpdatedPlaceholders { get; private set; } public ConcurrentHashSet CreatedSymLinks { get; } @@ -100,6 +102,11 @@ public override Result UpdatePlaceholderIfNeeded( out UpdateFailureCause failureCause) { failureCause = this.UpdatePlaceholderIfNeededFailureCause; + if (failureCause == UpdateFailureCause.NoFailure) + { + this.UpdatedPlaceholders[relativePath] = fileMode; + } + return this.UpdatePlaceholderIfNeededResult; } diff --git a/GVFS/GVFS.UnitTests/Platform.Mac/MacFileSystemVirtualizerTests.cs b/GVFS/GVFS.UnitTests/Platform.Mac/MacFileSystemVirtualizerTests.cs index 8835cbe33..353a82e26 100644 --- a/GVFS/GVFS.UnitTests/Platform.Mac/MacFileSystemVirtualizerTests.cs +++ b/GVFS/GVFS.UnitTests/Platform.Mac/MacFileSystemVirtualizerTests.cs @@ -52,19 +52,20 @@ public void DeleteFile() using (MockVirtualizationInstance mockVirtualization = new MockVirtualizationInstance()) using (MacFileSystemVirtualizer virtualizer = new MacFileSystemVirtualizer(this.Repo.Context, this.Repo.GitObjects, mockVirtualization)) { + const string DeleteTestFileName = "deleteMe.txt"; UpdateFailureReason failureReason = UpdateFailureReason.NoFailure; mockVirtualization.DeleteFileResult = Result.Success; mockVirtualization.DeleteFileUpdateFailureCause = UpdateFailureCause.NoFailure; virtualizer - .DeleteFile("test.txt", UpdatePlaceholderType.AllowReadOnly, out failureReason) + .DeleteFile(DeleteTestFileName, UpdatePlaceholderType.AllowReadOnly, out failureReason) .ShouldEqual(new FileSystemResult(FSResult.Ok, (int)mockVirtualization.DeleteFileResult)); failureReason.ShouldEqual((UpdateFailureReason)mockVirtualization.DeleteFileUpdateFailureCause); mockVirtualization.DeleteFileResult = Result.EFileNotFound; mockVirtualization.DeleteFileUpdateFailureCause = UpdateFailureCause.NoFailure; virtualizer - .DeleteFile("test.txt", UpdatePlaceholderType.AllowReadOnly, out failureReason) + .DeleteFile(DeleteTestFileName, UpdatePlaceholderType.AllowReadOnly, out failureReason) .ShouldEqual(new FileSystemResult(FSResult.FileOrPathNotFound, (int)mockVirtualization.DeleteFileResult)); failureReason.ShouldEqual((UpdateFailureReason)mockVirtualization.DeleteFileUpdateFailureCause); @@ -74,7 +75,7 @@ public void DeleteFile() // TODO: The result should probably be VirtualizationInvalidOperation but for now it's IOError mockVirtualization.DeleteFileUpdateFailureCause = UpdateFailureCause.DirtyData; virtualizer - .DeleteFile("test.txt", UpdatePlaceholderType.AllowReadOnly, out failureReason) + .DeleteFile(DeleteTestFileName, UpdatePlaceholderType.AllowReadOnly, out failureReason) .ShouldEqual(new FileSystemResult(FSResult.IOError, (int)mockVirtualization.DeleteFileResult)); failureReason.ShouldEqual((UpdateFailureReason)mockVirtualization.DeleteFileUpdateFailureCause); } @@ -83,9 +84,10 @@ public void DeleteFile() [TestCase] public void UpdatePlaceholderIfNeeded() { + const string UpdatePlaceholderFileName = "testUpdatePlaceholder.txt"; using (MockBackgroundFileSystemTaskRunner backgroundTaskRunner = new MockBackgroundFileSystemTaskRunner()) using (MockVirtualizationInstance mockVirtualization = new MockVirtualizationInstance()) - using (MockGitIndexProjection gitIndexProjection = new MockGitIndexProjection(new[] { "test.txt" })) + using (MockGitIndexProjection gitIndexProjection = new MockGitIndexProjection(new[] { UpdatePlaceholderFileName })) using (MacFileSystemVirtualizer virtualizer = new MacFileSystemVirtualizer(this.Repo.Context, this.Repo.GitObjects, mockVirtualization)) using (FileSystemCallbacks fileSystemCallbacks = new FileSystemCallbacks( this.Repo.Context, @@ -96,8 +98,7 @@ public void UpdatePlaceholderIfNeeded() backgroundTaskRunner, virtualizer)) { - string filePath = "test" + Path.DirectorySeparatorChar + "test.txt"; - gitIndexProjection.MockFileTypesAndModes.TryAdd(filePath, Regular644FileTypeAndMode); + gitIndexProjection.MockFileTypesAndModes.TryAdd(UpdatePlaceholderFileName, Regular644FileTypeAndMode); string error; fileSystemCallbacks.TryStart(out error).ShouldEqual(true); @@ -107,7 +108,7 @@ public void UpdatePlaceholderIfNeeded() mockVirtualization.UpdatePlaceholderIfNeededFailureCause = UpdateFailureCause.NoFailure; virtualizer .UpdatePlaceholderIfNeeded( - filePath, + UpdatePlaceholderFileName, DateTime.Now, DateTime.Now, DateTime.Now, @@ -119,12 +120,14 @@ public void UpdatePlaceholderIfNeeded() out failureReason) .ShouldEqual(new FileSystemResult(FSResult.Ok, (int)mockVirtualization.UpdatePlaceholderIfNeededResult)); failureReason.ShouldEqual((UpdateFailureReason)mockVirtualization.UpdatePlaceholderIfNeededFailureCause); + mockVirtualization.UpdatedPlaceholders.ShouldContain(path => path.Key.Equals(UpdatePlaceholderFileName) && path.Value == FileTypeAndMode.FileMode644); + mockVirtualization.UpdatedPlaceholders.Clear(); mockVirtualization.UpdatePlaceholderIfNeededResult = Result.EFileNotFound; mockVirtualization.UpdatePlaceholderIfNeededFailureCause = UpdateFailureCause.NoFailure; virtualizer .UpdatePlaceholderIfNeeded( - filePath, + UpdatePlaceholderFileName, DateTime.Now, DateTime.Now, DateTime.Now, @@ -144,7 +147,7 @@ public void UpdatePlaceholderIfNeeded() // TODO: The result should probably be VirtualizationInvalidOperation but for now it's IOError virtualizer .UpdatePlaceholderIfNeeded( - filePath, + UpdatePlaceholderFileName, DateTime.Now, DateTime.Now, DateTime.Now, @@ -163,9 +166,10 @@ public void UpdatePlaceholderIfNeeded() [TestCase] public void WritePlaceholderForSymLink() { + const string WriteSymLinkFileName = "testWriteSymLink.txt"; using (MockBackgroundFileSystemTaskRunner backgroundTaskRunner = new MockBackgroundFileSystemTaskRunner()) using (MockVirtualizationInstance mockVirtualization = new MockVirtualizationInstance()) - using (MockGitIndexProjection gitIndexProjection = new MockGitIndexProjection(new[] { "test.txt" })) + using (MockGitIndexProjection gitIndexProjection = new MockGitIndexProjection(new[] { WriteSymLinkFileName })) using (MacFileSystemVirtualizer virtualizer = new MacFileSystemVirtualizer(this.Repo.Context, this.Repo.GitObjects, mockVirtualization)) using (FileSystemCallbacks fileSystemCallbacks = new FileSystemCallbacks( this.Repo.Context, @@ -176,24 +180,24 @@ public void WritePlaceholderForSymLink() backgroundTaskRunner, virtualizer)) { - string filePath = "test" + Path.DirectorySeparatorChar + "test.txt"; - gitIndexProjection.MockFileTypesAndModes.TryAdd(filePath, new FileTypeAndMode(FileTypeAndMode.FileType.SymLink, mode: 0)); + gitIndexProjection.MockFileTypesAndModes.TryAdd(WriteSymLinkFileName, new FileTypeAndMode(FileTypeAndMode.FileType.SymLink, mode: 0)); + string error; fileSystemCallbacks.TryStart(out error).ShouldEqual(true); virtualizer.WritePlaceholderFile( - filePath, + WriteSymLinkFileName, endOfFile: 0, sha: string.Empty).ShouldEqual(new FileSystemResult(FSResult.Ok, (int)Result.Success)); mockVirtualization.CreatedPlaceholders.ShouldBeEmpty(); mockVirtualization.CreatedSymLinks.Count.ShouldEqual(1); - mockVirtualization.CreatedSymLinks.ShouldContain(entry => entry.Equals(filePath)); + mockVirtualization.CreatedSymLinks.ShouldContain(entry => entry.Equals(WriteSymLinkFileName)); // Creating a symlink should schedule a background task backgroundTaskRunner.Count.ShouldEqual(1); backgroundTaskRunner.BackgroundTasks[0].Operation.ShouldEqual(GVFS.Virtualization.Background.FileSystemTask.OperationType.OnFileSymLinkCreated); - backgroundTaskRunner.BackgroundTasks[0].VirtualPath.ShouldEqual(filePath); + backgroundTaskRunner.BackgroundTasks[0].VirtualPath.ShouldEqual(WriteSymLinkFileName); fileSystemCallbacks.Stop(); } @@ -202,9 +206,10 @@ public void WritePlaceholderForSymLink() [TestCase] public void UpdatePlaceholderToSymLink() { + const string PlaceholderToLinkFileName = "testUpdatePlaceholderToLink.txt"; using (MockBackgroundFileSystemTaskRunner backgroundTaskRunner = new MockBackgroundFileSystemTaskRunner()) using (MockVirtualizationInstance mockVirtualization = new MockVirtualizationInstance()) - using (MockGitIndexProjection gitIndexProjection = new MockGitIndexProjection(new[] { "test.txt" })) + using (MockGitIndexProjection gitIndexProjection = new MockGitIndexProjection(new[] { PlaceholderToLinkFileName })) using (MacFileSystemVirtualizer virtualizer = new MacFileSystemVirtualizer(this.Repo.Context, this.Repo.GitObjects, mockVirtualization)) using (FileSystemCallbacks fileSystemCallbacks = new FileSystemCallbacks( this.Repo.Context, @@ -215,8 +220,7 @@ public void UpdatePlaceholderToSymLink() backgroundTaskRunner, virtualizer)) { - string filePath = "test" + Path.DirectorySeparatorChar + "test.txt"; - gitIndexProjection.MockFileTypesAndModes.TryAdd(filePath, new FileTypeAndMode(FileTypeAndMode.FileType.SymLink, mode: 0)); + gitIndexProjection.MockFileTypesAndModes.TryAdd(PlaceholderToLinkFileName, new FileTypeAndMode(FileTypeAndMode.FileType.SymLink, mode: 0)); string error; fileSystemCallbacks.TryStart(out error).ShouldEqual(true); @@ -226,7 +230,7 @@ public void UpdatePlaceholderToSymLink() mockVirtualization.UpdatePlaceholderIfNeededFailureCause = UpdateFailureCause.NoFailure; virtualizer .UpdatePlaceholderIfNeeded( - filePath, + PlaceholderToLinkFileName, DateTime.Now, DateTime.Now, DateTime.Now, @@ -238,14 +242,14 @@ public void UpdatePlaceholderToSymLink() out failureReason) .ShouldEqual(new FileSystemResult(FSResult.Ok, (int)mockVirtualization.UpdatePlaceholderIfNeededResult)); failureReason.ShouldEqual((UpdateFailureReason)mockVirtualization.UpdatePlaceholderIfNeededFailureCause); - + mockVirtualization.UpdatedPlaceholders.Count.ShouldEqual(0, "UpdatePlaceholderIfNeeded should not be called when converting a placeholder to a link"); mockVirtualization.CreatedSymLinks.Count.ShouldEqual(1); - mockVirtualization.CreatedSymLinks.ShouldContain(entry => entry.Equals(filePath)); + mockVirtualization.CreatedSymLinks.ShouldContain(entry => entry.Equals(PlaceholderToLinkFileName)); // Creating a symlink should schedule a background task backgroundTaskRunner.Count.ShouldEqual(1); backgroundTaskRunner.BackgroundTasks[0].Operation.ShouldEqual(GVFS.Virtualization.Background.FileSystemTask.OperationType.OnFileSymLinkCreated); - backgroundTaskRunner.BackgroundTasks[0].VirtualPath.ShouldEqual(filePath); + backgroundTaskRunner.BackgroundTasks[0].VirtualPath.ShouldEqual(PlaceholderToLinkFileName); fileSystemCallbacks.Stop(); } @@ -266,9 +270,15 @@ public void ClearNegativePathCacheIsNoOp() [TestCase] public void OnEnumerateDirectoryReturnsSuccessWhenResultsNotInMemory() { + const string TestFileName = "test.txt"; + const string TestFolderName = "testFolder"; + string testFilePath = Path.Combine(TestFolderName, TestFileName); using (MockBackgroundFileSystemTaskRunner backgroundTaskRunner = new MockBackgroundFileSystemTaskRunner()) using (MockVirtualizationInstance mockVirtualization = new MockVirtualizationInstance()) - using (MockGitIndexProjection gitIndexProjection = new MockGitIndexProjection(new[] { "test.txt" })) + + // Don't include TestFolderName as MockGitIndexProjection returns the same list of files regardless of what folder name + // it is passed + using (MockGitIndexProjection gitIndexProjection = new MockGitIndexProjection(new[] { TestFileName })) using (MacFileSystemVirtualizer virtualizer = new MacFileSystemVirtualizer(this.Repo.Context, this.Repo.GitObjects, mockVirtualization)) using (FileSystemCallbacks fileSystemCallbacks = new FileSystemCallbacks( this.Repo.Context, @@ -279,16 +289,16 @@ public void OnEnumerateDirectoryReturnsSuccessWhenResultsNotInMemory() backgroundFileSystemTaskRunner: backgroundTaskRunner, fileSystemVirtualizer: virtualizer)) { - gitIndexProjection.MockFileTypesAndModes.TryAdd("test" + Path.DirectorySeparatorChar + "test.txt", Regular644FileTypeAndMode); + gitIndexProjection.MockFileTypesAndModes.TryAdd(testFilePath, Regular644FileTypeAndMode); string error; fileSystemCallbacks.TryStart(out error).ShouldEqual(true); Guid enumerationGuid = Guid.NewGuid(); gitIndexProjection.EnumerationInMemory = false; - mockVirtualization.OnEnumerateDirectory(1, "test", triggeringProcessId: 1, triggeringProcessName: "UnitTests").ShouldEqual(Result.Success); + mockVirtualization.OnEnumerateDirectory(1, TestFolderName, triggeringProcessId: 1, triggeringProcessName: "UnitTests").ShouldEqual(Result.Success); mockVirtualization.CreatedPlaceholders.ShouldContain( - kvp => kvp.Key.Equals(Path.Combine("test", "test.txt"), StringComparison.OrdinalIgnoreCase) && kvp.Value == FileTypeAndMode.FileMode644); + kvp => kvp.Key.Equals(testFilePath, StringComparison.OrdinalIgnoreCase) && kvp.Value == FileTypeAndMode.FileMode644); fileSystemCallbacks.Stop(); } } @@ -296,9 +306,15 @@ public void OnEnumerateDirectoryReturnsSuccessWhenResultsNotInMemory() [TestCase] public void OnEnumerateDirectoryReturnsSuccessWhenResultsInMemory() { + const string TestFileName = "test.txt"; + const string TestFolderName = "testFolder"; + string testFilePath = Path.Combine(TestFolderName, TestFileName); using (MockBackgroundFileSystemTaskRunner backgroundTaskRunner = new MockBackgroundFileSystemTaskRunner()) using (MockVirtualizationInstance mockVirtualization = new MockVirtualizationInstance()) - using (MockGitIndexProjection gitIndexProjection = new MockGitIndexProjection(new[] { "test.txt" })) + + // Don't include TestFolderName as MockGitIndexProjection returns the same list of files regardless of what folder name + // it is passed + using (MockGitIndexProjection gitIndexProjection = new MockGitIndexProjection(new[] { TestFileName })) using (MacFileSystemVirtualizer virtualizer = new MacFileSystemVirtualizer(this.Repo.Context, this.Repo.GitObjects, mockVirtualization)) using (FileSystemCallbacks fileSystemCallbacks = new FileSystemCallbacks( this.Repo.Context, @@ -309,17 +325,17 @@ public void OnEnumerateDirectoryReturnsSuccessWhenResultsInMemory() backgroundFileSystemTaskRunner: backgroundTaskRunner, fileSystemVirtualizer: virtualizer)) { - gitIndexProjection.MockFileTypesAndModes.TryAdd("test" + Path.DirectorySeparatorChar + "test.txt", Regular644FileTypeAndMode); + gitIndexProjection.MockFileTypesAndModes.TryAdd(testFilePath, Regular644FileTypeAndMode); string error; fileSystemCallbacks.TryStart(out error).ShouldEqual(true); Guid enumerationGuid = Guid.NewGuid(); gitIndexProjection.EnumerationInMemory = true; - mockVirtualization.OnEnumerateDirectory(1, "test", triggeringProcessId: 1, triggeringProcessName: "UnitTests").ShouldEqual(Result.Success); + mockVirtualization.OnEnumerateDirectory(1, TestFolderName, triggeringProcessId: 1, triggeringProcessName: "UnitTests").ShouldEqual(Result.Success); mockVirtualization.CreatedPlaceholders.ShouldContain( - kvp => kvp.Key.Equals(Path.Combine("test", "test.txt"), StringComparison.OrdinalIgnoreCase) && kvp.Value == FileTypeAndMode.FileMode644); - gitIndexProjection.ExpandedFolders.ShouldMatchInOrder("test"); + kvp => kvp.Key.Equals(testFilePath, StringComparison.OrdinalIgnoreCase) && kvp.Value == FileTypeAndMode.FileMode644); + gitIndexProjection.ExpandedFolders.ShouldMatchInOrder(TestFolderName); fileSystemCallbacks.Stop(); } } @@ -327,9 +343,19 @@ public void OnEnumerateDirectoryReturnsSuccessWhenResultsInMemory() [TestCase] public void OnEnumerateDirectorySetsFileModes() { + const string TestFile644Name = "test644.txt"; + const string TestFile664Name = "test664.txt"; + const string TestFile755Name = "test755.txt"; + const string TestFolderName = "testFolder"; + string TestFile644Path = Path.Combine(TestFolderName, TestFile644Name); + string TestFile664Path = Path.Combine(TestFolderName, TestFile664Name); + string TestFile755Path = Path.Combine(TestFolderName, TestFile755Name); using (MockBackgroundFileSystemTaskRunner backgroundTaskRunner = new MockBackgroundFileSystemTaskRunner()) using (MockVirtualizationInstance mockVirtualization = new MockVirtualizationInstance()) - using (MockGitIndexProjection gitIndexProjection = new MockGitIndexProjection(new[] { "test644.txt", "test664.txt", "test755.txt" })) + + // Don't include TestFolderName as MockGitIndexProjection returns the same list of files regardless of what folder name + // it is passed + using (MockGitIndexProjection gitIndexProjection = new MockGitIndexProjection(new[] { TestFile644Name, TestFile664Name, TestFile755Name })) using (MacFileSystemVirtualizer virtualizer = new MacFileSystemVirtualizer(this.Repo.Context, this.Repo.GitObjects, mockVirtualization)) using (FileSystemCallbacks fileSystemCallbacks = new FileSystemCallbacks( this.Repo.Context, @@ -340,22 +366,22 @@ public void OnEnumerateDirectorySetsFileModes() backgroundFileSystemTaskRunner: backgroundTaskRunner, fileSystemVirtualizer: virtualizer)) { - gitIndexProjection.MockFileTypesAndModes.TryAdd("test" + Path.DirectorySeparatorChar + "test644.txt", Regular644FileTypeAndMode); - gitIndexProjection.MockFileTypesAndModes.TryAdd("test" + Path.DirectorySeparatorChar + "test664.txt", new FileTypeAndMode(FileTypeAndMode.FileType.Regular, FileTypeAndMode.FileMode664)); - gitIndexProjection.MockFileTypesAndModes.TryAdd("test" + Path.DirectorySeparatorChar + "test755.txt", new FileTypeAndMode(FileTypeAndMode.FileType.Regular, FileTypeAndMode.FileMode755)); + gitIndexProjection.MockFileTypesAndModes.TryAdd(TestFile644Path, Regular644FileTypeAndMode); + gitIndexProjection.MockFileTypesAndModes.TryAdd(TestFile664Path, new FileTypeAndMode(FileTypeAndMode.FileType.Regular, FileTypeAndMode.FileMode664)); + gitIndexProjection.MockFileTypesAndModes.TryAdd(TestFile755Path, new FileTypeAndMode(FileTypeAndMode.FileType.Regular, FileTypeAndMode.FileMode755)); string error; fileSystemCallbacks.TryStart(out error).ShouldEqual(true); Guid enumerationGuid = Guid.NewGuid(); gitIndexProjection.EnumerationInMemory = true; - mockVirtualization.OnEnumerateDirectory(1, "test", triggeringProcessId: 1, triggeringProcessName: "UnitTests").ShouldEqual(Result.Success); + mockVirtualization.OnEnumerateDirectory(1, TestFolderName, triggeringProcessId: 1, triggeringProcessName: "UnitTests").ShouldEqual(Result.Success); mockVirtualization.CreatedPlaceholders.ShouldContain( - kvp => kvp.Key.Equals(Path.Combine("test", "test644.txt"), StringComparison.OrdinalIgnoreCase) && kvp.Value == FileTypeAndMode.FileMode644); + kvp => kvp.Key.Equals(TestFile644Path, StringComparison.OrdinalIgnoreCase) && kvp.Value == FileTypeAndMode.FileMode644); mockVirtualization.CreatedPlaceholders.ShouldContain( - kvp => kvp.Key.Equals(Path.Combine("test", "test664.txt"), StringComparison.OrdinalIgnoreCase) && kvp.Value == FileTypeAndMode.FileMode664); + kvp => kvp.Key.Equals(TestFile664Path, StringComparison.OrdinalIgnoreCase) && kvp.Value == FileTypeAndMode.FileMode664); mockVirtualization.CreatedPlaceholders.ShouldContain( - kvp => kvp.Key.Equals(Path.Combine("test", "test755.txt"), StringComparison.OrdinalIgnoreCase) && kvp.Value == FileTypeAndMode.FileMode755); + kvp => kvp.Key.Equals(TestFile755Path, StringComparison.OrdinalIgnoreCase) && kvp.Value == FileTypeAndMode.FileMode755); fileSystemCallbacks.Stop(); } } @@ -363,9 +389,10 @@ public void OnEnumerateDirectorySetsFileModes() [TestCase] public void OnGetFileStreamReturnsSuccessWhenFileStreamAvailable() { + const string TestFileName = "test.txt"; using (MockBackgroundFileSystemTaskRunner backgroundTaskRunner = new MockBackgroundFileSystemTaskRunner()) using (MockVirtualizationInstance mockVirtualization = new MockVirtualizationInstance()) - using (MockGitIndexProjection gitIndexProjection = new MockGitIndexProjection(new[] { "test.txt" })) + using (MockGitIndexProjection gitIndexProjection = new MockGitIndexProjection(new[] { TestFileName })) using (MacFileSystemVirtualizer virtualizer = new MacFileSystemVirtualizer(this.Repo.Context, this.Repo.GitObjects, mockVirtualization)) using (FileSystemCallbacks fileSystemCallbacks = new FileSystemCallbacks( this.Repo.Context, @@ -389,7 +416,7 @@ public void OnGetFileStreamReturnsSuccessWhenFileStreamAvailable() mockVirtualization.OnGetFileStream( commandId: 1, - relativePath: "test.txt", + relativePath: TestFileName, providerId: placeholderVersion, contentId: contentId, triggeringProcessId: 2, @@ -406,9 +433,10 @@ public void OnGetFileStreamReturnsSuccessWhenFileStreamAvailable() [Category(CategoryConstants.ExceptionExpected)] public void OnGetFileStreamReturnsErrorWhenWriteFileContentsFails() { + const string TestFileName = "test.txt"; using (MockBackgroundFileSystemTaskRunner backgroundTaskRunner = new MockBackgroundFileSystemTaskRunner()) using (MockVirtualizationInstance mockVirtualization = new MockVirtualizationInstance()) - using (MockGitIndexProjection gitIndexProjection = new MockGitIndexProjection(new[] { "test.txt" })) + using (MockGitIndexProjection gitIndexProjection = new MockGitIndexProjection(new[] { TestFileName })) using (MacFileSystemVirtualizer virtualizer = new MacFileSystemVirtualizer(this.Repo.Context, this.Repo.GitObjects, mockVirtualization)) using (FileSystemCallbacks fileSystemCallbacks = new FileSystemCallbacks( this.Repo.Context, @@ -432,7 +460,7 @@ public void OnGetFileStreamReturnsErrorWhenWriteFileContentsFails() mockVirtualization.OnGetFileStream( commandId: 1, - relativePath: "test.txt", + relativePath: TestFileName, providerId: placeholderVersion, contentId: contentId, triggeringProcessId: 2, From 558b7e2c0aa300b31c0898a1a6f65ddc9cded279 Mon Sep 17 00:00:00 2001 From: Kevin Willford Date: Mon, 1 Oct 2018 15:58:16 -0600 Subject: [PATCH 182/272] Make checks on the modified paths be more strict --- .../EnlistmentPerFixture/GitFilesTests.cs | 6 ++--- .../ModifiedPathsTests.cs | 12 +++------ .../Tests/GitCommands/GitCommandsTests.cs | 6 +++-- .../GVFS.FunctionalTests/Tools/GVFSHelpers.cs | 25 ++++++++++++++++--- 4 files changed, 31 insertions(+), 18 deletions(-) diff --git a/GVFS/GVFS.FunctionalTests/Tests/EnlistmentPerFixture/GitFilesTests.cs b/GVFS/GVFS.FunctionalTests/Tests/EnlistmentPerFixture/GitFilesTests.cs index 5352f7fc8..3313fb8f6 100644 --- a/GVFS/GVFS.FunctionalTests/Tests/EnlistmentPerFixture/GitFilesTests.cs +++ b/GVFS/GVFS.FunctionalTests/Tests/EnlistmentPerFixture/GitFilesTests.cs @@ -154,13 +154,13 @@ public void CaseOnlyRenameOfNewFolderKeepsModifiedPathsEntries() this.fileSystem.CreateEmptyFile(Path.Combine(this.Enlistment.RepoRoot, "Folder", "testfile")); this.Enlistment.WaitForBackgroundOperations().ShouldEqual(true, "Background operations failed to complete."); - GVFSHelpers.ModifiedPathsShouldContain(this.fileSystem, this.Enlistment.DotGVFSRoot, "A Folder/"); + GVFSHelpers.ModifiedPathsShouldContain(this.fileSystem, this.Enlistment.DotGVFSRoot, "Folder/"); this.fileSystem.RenameDirectory(this.Enlistment.RepoRoot, "Folder", "folder"); this.Enlistment.WaitForBackgroundOperations().ShouldEqual(true, "Background operations failed to complete."); - GVFSHelpers.ModifiedPathsShouldContain(this.fileSystem, this.Enlistment.DotGVFSRoot, "A folder/"); - GVFSHelpers.ModifiedPathsShouldNotContain(this.fileSystem, this.Enlistment.DotGVFSRoot, "A folder/testfile"); + GVFSHelpers.ModifiedPathsShouldContain(this.fileSystem, this.Enlistment.DotGVFSRoot, "folder/"); + GVFSHelpers.ModifiedPathsShouldNotContain(this.fileSystem, this.Enlistment.DotGVFSRoot, "folder/testfile"); } [TestCase, Order(7)] diff --git a/GVFS/GVFS.FunctionalTests/Tests/EnlistmentPerTestCase/ModifiedPathsTests.cs b/GVFS/GVFS.FunctionalTests/Tests/EnlistmentPerTestCase/ModifiedPathsTests.cs index ee4c1e060..f8e42331e 100644 --- a/GVFS/GVFS.FunctionalTests/Tests/EnlistmentPerTestCase/ModifiedPathsTests.cs +++ b/GVFS/GVFS.FunctionalTests/Tests/EnlistmentPerTestCase/ModifiedPathsTests.cs @@ -53,7 +53,7 @@ public void DeletedTempFileIsRemovedFromModifiedFiles(FileSystemRunner fileSyste tempFile.ShouldNotExistOnDisk(fileSystem); this.Enlistment.UnmountGVFS(); - this.ValidateModifiedPathsDoNotContain(fileSystem, "temp.txt"); + GVFSHelpers.ModifiedPathsShouldNotContain(fileSystem, this.Enlistment.DotGVFSRoot, "temp.txt"); } [TestCaseSource(typeof(FileSystemRunner), FileSystemRunner.TestRunners)] @@ -64,7 +64,7 @@ public void DeletedTempFolderIsRemovedFromModifiedFiles(FileSystemRunner fileSys tempFolder.ShouldNotExistOnDisk(fileSystem); this.Enlistment.UnmountGVFS(); - this.ValidateModifiedPathsDoNotContain(fileSystem, "Temp/"); + GVFSHelpers.ModifiedPathsShouldNotContain(fileSystem, this.Enlistment.DotGVFSRoot, "Temp/"); } [TestCaseSource(typeof(FileSystemRunner), FileSystemRunner.TestRunners)] @@ -79,7 +79,7 @@ public void DeletedTempFolderDeletesFilesFromModifiedFiles(FileSystemRunner file tempFile2.ShouldNotExistOnDisk(fileSystem); this.Enlistment.UnmountGVFS(); - this.ValidateModifiedPathsDoNotContain(fileSystem, "Temp/", "Temp/temp1.txt", "Temp/temp2.txt"); + GVFSHelpers.ModifiedPathsShouldNotContain(fileSystem, this.Enlistment.DotGVFSRoot, "Temp/", "Temp/temp1.txt", "Temp/temp2.txt"); } [Category(Categories.MacTODO.M2)] @@ -219,11 +219,5 @@ private string CreateFile(FileSystemRunner fileSystem, string relativePath) tempFile.ShouldBeAFile(fileSystem); return tempFile; } - - private void ValidateModifiedPathsDoNotContain(FileSystemRunner fileSystem, params string[] paths) - { - GVFSHelpers.ModifiedPathsShouldNotContain(fileSystem, this.Enlistment.DotGVFSRoot, paths.Select(x => $"A {x}" + Environment.NewLine).ToArray()); - GVFSHelpers.ModifiedPathsShouldNotContain(fileSystem, this.Enlistment.DotGVFSRoot, paths.Select(x => $"D {x}" + Environment.NewLine).ToArray()); - } } } diff --git a/GVFS/GVFS.FunctionalTests/Tests/GitCommands/GitCommandsTests.cs b/GVFS/GVFS.FunctionalTests/Tests/GitCommands/GitCommandsTests.cs index c623f5ea7..31ac5cb08 100644 --- a/GVFS/GVFS.FunctionalTests/Tests/GitCommands/GitCommandsTests.cs +++ b/GVFS/GVFS.FunctionalTests/Tests/GitCommands/GitCommandsTests.cs @@ -980,15 +980,17 @@ public void EditFileNeedingUtf8Encoding() { this.ValidateGitCommand("checkout -b tests/functional/EditFileNeedingUtf8Encoding"); this.ValidateGitCommand("status"); + string virtualFile = Path.Combine(this.Enlistment.RepoRoot, EncodingFileFolder, EncodingFilename); string controlFile = Path.Combine(this.ControlGitRepo.RootPath, EncodingFileFolder, EncodingFilename); + string relativeGitPath = EncodingFileFolder + "/" + EncodingFilename; string contents = virtualFile.ShouldBeAFile(this.FileSystem).WithContents(); string expectedContents = controlFile.ShouldBeAFile(this.FileSystem).WithContents(); contents.ShouldEqual(expectedContents); // Confirm that the entry is not in the the modified paths database - GVFSHelpers.ModifiedPathsShouldNotContain(this.FileSystem, this.Enlistment.DotGVFSRoot, EncodingFilename); + GVFSHelpers.ModifiedPathsShouldNotContain(this.FileSystem, this.Enlistment.DotGVFSRoot, relativeGitPath); this.ValidateGitCommand("status"); this.AppendAllText(ContentWhenEditingFile, virtualFile); @@ -997,7 +999,7 @@ public void EditFileNeedingUtf8Encoding() this.ValidateGitCommand("status"); // Confirm that the entry was added to the modified paths database - GVFSHelpers.ModifiedPathsShouldContain(this.FileSystem, this.Enlistment.DotGVFSRoot, EncodingFilename); + GVFSHelpers.ModifiedPathsShouldContain(this.FileSystem, this.Enlistment.DotGVFSRoot, relativeGitPath); } [TestCase] diff --git a/GVFS/GVFS.FunctionalTests/Tools/GVFSHelpers.cs b/GVFS/GVFS.FunctionalTests/Tools/GVFSHelpers.cs index 95e988c18..58abd7985 100644 --- a/GVFS/GVFS.FunctionalTests/Tools/GVFSHelpers.cs +++ b/GVFS/GVFS.FunctionalTests/Tools/GVFSHelpers.cs @@ -4,10 +4,10 @@ using Microsoft.Data.Sqlite; using Newtonsoft.Json; using NUnit.Framework; +using System; using System.Collections.Generic; using System.IO; using System.Linq; -using System.Reflection; namespace GVFS.FunctionalTests.Tools { @@ -19,6 +19,9 @@ public static class GVFSHelpers public static readonly string PlaceholderListFile = Path.Combine("databases", "PlaceholderList.dat"); public static readonly string RepoMetadataName = Path.Combine("databases", "RepoMetadata.dat"); + private const string ModifedPathsLineAddPrefix = "A "; + private const string ModifedPathsLineDeletePrefix = "D "; + private const string DiskLayoutMajorVersionKey = "DiskLayoutVersion"; private const string DiskLayoutMinorVersionKey = "DiskLayoutMinorVersion"; private const string LocalCacheRootKey = "LocalCacheRoot"; @@ -111,15 +114,29 @@ public static void ModifiedPathsShouldContain(FileSystemRunner fileSystem, strin { string modifiedPathsDatabase = Path.Combine(dotGVFSRoot, TestConstants.Databases.ModifiedPaths); modifiedPathsDatabase.ShouldBeAFile(fileSystem); - GVFSHelpers.ReadAllTextFromWriteLockedFile(modifiedPathsDatabase).ShouldContain( - gitPaths.Select(path => path + ModifiedPathsNewLine).ToArray()); + string modifedPathsContents = GVFSHelpers.ReadAllTextFromWriteLockedFile(modifiedPathsDatabase); + string[] modifedPathLines = modifedPathsContents.Split(new[] { ModifiedPathsNewLine }, StringSplitOptions.None); + foreach (string gitPath in gitPaths) + { + modifedPathLines.ShouldContain(path => path.Equals(ModifedPathsLineAddPrefix + gitPath)); + } } public static void ModifiedPathsShouldNotContain(FileSystemRunner fileSystem, string dotGVFSRoot, params string[] gitPaths) { string modifiedPathsDatabase = Path.Combine(dotGVFSRoot, TestConstants.Databases.ModifiedPaths); modifiedPathsDatabase.ShouldBeAFile(fileSystem); - GVFSHelpers.ReadAllTextFromWriteLockedFile(modifiedPathsDatabase).ShouldNotContain(ignoreCase: true, unexpectedSubstrings: gitPaths); + string modifedPathsContents = GVFSHelpers.ReadAllTextFromWriteLockedFile(modifiedPathsDatabase); + string[] modifedPathLines = modifedPathsContents.Split(new[] { ModifiedPathsNewLine }, StringSplitOptions.None); + foreach (string gitPath in gitPaths) + { + modifedPathLines.ShouldNotContain( + path => + { + return path.Equals(ModifedPathsLineAddPrefix + gitPath, StringComparison.OrdinalIgnoreCase) || + path.Equals(ModifedPathsLineDeletePrefix + gitPath, StringComparison.OrdinalIgnoreCase); + }); + } } private static byte[] StringToShaBytes(string sha) From 5548b3d198f99ba5af9092506dfc854475a107e2 Mon Sep 17 00:00:00 2001 From: William Baker Date: Mon, 1 Oct 2018 15:51:16 -0700 Subject: [PATCH 183/272] Make FileTypeAndMode an internal struct and fix StyleCop unit test issue --- .../MacFileSystemVirtualizer.cs | 27 ++++--- .../Projection/MockGitIndexProjection.cs | 19 +++-- .../MacFileSystemVirtualizerTests.cs | 77 ++++++++++++++----- .../Projection/FileTypeAndMode.cs | 63 --------------- .../GitIndexProjection.FileTypeAndMode.cs | 47 +++++++++++ .../GitIndexProjection.GitIndexParser.cs | 12 +-- .../Projection/GitIndexProjection.cs | 29 +++++-- 7 files changed, 160 insertions(+), 114 deletions(-) delete mode 100644 GVFS/GVFS.Virtualization/Projection/FileTypeAndMode.cs create mode 100644 GVFS/GVFS.Virtualization/Projection/GitIndexProjection.FileTypeAndMode.cs diff --git a/GVFS/GVFS.Platform.Mac/MacFileSystemVirtualizer.cs b/GVFS/GVFS.Platform.Mac/MacFileSystemVirtualizer.cs index e351f4f75..4b20c7e4a 100644 --- a/GVFS/GVFS.Platform.Mac/MacFileSystemVirtualizer.cs +++ b/GVFS/GVFS.Platform.Mac/MacFileSystemVirtualizer.cs @@ -83,20 +83,22 @@ public override FileSystemResult WritePlaceholderFile( string sha) { // TODO(Mac): Add functional tests that validate file mode is set correctly - FileTypeAndMode fileTypeAndMode = this.FileSystemCallbacks.GitIndexProjection.GetFileTypeAndMode(relativePath); + GitIndexProjection.FileType fileType; + ushort fileMode; + this.FileSystemCallbacks.GitIndexProjection.GetFileTypeAndMode(relativePath, out fileType, out fileMode); - if (fileTypeAndMode.Type == FileTypeAndMode.FileType.Regular) + if (fileType == GitIndexProjection.FileType.Regular) { Result result = this.virtualizationInstance.WritePlaceholderFile( relativePath, PlaceholderVersionId, ToVersionIdByteArray(FileSystemVirtualizer.ConvertShaToContentId(sha)), (ulong)endOfFile, - fileTypeAndMode.Mode); + fileMode); return new FileSystemResult(ResultToFSResult(result), unchecked((int)result)); } - else if (fileTypeAndMode.Type == FileTypeAndMode.FileType.SymLink) + else if (fileType == GitIndexProjection.FileType.SymLink) { string symLinkTarget; if (this.TryGetSymLinkTarget(sha, out symLinkTarget)) @@ -116,7 +118,8 @@ public override FileSystemResult WritePlaceholderFile( else { EventMetadata metadata = this.CreateEventMetadata(relativePath); - metadata.Add("FileType", fileTypeAndMode.Type); + metadata.Add(nameof(fileType), fileType); + metadata.Add(nameof(fileMode), fileMode); this.Context.Tracer.RelatedError(metadata, $"{nameof(this.WritePlaceholderFile)}: Unsupported fileType"); return new FileSystemResult(FSResult.IOError, 0); } @@ -145,23 +148,25 @@ public override FileSystemResult UpdatePlaceholderIfNeeded( // TODO(Mac): Add functional tests that include: // - Mode + content changes between commits // - Mode only changes (without any change to content, see issue #223) - FileTypeAndMode fileTypeAndMode = this.FileSystemCallbacks.GitIndexProjection.GetFileTypeAndMode(relativePath); + GitIndexProjection.FileType fileType; + ushort fileMode; + this.FileSystemCallbacks.GitIndexProjection.GetFileTypeAndMode(relativePath, out fileType, out fileMode); - if (fileTypeAndMode.Type == FileTypeAndMode.FileType.Regular) + if (fileType == GitIndexProjection.FileType.Regular) { Result result = this.virtualizationInstance.UpdatePlaceholderIfNeeded( relativePath, PlaceholderVersionId, ToVersionIdByteArray(ConvertShaToContentId(shaContentId)), (ulong)endOfFile, - fileTypeAndMode.Mode, + fileMode, (UpdateType)updateFlags, out failureCause); failureReason = (UpdateFailureReason)failureCause; return new FileSystemResult(ResultToFSResult(result), unchecked((int)result)); } - else if (fileTypeAndMode.Type == FileTypeAndMode.FileType.SymLink) + else if (fileType == GitIndexProjection.FileType.SymLink) { string symLinkTarget; if (this.TryGetSymLinkTarget(shaContentId, out symLinkTarget)) @@ -187,8 +192,8 @@ public override FileSystemResult UpdatePlaceholderIfNeeded( else { EventMetadata metadata = this.CreateEventMetadata(relativePath); - metadata.Add("FileType", fileTypeAndMode.Type); - metadata.Add("FileMode", fileTypeAndMode.Mode); + metadata.Add(nameof(fileType), fileType); + metadata.Add(nameof(fileMode), fileMode); this.Context.Tracer.RelatedError(metadata, $"{nameof(this.UpdatePlaceholderIfNeeded)}: Unsupported fileType"); failureReason = UpdateFailureReason.NoFailure; return new FileSystemResult(FSResult.IOError, 0); diff --git a/GVFS/GVFS.UnitTests/Mock/Virtualization/Projection/MockGitIndexProjection.cs b/GVFS/GVFS.UnitTests/Mock/Virtualization/Projection/MockGitIndexProjection.cs index ca877baa8..c83730a73 100644 --- a/GVFS/GVFS.UnitTests/Mock/Virtualization/Projection/MockGitIndexProjection.cs +++ b/GVFS/GVFS.UnitTests/Mock/Virtualization/Projection/MockGitIndexProjection.cs @@ -36,7 +36,7 @@ public MockGitIndexProjection(IEnumerable projectedFiles) this.PlaceholdersCreated = new ConcurrentHashSet(); this.ExpandedFolders = new ConcurrentHashSet(); - this.MockFileTypesAndModes = new ConcurrentDictionary(); + this.MockFileTypesAndModes = new ConcurrentDictionary(); this.unblockGetProjectedItems = new ManualResetEvent(true); this.waitForGetProjectedItems = new ManualResetEvent(true); @@ -56,7 +56,7 @@ public MockGitIndexProjection(IEnumerable projectedFiles) public ConcurrentHashSet ExpandedFolders { get; } - public ConcurrentDictionary MockFileTypesAndModes { get; } + public ConcurrentDictionary MockFileTypesAndModes { get; } public bool ThrowOperationCanceledExceptionOnProjectionRequest { get; set; } @@ -161,15 +161,18 @@ public override bool TryGetProjectedItemsFromMemory(string folderPath, out List< return false; } - public override FileTypeAndMode GetFileTypeAndMode(string path) + public override void GetFileTypeAndMode(string path, out FileType fileType, out ushort fileMode) { - FileTypeAndMode result; - if (this.MockFileTypesAndModes.TryGetValue(path, out result)) + fileType = FileType.Invalid; + fileMode = 0; + + ushort mockFileTypeAndMode; + if (this.MockFileTypesAndModes.TryGetValue(path, out mockFileTypeAndMode)) { - return result; + FileTypeAndMode typeAndMode = new FileTypeAndMode(mockFileTypeAndMode); + fileType = typeAndMode.Type; + fileMode = typeAndMode.Mode; } - - return new FileTypeAndMode(0); } public override List GetProjectedItems( diff --git a/GVFS/GVFS.UnitTests/Platform.Mac/MacFileSystemVirtualizerTests.cs b/GVFS/GVFS.UnitTests/Platform.Mac/MacFileSystemVirtualizerTests.cs index 353a82e26..96c2d6e28 100644 --- a/GVFS/GVFS.UnitTests/Platform.Mac/MacFileSystemVirtualizerTests.cs +++ b/GVFS/GVFS.UnitTests/Platform.Mac/MacFileSystemVirtualizerTests.cs @@ -22,7 +22,6 @@ namespace GVFS.UnitTests.Platform.Mac [TestFixture] public class MacFileSystemVirtualizerTests : TestsWithCommonRepo { - private static readonly FileTypeAndMode Regular644FileTypeAndMode = new FileTypeAndMode(FileTypeAndMode.FileType.Regular, FileTypeAndMode.FileMode644); private static readonly Dictionary MappedResults = new Dictionary() { { Result.Success, FSResult.Ok }, @@ -98,7 +97,10 @@ public void UpdatePlaceholderIfNeeded() backgroundTaskRunner, virtualizer)) { - gitIndexProjection.MockFileTypesAndModes.TryAdd(UpdatePlaceholderFileName, Regular644FileTypeAndMode); + gitIndexProjection.MockFileTypesAndModes.TryAdd( + UpdatePlaceholderFileName, + ConvertFileTypeAndModeToIndexFormat(GitIndexProjection.FileType.Regular, GitIndexProjection.FileMode644)); + string error; fileSystemCallbacks.TryStart(out error).ShouldEqual(true); @@ -120,7 +122,7 @@ public void UpdatePlaceholderIfNeeded() out failureReason) .ShouldEqual(new FileSystemResult(FSResult.Ok, (int)mockVirtualization.UpdatePlaceholderIfNeededResult)); failureReason.ShouldEqual((UpdateFailureReason)mockVirtualization.UpdatePlaceholderIfNeededFailureCause); - mockVirtualization.UpdatedPlaceholders.ShouldContain(path => path.Key.Equals(UpdatePlaceholderFileName) && path.Value == FileTypeAndMode.FileMode644); + mockVirtualization.UpdatedPlaceholders.ShouldContain(path => path.Key.Equals(UpdatePlaceholderFileName) && path.Value == GitIndexProjection.FileMode644); mockVirtualization.UpdatedPlaceholders.Clear(); mockVirtualization.UpdatePlaceholderIfNeededResult = Result.EFileNotFound; @@ -180,7 +182,9 @@ public void WritePlaceholderForSymLink() backgroundTaskRunner, virtualizer)) { - gitIndexProjection.MockFileTypesAndModes.TryAdd(WriteSymLinkFileName, new FileTypeAndMode(FileTypeAndMode.FileType.SymLink, mode: 0)); + gitIndexProjection.MockFileTypesAndModes.TryAdd( + WriteSymLinkFileName, + ConvertFileTypeAndModeToIndexFormat(GitIndexProjection.FileType.SymLink, fileMode: 0)); string error; fileSystemCallbacks.TryStart(out error).ShouldEqual(true); @@ -220,7 +224,10 @@ public void UpdatePlaceholderToSymLink() backgroundTaskRunner, virtualizer)) { - gitIndexProjection.MockFileTypesAndModes.TryAdd(PlaceholderToLinkFileName, new FileTypeAndMode(FileTypeAndMode.FileType.SymLink, mode: 0)); + gitIndexProjection.MockFileTypesAndModes.TryAdd( + PlaceholderToLinkFileName, + ConvertFileTypeAndModeToIndexFormat(GitIndexProjection.FileType.SymLink, fileMode: 0)); + string error; fileSystemCallbacks.TryStart(out error).ShouldEqual(true); @@ -289,7 +296,9 @@ public void OnEnumerateDirectoryReturnsSuccessWhenResultsNotInMemory() backgroundFileSystemTaskRunner: backgroundTaskRunner, fileSystemVirtualizer: virtualizer)) { - gitIndexProjection.MockFileTypesAndModes.TryAdd(testFilePath, Regular644FileTypeAndMode); + gitIndexProjection.MockFileTypesAndModes.TryAdd( + testFilePath, + ConvertFileTypeAndModeToIndexFormat(GitIndexProjection.FileType.Regular, GitIndexProjection.FileMode644)); string error; fileSystemCallbacks.TryStart(out error).ShouldEqual(true); @@ -298,7 +307,7 @@ public void OnEnumerateDirectoryReturnsSuccessWhenResultsNotInMemory() gitIndexProjection.EnumerationInMemory = false; mockVirtualization.OnEnumerateDirectory(1, TestFolderName, triggeringProcessId: 1, triggeringProcessName: "UnitTests").ShouldEqual(Result.Success); mockVirtualization.CreatedPlaceholders.ShouldContain( - kvp => kvp.Key.Equals(testFilePath, StringComparison.OrdinalIgnoreCase) && kvp.Value == FileTypeAndMode.FileMode644); + kvp => kvp.Key.Equals(testFilePath, StringComparison.OrdinalIgnoreCase) && kvp.Value == GitIndexProjection.FileMode644); fileSystemCallbacks.Stop(); } } @@ -325,7 +334,9 @@ public void OnEnumerateDirectoryReturnsSuccessWhenResultsInMemory() backgroundFileSystemTaskRunner: backgroundTaskRunner, fileSystemVirtualizer: virtualizer)) { - gitIndexProjection.MockFileTypesAndModes.TryAdd(testFilePath, Regular644FileTypeAndMode); + gitIndexProjection.MockFileTypesAndModes.TryAdd( + testFilePath, + ConvertFileTypeAndModeToIndexFormat(GitIndexProjection.FileType.Regular, GitIndexProjection.FileMode644)); string error; fileSystemCallbacks.TryStart(out error).ShouldEqual(true); @@ -334,7 +345,7 @@ public void OnEnumerateDirectoryReturnsSuccessWhenResultsInMemory() gitIndexProjection.EnumerationInMemory = true; mockVirtualization.OnEnumerateDirectory(1, TestFolderName, triggeringProcessId: 1, triggeringProcessName: "UnitTests").ShouldEqual(Result.Success); mockVirtualization.CreatedPlaceholders.ShouldContain( - kvp => kvp.Key.Equals(testFilePath, StringComparison.OrdinalIgnoreCase) && kvp.Value == FileTypeAndMode.FileMode644); + kvp => kvp.Key.Equals(testFilePath, StringComparison.OrdinalIgnoreCase) && kvp.Value == GitIndexProjection.FileMode644); gitIndexProjection.ExpandedFolders.ShouldMatchInOrder(TestFolderName); fileSystemCallbacks.Stop(); } @@ -347,9 +358,9 @@ public void OnEnumerateDirectorySetsFileModes() const string TestFile664Name = "test664.txt"; const string TestFile755Name = "test755.txt"; const string TestFolderName = "testFolder"; - string TestFile644Path = Path.Combine(TestFolderName, TestFile644Name); - string TestFile664Path = Path.Combine(TestFolderName, TestFile664Name); - string TestFile755Path = Path.Combine(TestFolderName, TestFile755Name); + string testFile644Path = Path.Combine(TestFolderName, TestFile644Name); + string testFile664Path = Path.Combine(TestFolderName, TestFile664Name); + string testFile755Path = Path.Combine(TestFolderName, TestFile755Name); using (MockBackgroundFileSystemTaskRunner backgroundTaskRunner = new MockBackgroundFileSystemTaskRunner()) using (MockVirtualizationInstance mockVirtualization = new MockVirtualizationInstance()) @@ -366,9 +377,15 @@ public void OnEnumerateDirectorySetsFileModes() backgroundFileSystemTaskRunner: backgroundTaskRunner, fileSystemVirtualizer: virtualizer)) { - gitIndexProjection.MockFileTypesAndModes.TryAdd(TestFile644Path, Regular644FileTypeAndMode); - gitIndexProjection.MockFileTypesAndModes.TryAdd(TestFile664Path, new FileTypeAndMode(FileTypeAndMode.FileType.Regular, FileTypeAndMode.FileMode664)); - gitIndexProjection.MockFileTypesAndModes.TryAdd(TestFile755Path, new FileTypeAndMode(FileTypeAndMode.FileType.Regular, FileTypeAndMode.FileMode755)); + gitIndexProjection.MockFileTypesAndModes.TryAdd( + testFile644Path, + ConvertFileTypeAndModeToIndexFormat(GitIndexProjection.FileType.Regular, GitIndexProjection.FileMode644)); + gitIndexProjection.MockFileTypesAndModes.TryAdd( + testFile664Path, + ConvertFileTypeAndModeToIndexFormat(GitIndexProjection.FileType.Regular, GitIndexProjection.FileMode664)); + gitIndexProjection.MockFileTypesAndModes.TryAdd( + testFile755Path, + ConvertFileTypeAndModeToIndexFormat(GitIndexProjection.FileType.Regular, GitIndexProjection.FileMode755)); string error; fileSystemCallbacks.TryStart(out error).ShouldEqual(true); @@ -377,11 +394,11 @@ public void OnEnumerateDirectorySetsFileModes() gitIndexProjection.EnumerationInMemory = true; mockVirtualization.OnEnumerateDirectory(1, TestFolderName, triggeringProcessId: 1, triggeringProcessName: "UnitTests").ShouldEqual(Result.Success); mockVirtualization.CreatedPlaceholders.ShouldContain( - kvp => kvp.Key.Equals(TestFile644Path, StringComparison.OrdinalIgnoreCase) && kvp.Value == FileTypeAndMode.FileMode644); + kvp => kvp.Key.Equals(testFile644Path, StringComparison.OrdinalIgnoreCase) && kvp.Value == GitIndexProjection.FileMode644); mockVirtualization.CreatedPlaceholders.ShouldContain( - kvp => kvp.Key.Equals(TestFile664Path, StringComparison.OrdinalIgnoreCase) && kvp.Value == FileTypeAndMode.FileMode664); + kvp => kvp.Key.Equals(testFile664Path, StringComparison.OrdinalIgnoreCase) && kvp.Value == GitIndexProjection.FileMode664); mockVirtualization.CreatedPlaceholders.ShouldContain( - kvp => kvp.Key.Equals(TestFile755Path, StringComparison.OrdinalIgnoreCase) && kvp.Value == FileTypeAndMode.FileMode755); + kvp => kvp.Key.Equals(testFile755Path, StringComparison.OrdinalIgnoreCase) && kvp.Value == GitIndexProjection.FileMode755); fileSystemCallbacks.Stop(); } } @@ -470,5 +487,29 @@ public void OnGetFileStreamReturnsErrorWhenWriteFileContentsFails() fileSystemCallbacks.Stop(); } } + + private static ushort ConvertFileTypeAndModeToIndexFormat(GitIndexProjection.FileType fileType, ushort fileMode) + { + // Values used in the index file to indicate the type of the file + const ushort RegularFileIndexEntry = 0x8000; + const ushort SymLinkFileIndexEntry = 0xA000; + const ushort GitLinkFileIndexEntry = 0xE000; + + switch (fileType) + { + case GitIndexProjection.FileType.Regular: + return (ushort)(RegularFileIndexEntry | fileMode); + + case GitIndexProjection.FileType.SymLink: + return (ushort)(SymLinkFileIndexEntry | fileMode); + + case GitIndexProjection.FileType.GitLink: + return (ushort)(GitLinkFileIndexEntry | fileMode); + + default: + Assert.Fail($"Invalid fileType {fileType}"); + return 0; + } + } } } diff --git a/GVFS/GVFS.Virtualization/Projection/FileTypeAndMode.cs b/GVFS/GVFS.Virtualization/Projection/FileTypeAndMode.cs deleted file mode 100644 index eaea06fd7..000000000 --- a/GVFS/GVFS.Virtualization/Projection/FileTypeAndMode.cs +++ /dev/null @@ -1,63 +0,0 @@ -using System; -namespace GVFS.Virtualization.Projection -{ - public struct FileTypeAndMode - { - public static readonly ushort FileMode755 = Convert.ToUInt16("755", 8); - public static readonly ushort FileMode664 = Convert.ToUInt16("664", 8); - public static readonly ushort FileMode644 = Convert.ToUInt16("644", 8); - - // Bitmasks for extracting file type and mode from the ushort stored in the index - private const ushort FileTypeMask = 0xF000; - private const ushort FileModeMask = 0x1FF; - - // Values used in the index file to indicate the type of the file - private const ushort RegularFileIndexEntry = 0x8000; - private const ushort SymLinkFileIndexEntry = 0xA000; - private const ushort GitLinkFileIndexEntry = 0xE000; - - public FileTypeAndMode(ushort typeAndModeInIndexFormat) - { - switch (typeAndModeInIndexFormat & FileTypeMask) - { - case RegularFileIndexEntry: - this.Type = FileType.Regular; - break; - case SymLinkFileIndexEntry: - this.Type = FileType.SymLink; - break; - case GitLinkFileIndexEntry: - this.Type = FileType.GitLink; - break; - default: - this.Type = FileType.Invalid; - break; - } - - this.Mode = (ushort)(typeAndModeInIndexFormat & FileModeMask); - } - - public FileTypeAndMode(FileType type, ushort mode) - { - this.Type = type; - this.Mode = mode; - } - - public enum FileType : short - { - Invalid, - - Regular, - SymLink, - GitLink, - } - - public FileType Type { get; } - public ushort Mode { get; } - - public string GetModeAsOctalString() - { - return Convert.ToString(this.Mode, 8); - } - } -} diff --git a/GVFS/GVFS.Virtualization/Projection/GitIndexProjection.FileTypeAndMode.cs b/GVFS/GVFS.Virtualization/Projection/GitIndexProjection.FileTypeAndMode.cs new file mode 100644 index 000000000..7c20ca2c5 --- /dev/null +++ b/GVFS/GVFS.Virtualization/Projection/GitIndexProjection.FileTypeAndMode.cs @@ -0,0 +1,47 @@ +using System; +namespace GVFS.Virtualization.Projection +{ + public partial class GitIndexProjection + { + internal struct FileTypeAndMode + { + // Bitmasks for extracting file type and mode from the ushort stored in the index + private const ushort FileTypeMask = 0xF000; + private const ushort FileModeMask = 0x1FF; + + // Values used in the index file to indicate the type of the file + private const ushort RegularFileIndexEntry = 0x8000; + private const ushort SymLinkFileIndexEntry = 0xA000; + private const ushort GitLinkFileIndexEntry = 0xE000; + + public FileTypeAndMode(ushort typeAndModeInIndexFormat) + { + switch (typeAndModeInIndexFormat & FileTypeMask) + { + case RegularFileIndexEntry: + this.Type = FileType.Regular; + break; + case SymLinkFileIndexEntry: + this.Type = FileType.SymLink; + break; + case GitLinkFileIndexEntry: + this.Type = FileType.GitLink; + break; + default: + this.Type = FileType.Invalid; + break; + } + + this.Mode = (ushort)(typeAndModeInIndexFormat & FileModeMask); + } + + public FileType Type { get; } + public ushort Mode { get; } + + public string GetModeAsOctalString() + { + return Convert.ToString(this.Mode, 8); + } + } + } +} diff --git a/GVFS/GVFS.Virtualization/Projection/GitIndexProjection.GitIndexParser.cs b/GVFS/GVFS.Virtualization/Projection/GitIndexProjection.GitIndexParser.cs index 3753abb7c..4893ede46 100644 --- a/GVFS/GVFS.Virtualization/Projection/GitIndexProjection.GitIndexParser.cs +++ b/GVFS/GVFS.Virtualization/Projection/GitIndexProjection.GitIndexParser.cs @@ -187,18 +187,18 @@ private FileSystemTaskResult ParseIndex(ITracer tracer, Stream indexStream, Func switch (typeAndMode.Type) { - case FileTypeAndMode.FileType.Regular: - if (typeAndMode.Mode != FileTypeAndMode.FileMode755 && - typeAndMode.Mode != FileTypeAndMode.FileMode644 && - typeAndMode.Mode != FileTypeAndMode.FileMode664) + case FileType.Regular: + if (typeAndMode.Mode != FileMode755 && + typeAndMode.Mode != FileMode644 && + typeAndMode.Mode != FileMode664) { throw new InvalidDataException($"Invalid file mode {typeAndMode.GetModeAsOctalString()} found for regular file in index"); } break; - case FileTypeAndMode.FileType.SymLink: - case FileTypeAndMode.FileType.GitLink: + case FileType.SymLink: + case FileType.GitLink: if (typeAndMode.Mode != 0) { throw new InvalidDataException($"Invalid file mode {typeAndMode.GetModeAsOctalString()} found for link file({typeAndMode.Type:X}) in index"); diff --git a/GVFS/GVFS.Virtualization/Projection/GitIndexProjection.cs b/GVFS/GVFS.Virtualization/Projection/GitIndexProjection.cs index 211c2ce5b..c99808b74 100644 --- a/GVFS/GVFS.Virtualization/Projection/GitIndexProjection.cs +++ b/GVFS/GVFS.Virtualization/Projection/GitIndexProjection.cs @@ -22,6 +22,10 @@ public partial class GitIndexProjection : IDisposable, IProfilerOnlyIndexProject { public const string ProjectionIndexBackupName = "GVFS_projection"; + public static readonly ushort FileMode755 = Convert.ToUInt16("755", 8); + public static readonly ushort FileMode664 = Convert.ToUInt16("664", 8); + public static readonly ushort FileMode644 = Convert.ToUInt16("644", 8); + private const int IndexFileStreamBufferSize = 512 * 1024; private const UpdatePlaceholderType FolderPlaceholderDeleteFlags = @@ -37,8 +41,6 @@ public partial class GitIndexProjection : IDisposable, IProfilerOnlyIndexProject private const int ExternalLockReleaseTimeoutMs = 50; - private static readonly FileTypeAndMode Regular644FileTypeAndMode = new FileTypeAndMode(FileTypeAndMode.FileType.Regular, FileTypeAndMode.FileMode644); - private char[] gitPathSeparatorCharArray = new char[] { GVFSConstants.GitPathSeparator }; private GVFSContext context; @@ -118,6 +120,15 @@ protected GitIndexProjection() { } + public enum FileType : short + { + Invalid, + + Regular, + SymLink, + GitLink, + } + public int EstimatedPlaceholderCount { get @@ -356,13 +367,16 @@ public virtual bool TryGetProjectedItemsFromMemory(string folderPath, out List

Date: Wed, 5 Sep 2018 13:26:42 -0600 Subject: [PATCH 184/272] Compress the entries in the modified paths database when mounting This will search the modified paths for a parent folder and if one exists it will remove the entry from the modified paths list --- GVFS/GVFS.Common/ModifiedPathsDatabase.cs | 28 +++++++++++++++++++ .../Common/ModifiedPathsDatabaseTests.cs | 26 ++++++++++++++++- .../FileSystemCallbacks.cs | 3 ++ 3 files changed, 56 insertions(+), 1 deletion(-) diff --git a/GVFS/GVFS.Common/ModifiedPathsDatabase.cs b/GVFS/GVFS.Common/ModifiedPathsDatabase.cs index 5a7e34c43..9fef5318b 100644 --- a/GVFS/GVFS.Common/ModifiedPathsDatabase.cs +++ b/GVFS/GVFS.Common/ModifiedPathsDatabase.cs @@ -52,6 +52,34 @@ public static bool TryLoadOrCreate(ITracer tracer, string dataDirectory, Physica return true; } + public void Compress(ITracer tracer) + { + int startingCount = this.modifiedPaths.Count; + using (ITracer activity = tracer.StartActivity("Compress ModifiedPaths", EventLevel.Informational)) + { + foreach (var item in this.modifiedPaths) + { + int pathSeparatorIndex = item.IndexOf('/'); + while (pathSeparatorIndex >= 0 && pathSeparatorIndex < item.Length - 1) + { + string folder = item.Substring(0, pathSeparatorIndex + 1); + if (this.modifiedPaths.Contains(folder)) + { + this.modifiedPaths.TryRemove(item); + break; + } + + pathSeparatorIndex = item.IndexOf('/', pathSeparatorIndex + 1); + } + } + + EventMetadata metadata = new EventMetadata(); + metadata.Add(nameof(startingCount), startingCount); + metadata.Add("EndCount", this.modifiedPaths.Count); + activity.Stop(metadata); + } + } + public bool Contains(string path, bool isFolder) { string entry = this.NormalizeEntryString(path, isFolder); diff --git a/GVFS/GVFS.UnitTests/Common/ModifiedPathsDatabaseTests.cs b/GVFS/GVFS.UnitTests/Common/ModifiedPathsDatabaseTests.cs index 663a76bb0..59b52e921 100644 --- a/GVFS/GVFS.UnitTests/Common/ModifiedPathsDatabaseTests.cs +++ b/GVFS/GVFS.UnitTests/Common/ModifiedPathsDatabaseTests.cs @@ -1,6 +1,7 @@ using GVFS.Common; using GVFS.Tests.Should; using GVFS.UnitTests.Mock; +using GVFS.UnitTests.Mock.Common; using GVFS.UnitTests.Mock.FileSystem; using NUnit.Framework; using System; @@ -17,6 +18,17 @@ public class ModifiedPathsDatabaseTests private const string ExistingEntries = @"A file.txt A dir/file2.txt A dir1/dir2/file3.txt +"; + private const string EntriesToCompress = @"A file.txt +A dir/file2.txt +A dir/dir3/dir4/ +A dir1/dir2/file3.txt +A dir/ +A dir1/dir2/ +A dir1/file.txt +A dir1/dir2/dir3/dir4/dir5/ +A dir/dir2/file3.txt +A dir/dir4/dir5/ "; [TestCase] @@ -119,7 +131,19 @@ public void EntryNotAddedIfParentDirectoryExists() modifiedPathsDatabase.Contains("dir2/dir", isFolder: true).ShouldBeTrue(); } - private static void TestAddingPath(string path, bool isFolder = false) + [TestCase] + public void CompressEntries() + { + ModifiedPathsDatabase mpd = CreateModifiedPathsDatabase(EntriesToCompress); + mpd.Compress(new MockTracer()); + mpd.Count.ShouldEqual(4); + mpd.Contains("file.txt", isFolder: false).ShouldBeTrue(); + mpd.Contains("dir/", isFolder: true).ShouldBeTrue(); + mpd.Contains("dir1/dir2/", isFolder: true).ShouldBeTrue(); + mpd.Contains("dir1/file.txt", isFolder: false).ShouldBeTrue(); + } + + private static void TestAddingPath(string path, bool isFolder = false) { TestAddingPath(path, path, isFolder); } diff --git a/GVFS/GVFS.Virtualization/FileSystemCallbacks.cs b/GVFS/GVFS.Virtualization/FileSystemCallbacks.cs index d7c0d57b4..44bce1687 100644 --- a/GVFS/GVFS.Virtualization/FileSystemCallbacks.cs +++ b/GVFS/GVFS.Virtualization/FileSystemCallbacks.cs @@ -89,6 +89,9 @@ public FileSystemCallbacks( throw new InvalidRepoException(error); } + this.modifiedPaths.Compress(this.context.Tracer); + this.modifiedPaths.WriteAllEntriesAndFlush(); + this.BlobSizes = blobSizes; this.BlobSizes.Initialize(); From d28270e8ea06f8f47a4ad827df3f6f04545663c9 Mon Sep 17 00:00:00 2001 From: Kevin Willford Date: Wed, 26 Sep 2018 15:15:26 -0600 Subject: [PATCH 185/272] Fix test for modified paths that was broken --- .../Tests/EnlistmentPerTestCase/ModifiedPathsTests.cs | 5 ----- 1 file changed, 5 deletions(-) diff --git a/GVFS/GVFS.FunctionalTests/Tests/EnlistmentPerTestCase/ModifiedPathsTests.cs b/GVFS/GVFS.FunctionalTests/Tests/EnlistmentPerTestCase/ModifiedPathsTests.cs index f8e42331e..f2005745e 100644 --- a/GVFS/GVFS.FunctionalTests/Tests/EnlistmentPerTestCase/ModifiedPathsTests.cs +++ b/GVFS/GVFS.FunctionalTests/Tests/EnlistmentPerTestCase/ModifiedPathsTests.cs @@ -37,11 +37,6 @@ public class ModifiedPathsTests : TestsWithEnlistmentPerTestCase $"A {RenameNewDotGitFileTarget}", $"A {FileToCreateOutsideRepo}", $"A {FolderToCreateOutsideRepo}/", - $"A {FolderToDelete}/CreateCommonAssemblyVersion.bat", - $"A {FolderToDelete}/CreateCommonCliAssemblyVersion.bat", - $"A {FolderToDelete}/CreateCommonVersionHeader.bat", - $"A {FolderToDelete}/RunFunctionalTests.bat", - $"A {FolderToDelete}/RunUnitTests.bat", $"A {FolderToDelete}/", }; From 36fef4d4335fead8127d68cb2f48c4ecc2ebd9c0 Mon Sep 17 00:00:00 2001 From: Kevin Willford Date: Wed, 26 Sep 2018 15:20:18 -0600 Subject: [PATCH 186/272] Use better names in the modified paths test --- .../Common/ModifiedPathsDatabaseTests.cs | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/GVFS/GVFS.UnitTests/Common/ModifiedPathsDatabaseTests.cs b/GVFS/GVFS.UnitTests/Common/ModifiedPathsDatabaseTests.cs index 59b52e921..bc3106da5 100644 --- a/GVFS/GVFS.UnitTests/Common/ModifiedPathsDatabaseTests.cs +++ b/GVFS/GVFS.UnitTests/Common/ModifiedPathsDatabaseTests.cs @@ -134,16 +134,16 @@ public void EntryNotAddedIfParentDirectoryExists() [TestCase] public void CompressEntries() { - ModifiedPathsDatabase mpd = CreateModifiedPathsDatabase(EntriesToCompress); - mpd.Compress(new MockTracer()); - mpd.Count.ShouldEqual(4); - mpd.Contains("file.txt", isFolder: false).ShouldBeTrue(); - mpd.Contains("dir/", isFolder: true).ShouldBeTrue(); - mpd.Contains("dir1/dir2/", isFolder: true).ShouldBeTrue(); - mpd.Contains("dir1/file.txt", isFolder: false).ShouldBeTrue(); - } + ModifiedPathsDatabase modifiedPathsDatabase = CreateModifiedPathsDatabase(EntriesToCompress); + modifiedPathsDatabase.Compress(new MockTracer()); + modifiedPathsDatabase.Count.ShouldEqual(4); + modifiedPathsDatabase.Contains("file.txt", isFolder: false).ShouldBeTrue(); + modifiedPathsDatabase.Contains("dir/", isFolder: true).ShouldBeTrue(); + modifiedPathsDatabase.Contains("dir1/dir2/", isFolder: true).ShouldBeTrue(); + modifiedPathsDatabase.Contains("dir1/file.txt", isFolder: false).ShouldBeTrue(); + } - private static void TestAddingPath(string path, bool isFolder = false) + private static void TestAddingPath(string path, bool isFolder = false) { TestAddingPath(path, path, isFolder); } From 2bbca795df85e63392955a754c9dbe4dd9ea40bb Mon Sep 17 00:00:00 2001 From: Kevin Willford Date: Wed, 26 Sep 2018 15:24:23 -0600 Subject: [PATCH 187/272] Move code to remove modified path entries to the TryStart method --- GVFS/GVFS.Virtualization/FileSystemCallbacks.cs | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/GVFS/GVFS.Virtualization/FileSystemCallbacks.cs b/GVFS/GVFS.Virtualization/FileSystemCallbacks.cs index 44bce1687..6ff4d40f3 100644 --- a/GVFS/GVFS.Virtualization/FileSystemCallbacks.cs +++ b/GVFS/GVFS.Virtualization/FileSystemCallbacks.cs @@ -89,9 +89,6 @@ public FileSystemCallbacks( throw new InvalidRepoException(error); } - this.modifiedPaths.Compress(this.context.Tracer); - this.modifiedPaths.WriteAllEntriesAndFlush(); - this.BlobSizes = blobSizes; this.BlobSizes.Initialize(); @@ -174,6 +171,9 @@ public static bool IsPathInsideDotGit(string relativePath) public bool TryStart(out string error) { + this.modifiedPaths.Compress(this.context.Tracer); + this.modifiedPaths.WriteAllEntriesAndFlush(); + if (!this.fileSystemVirtualizer.TryStart(this, out error)) { return false; From f0bcb3425f4928848d7f4c4918c84d7ac1a76208 Mon Sep 17 00:00:00 2001 From: Kevin Willford Date: Wed, 26 Sep 2018 16:09:52 -0600 Subject: [PATCH 188/272] Rename method and use Split method for building parent paths --- GVFS/GVFS.Common/ModifiedPathsDatabase.cs | 26 ++++++++++++------- .../Common/ModifiedPathsDatabaseTests.cs | 2 +- .../FileSystemCallbacks.cs | 2 +- 3 files changed, 19 insertions(+), 11 deletions(-) diff --git a/GVFS/GVFS.Common/ModifiedPathsDatabase.cs b/GVFS/GVFS.Common/ModifiedPathsDatabase.cs index 9fef5318b..6f40923b3 100644 --- a/GVFS/GVFS.Common/ModifiedPathsDatabase.cs +++ b/GVFS/GVFS.Common/ModifiedPathsDatabase.cs @@ -1,6 +1,8 @@ using System; using System.Collections.Generic; using System.IO; +using System.Linq; +using System.Text; using GVFS.Common.FileSystem; using GVFS.Common.Tracing; @@ -52,24 +54,30 @@ public static bool TryLoadOrCreate(ITracer tracer, string dataDirectory, Physica return true; } - public void Compress(ITracer tracer) + ///

+ /// This method will examine the modified paths to check if there is already a parent folder entry in + /// the modified paths. If there is a parent folder the entry does not need to be in the modified paths + /// and will be removed because the parent folder is recursive and covers any children. + /// + public void RemoveEntriesWithParentFolderEntry(ITracer tracer) { int startingCount = this.modifiedPaths.Count; using (ITracer activity = tracer.StartActivity("Compress ModifiedPaths", EventLevel.Informational)) { - foreach (var item in this.modifiedPaths) + StringBuilder parentFolder = new StringBuilder(); + foreach (string modifiedPath in this.modifiedPaths) { - int pathSeparatorIndex = item.IndexOf('/'); - while (pathSeparatorIndex >= 0 && pathSeparatorIndex < item.Length - 1) + string[] pathParts = modifiedPath.Split(new char[] { '/' }, StringSplitOptions.RemoveEmptyEntries); + + parentFolder.Clear(); + foreach (string pathPart in pathParts.Take(pathParts.Length - 1)) { - string folder = item.Substring(0, pathSeparatorIndex + 1); - if (this.modifiedPaths.Contains(folder)) + parentFolder.Append(pathPart + "/"); + if (this.modifiedPaths.Contains(parentFolder.ToString())) { - this.modifiedPaths.TryRemove(item); + this.modifiedPaths.TryRemove(modifiedPath); break; } - - pathSeparatorIndex = item.IndexOf('/', pathSeparatorIndex + 1); } } diff --git a/GVFS/GVFS.UnitTests/Common/ModifiedPathsDatabaseTests.cs b/GVFS/GVFS.UnitTests/Common/ModifiedPathsDatabaseTests.cs index bc3106da5..ced7ba585 100644 --- a/GVFS/GVFS.UnitTests/Common/ModifiedPathsDatabaseTests.cs +++ b/GVFS/GVFS.UnitTests/Common/ModifiedPathsDatabaseTests.cs @@ -135,7 +135,7 @@ public void EntryNotAddedIfParentDirectoryExists() public void CompressEntries() { ModifiedPathsDatabase modifiedPathsDatabase = CreateModifiedPathsDatabase(EntriesToCompress); - modifiedPathsDatabase.Compress(new MockTracer()); + modifiedPathsDatabase.RemoveEntriesWithParentFolder(new MockTracer()); modifiedPathsDatabase.Count.ShouldEqual(4); modifiedPathsDatabase.Contains("file.txt", isFolder: false).ShouldBeTrue(); modifiedPathsDatabase.Contains("dir/", isFolder: true).ShouldBeTrue(); diff --git a/GVFS/GVFS.Virtualization/FileSystemCallbacks.cs b/GVFS/GVFS.Virtualization/FileSystemCallbacks.cs index 6ff4d40f3..627c895a5 100644 --- a/GVFS/GVFS.Virtualization/FileSystemCallbacks.cs +++ b/GVFS/GVFS.Virtualization/FileSystemCallbacks.cs @@ -171,7 +171,7 @@ public static bool IsPathInsideDotGit(string relativePath) public bool TryStart(out string error) { - this.modifiedPaths.Compress(this.context.Tracer); + this.modifiedPaths.RemoveEntriesWithParentFolderEntry(this.context.Tracer); this.modifiedPaths.WriteAllEntriesAndFlush(); if (!this.fileSystemVirtualizer.TryStart(this, out error)) From 02442473d9c1fba9d8361f7a4305e4dee06f59b0 Mon Sep 17 00:00:00 2001 From: Kevin Willford Date: Thu, 27 Sep 2018 12:28:43 -0600 Subject: [PATCH 189/272] Add more tests for modified paths --- .../EnlistmentPerTestCase/ModifiedPathsTests.cs | 13 ++++++++++--- .../Common/ModifiedPathsDatabaseTests.cs | 11 ++++++++--- 2 files changed, 18 insertions(+), 6 deletions(-) diff --git a/GVFS/GVFS.FunctionalTests/Tests/EnlistmentPerTestCase/ModifiedPathsTests.cs b/GVFS/GVFS.FunctionalTests/Tests/EnlistmentPerTestCase/ModifiedPathsTests.cs index f2005745e..f2f255180 100644 --- a/GVFS/GVFS.FunctionalTests/Tests/EnlistmentPerTestCase/ModifiedPathsTests.cs +++ b/GVFS/GVFS.FunctionalTests/Tests/EnlistmentPerTestCase/ModifiedPathsTests.cs @@ -133,9 +133,16 @@ public void ModifiedPathsSavedAfterRemount(FileSystemRunner fileSystem) folderToCreateOutsideRepoTargetPath.ShouldBeADirectory(fileSystem); folderToCreateOutsideRepoPath.ShouldNotExistOnDisk(fileSystem); - string folderToDelete = this.Enlistment.GetVirtualPathTo(FolderToDelete); - fileSystem.DeleteDirectory(folderToDelete); - folderToDelete.ShouldNotExistOnDisk(fileSystem); + string folderToDeleteFullPath = this.Enlistment.GetVirtualPathTo(FolderToDelete); + fileSystem.WriteAllText(Path.Combine(folderToDeleteFullPath, "NewFile.txt"), "Contents for new file"); + string newFileToDelete = Path.Combine(folderToDeleteFullPath, "NewFileToDelete.txt"); + fileSystem.WriteAllText(newFileToDelete, "Contents for new file"); + fileSystem.DeleteFile(newFileToDelete); + fileSystem.WriteAllText(Path.Combine(folderToDeleteFullPath, "CreateCommonVersionHeader.bat"), "Changing the file contents"); + fileSystem.DeleteFile(Path.Combine(folderToDeleteFullPath, "RunUnitTests.bat")); + + fileSystem.DeleteDirectory(folderToDeleteFullPath); + folderToDeleteFullPath.ShouldNotExistOnDisk(fileSystem); // Remount this.Enlistment.UnmountGVFS(); diff --git a/GVFS/GVFS.UnitTests/Common/ModifiedPathsDatabaseTests.cs b/GVFS/GVFS.UnitTests/Common/ModifiedPathsDatabaseTests.cs index ced7ba585..40bacf08f 100644 --- a/GVFS/GVFS.UnitTests/Common/ModifiedPathsDatabaseTests.cs +++ b/GVFS/GVFS.UnitTests/Common/ModifiedPathsDatabaseTests.cs @@ -20,15 +20,19 @@ A dir/file2.txt A dir1/dir2/file3.txt "; private const string EntriesToCompress = @"A file.txt +D deleted.txt A dir/file2.txt A dir/dir3/dir4/ A dir1/dir2/file3.txt A dir/ +D deleted/ A dir1/dir2/ A dir1/file.txt A dir1/dir2/dir3/dir4/dir5/ A dir/dir2/file3.txt A dir/dir4/dir5/ +D dir/dir2/deleted.txt +A dir1/dir2 "; [TestCase] @@ -132,13 +136,14 @@ public void EntryNotAddedIfParentDirectoryExists() } [TestCase] - public void CompressEntries() + public void RemoveEntriesWithParentFolderEntry() { ModifiedPathsDatabase modifiedPathsDatabase = CreateModifiedPathsDatabase(EntriesToCompress); - modifiedPathsDatabase.RemoveEntriesWithParentFolder(new MockTracer()); - modifiedPathsDatabase.Count.ShouldEqual(4); + modifiedPathsDatabase.RemoveEntriesWithParentFolderEntry(new MockTracer()); + modifiedPathsDatabase.Count.ShouldEqual(5); modifiedPathsDatabase.Contains("file.txt", isFolder: false).ShouldBeTrue(); modifiedPathsDatabase.Contains("dir/", isFolder: true).ShouldBeTrue(); + modifiedPathsDatabase.Contains("dir1/dir2", isFolder: false).ShouldBeTrue(); modifiedPathsDatabase.Contains("dir1/dir2/", isFolder: true).ShouldBeTrue(); modifiedPathsDatabase.Contains("dir1/file.txt", isFolder: false).ShouldBeTrue(); } From c3f5aaec9726d1a17c5595e33e741749ee57d426 Mon Sep 17 00:00:00 2001 From: Kevin Willford Date: Mon, 1 Oct 2018 23:07:21 -0600 Subject: [PATCH 190/272] Change to use ContainsParentDirectory for removing modified paths --- GVFS/GVFS.Common/ModifiedPathsDatabase.cs | 17 +++-------------- 1 file changed, 3 insertions(+), 14 deletions(-) diff --git a/GVFS/GVFS.Common/ModifiedPathsDatabase.cs b/GVFS/GVFS.Common/ModifiedPathsDatabase.cs index 6f40923b3..95fb18e73 100644 --- a/GVFS/GVFS.Common/ModifiedPathsDatabase.cs +++ b/GVFS/GVFS.Common/ModifiedPathsDatabase.cs @@ -1,8 +1,6 @@ using System; using System.Collections.Generic; using System.IO; -using System.Linq; -using System.Text; using GVFS.Common.FileSystem; using GVFS.Common.Tracing; @@ -62,22 +60,13 @@ public static bool TryLoadOrCreate(ITracer tracer, string dataDirectory, Physica public void RemoveEntriesWithParentFolderEntry(ITracer tracer) { int startingCount = this.modifiedPaths.Count; - using (ITracer activity = tracer.StartActivity("Compress ModifiedPaths", EventLevel.Informational)) + using (ITracer activity = tracer.StartActivity(nameof(this.RemoveEntriesWithParentFolderEntry), EventLevel.Informational)) { - StringBuilder parentFolder = new StringBuilder(); foreach (string modifiedPath in this.modifiedPaths) { - string[] pathParts = modifiedPath.Split(new char[] { '/' }, StringSplitOptions.RemoveEmptyEntries); - - parentFolder.Clear(); - foreach (string pathPart in pathParts.Take(pathParts.Length - 1)) + if (this.ContainsParentDirectory(modifiedPath)) { - parentFolder.Append(pathPart + "/"); - if (this.modifiedPaths.Contains(parentFolder.ToString())) - { - this.modifiedPaths.TryRemove(modifiedPath); - break; - } + this.modifiedPaths.TryRemove(modifiedPath); } } From ec71c1454ecc0522d9e23573b79793e37e31b959 Mon Sep 17 00:00:00 2001 From: William Baker Date: Mon, 1 Oct 2018 16:53:53 -0700 Subject: [PATCH 191/272] Windows: Use a faster comparison when enumeration filter strings don't have wildcards --- .../ActiveEnumeration.cs | 33 ++++++++++++++++--- .../WindowsFileSystemVirtualizer.cs | 4 +-- .../Virtualization/ActiveEnumerationTests.cs | 2 +- 3 files changed, 31 insertions(+), 8 deletions(-) diff --git a/GVFS/GVFS.Platform.Windows/ActiveEnumeration.cs b/GVFS/GVFS.Platform.Windows/ActiveEnumeration.cs index 6caa10785..55cb7c0ff 100644 --- a/GVFS/GVFS.Platform.Windows/ActiveEnumeration.cs +++ b/GVFS/GVFS.Platform.Windows/ActiveEnumeration.cs @@ -6,10 +6,11 @@ namespace GVFS.Platform.Windows { public class ActiveEnumeration { - private static FileNamePatternMatcher doesPatternMatch = null; + private static FileNamePatternMatcher wildcardPatternMatcher = null; // Use our own enumerator to avoid having to dispose anything private ProjectedFileInfoEnumerator fileInfoEnumerator; + private FileNamePatternMatcher patternMatcher; private string filterString = null; @@ -36,12 +37,13 @@ public ProjectedFileInfo Current } /// - /// Sets the pattern matching delegate that will be used for file name comparisons + /// Sets the pattern matching delegate that will be used for file name comparisons when the filter + /// contains wildcards. /// /// FileNamePatternMatcher to be used by ActiveEnumeration - public static void SetPatternMatcher(FileNamePatternMatcher patternMatcher) + public static void SetWildcardPatternMatcher(FileNamePatternMatcher patternMatcher) { - doesPatternMatch = patternMatcher; + wildcardPatternMatcher = patternMatcher; } /// @@ -107,15 +109,31 @@ public string GetFilterString() return this.filterString; } + private static bool NameMatchsNoWildcardFilter(string name, string filter) + { + return string.Equals(name, filter, System.StringComparison.OrdinalIgnoreCase); + } + private void SaveFilter(string filter) { if (string.IsNullOrEmpty(filter)) { this.filterString = string.Empty; + this.patternMatcher = null; } else { this.filterString = filter; + + if (ProjFS.Utils.DoesNameContainWildCards(this.filterString)) + { + this.patternMatcher = wildcardPatternMatcher; + } + else + { + this.patternMatcher = NameMatchsNoWildcardFilter; + } + if (this.IsCurrentValid && this.IsCurrentHidden()) { this.MoveNext(); @@ -125,7 +143,12 @@ private void SaveFilter(string filter) private bool IsCurrentHidden() { - return !doesPatternMatch(this.Current.Name, this.GetFilterString()); + if (this.patternMatcher == null) + { + return false; + } + + return !this.patternMatcher(this.Current.Name, this.GetFilterString()); } private void ResetEnumerator() diff --git a/GVFS/GVFS.Platform.Windows/WindowsFileSystemVirtualizer.cs b/GVFS/GVFS.Platform.Windows/WindowsFileSystemVirtualizer.cs index de1963b99..a5d02a6f3 100644 --- a/GVFS/GVFS.Platform.Windows/WindowsFileSystemVirtualizer.cs +++ b/GVFS/GVFS.Platform.Windows/WindowsFileSystemVirtualizer.cs @@ -272,11 +272,11 @@ private void InitializeEnumerationPatternMatcher() if (projFSPatternMatchingWorks) { - ActiveEnumeration.SetPatternMatcher(Utils.IsFileNameMatch); + ActiveEnumeration.SetWildcardPatternMatcher(Utils.IsFileNameMatch); } else { - ActiveEnumeration.SetPatternMatcher(InternalFileNameMatchesFilter); + ActiveEnumeration.SetWildcardPatternMatcher(InternalFileNameMatchesFilter); } this.Context.Tracer.RelatedEvent( diff --git a/GVFS/GVFS.UnitTests.Windows/Windows/Virtualization/ActiveEnumerationTests.cs b/GVFS/GVFS.UnitTests.Windows/Windows/Virtualization/ActiveEnumerationTests.cs index f74f42815..28e7de98f 100644 --- a/GVFS/GVFS.UnitTests.Windows/Windows/Virtualization/ActiveEnumerationTests.cs +++ b/GVFS/GVFS.UnitTests.Windows/Windows/Virtualization/ActiveEnumerationTests.cs @@ -23,7 +23,7 @@ public class ActiveEnumerationTests public ActiveEnumerationTests(PatternMatcherWrapper wrapper) { - ActiveEnumeration.SetPatternMatcher(wrapper.Matcher); + ActiveEnumeration.SetWildcardPatternMatcher(wrapper.Matcher); } public static object[] Runners From 47ad31a761115e5aee7b3e5263f57f0f5150783b Mon Sep 17 00:00:00 2001 From: William Baker Date: Thu, 27 Sep 2018 15:18:37 -0700 Subject: [PATCH 192/272] Mac: Update MirrorProvider to support symbolic links --- .../MacFileSystemVirtualizer.cs | 63 +++++++++- .../MirrorProvider/FileSystemVirtualizer.cs | 117 ++++++++++++------ .../MirrorProvider/ProjectedFileInfo.cs | 17 ++- 3 files changed, 157 insertions(+), 40 deletions(-) diff --git a/MirrorProvider/MirrorProvider.Mac/MacFileSystemVirtualizer.cs b/MirrorProvider/MirrorProvider.Mac/MacFileSystemVirtualizer.cs index 3d9396c82..b8df16789 100644 --- a/MirrorProvider/MirrorProvider.Mac/MacFileSystemVirtualizer.cs +++ b/MirrorProvider/MirrorProvider.Mac/MacFileSystemVirtualizer.cs @@ -1,7 +1,9 @@ using PrjFSLib.Mac; using System; using System.IO; - +using System.Runtime.InteropServices; +using System.Text; + namespace MirrorProvider.Mac { public class MacFileSystemVirtualizer : FileSystemVirtualizer @@ -58,7 +60,7 @@ private Result OnEnumerateDirectory( foreach (ProjectedFileInfo child in this.GetChildItems(relativePath)) { - if (child.IsDirectory) + if (child.Type == ProjectedFileInfo.FileType.Directory) { Result result = this.virtualizationInstance.WritePlaceholderDirectory( Path.Combine(relativePath, child.Name)); @@ -69,6 +71,28 @@ private Result OnEnumerateDirectory( return result; } } + else if (child.Type == ProjectedFileInfo.FileType.SymLink) + { + string childRelativePath = Path.Combine(relativePath, child.Name); + + string symLinkTarget; + if (this.TryGetSymLinkTarget(childRelativePath, out symLinkTarget)) + { + Result result = this.virtualizationInstance.WriteSymLink( + childRelativePath, + symLinkTarget); + + if (result != Result.Success) + { + Console.WriteLine($"WriteSymLink failed: {result}"); + return result; + } + } + else + { + return Result.EIOError; + } + } else { // The MirrorProvider marks every file as executable (mode 755), but this is just a shortcut to avoid the pain of @@ -176,6 +200,35 @@ private void OnHardLinkCreated(string relativeNewLinkPath) { Console.WriteLine($"OnHardLinkCreated: {relativeNewLinkPath}"); } + + private bool TryGetSymLinkTarget(string relativePath, out string symLinkTarget) + { + symLinkTarget = null; + string fullPathInMirror = this.GetFullPathInMirror(relativePath); + + const ulong BufSize = 4096; + byte[] targetBuffer = new byte[BufSize]; + long bytesRead = ReadLink(fullPathInMirror, targetBuffer, BufSize); + if (bytesRead < 0) + { + Console.WriteLine($"GetSymLinkTarget failed: {Marshal.GetLastWin32Error()}"); + return false; + } + + targetBuffer[bytesRead] = 0; + symLinkTarget = Encoding.UTF8.GetString(targetBuffer); + + if (symLinkTarget.StartsWith(this.Enlistment.MirrorRoot, StringComparison.OrdinalIgnoreCase)) + { + // Link target is an absolute path inside the MirrorRoot. + // The target needs to be adjusted to point inside the src root + symLinkTarget = Path.Combine( + this.Enlistment.SrcRoot.TrimEnd(Path.DirectorySeparatorChar), + symLinkTarget.Substring(this.Enlistment.MirrorRoot.Length).TrimStart(Path.DirectorySeparatorChar)); + } + + return true; + } private static byte[] ToVersionIdByteArray(byte version) { @@ -184,5 +237,11 @@ private static byte[] ToVersionIdByteArray(byte version) return bytes; } + + [DllImport("libc", EntryPoint = "readlink", SetLastError = true)] + private static extern long ReadLink( + string path, + byte[] buf, + ulong bufsize); } } diff --git a/MirrorProvider/MirrorProvider/FileSystemVirtualizer.cs b/MirrorProvider/MirrorProvider/FileSystemVirtualizer.cs index 40efddbe9..cb73aa0b0 100644 --- a/MirrorProvider/MirrorProvider/FileSystemVirtualizer.cs +++ b/MirrorProvider/MirrorProvider/FileSystemVirtualizer.cs @@ -1,25 +1,30 @@ using System; using System.Collections.Generic; using System.IO; -using System.Linq; - +using System.Linq; + namespace MirrorProvider { public abstract class FileSystemVirtualizer { - private Enlistment enlistment; + protected Enlistment Enlistment { get; private set; } public abstract bool TryConvertVirtualizationRoot(string directory, out string error); public virtual bool TryStartVirtualizationInstance(Enlistment enlistment, out string error) { - this.enlistment = enlistment; + this.Enlistment = enlistment; error = null; return true; } + protected string GetFullPathInMirror(string relativePath) + { + return Path.Combine(this.Enlistment.MirrorRoot, relativePath); + } + protected bool DirectoryExists(string relativePath) { - string fullPathInMirror = Path.Combine(this.enlistment.MirrorRoot, relativePath); + string fullPathInMirror = this.GetFullPathInMirror(relativePath); DirectoryInfo dirInfo = new DirectoryInfo(fullPathInMirror); return dirInfo.Exists; @@ -27,7 +32,7 @@ protected bool DirectoryExists(string relativePath) protected bool FileExists(string relativePath) { - string fullPathInMirror = Path.Combine(this.enlistment.MirrorRoot, relativePath); + string fullPathInMirror = this.GetFullPathInMirror(relativePath); FileInfo fileInfo = new FileInfo(fullPathInMirror); return fileInfo.Exists; @@ -35,18 +40,18 @@ protected bool FileExists(string relativePath) protected ProjectedFileInfo GetFileInfo(string relativePath) { - string fullPathInMirror = Path.Combine(this.enlistment.MirrorRoot, relativePath); + string fullPathInMirror = this.GetFullPathInMirror(relativePath); string fullParentPath = Path.GetDirectoryName(fullPathInMirror); string fileName = Path.GetFileName(relativePath); string actualCaseName; - if (this.DirectoryExists(fullParentPath, fileName, out actualCaseName)) + ProjectedFileInfo.FileType type; + if (this.FileOrDirectoryExists(fullParentPath, fileName, out actualCaseName, out type)) { - return new ProjectedFileInfo(actualCaseName, size: 0, isDirectory: true); - } - else if (this.FileExists(fullParentPath, fileName, out actualCaseName)) - { - return new ProjectedFileInfo(actualCaseName, size: new FileInfo(fullPathInMirror).Length, isDirectory: false); + return new ProjectedFileInfo( + actualCaseName, + size: (type == ProjectedFileInfo.FileType.File) ? new FileInfo(fullPathInMirror).Length : 0, + type: type); } return null; @@ -54,7 +59,7 @@ protected ProjectedFileInfo GetFileInfo(string relativePath) protected IEnumerable GetChildItems(string relativePath) { - string fullPathInMirror = Path.Combine(this.enlistment.MirrorRoot, relativePath); + string fullPathInMirror = this.GetFullPathInMirror(relativePath); DirectoryInfo dirInfo = new DirectoryInfo(fullPathInMirror); if (!dirInfo.Exists) @@ -62,20 +67,38 @@ protected IEnumerable GetChildItems(string relativePath) yield break; } - foreach (FileInfo file in dirInfo.GetFiles()) + foreach (FileSystemInfo fileSystemInfo in dirInfo.GetFileSystemInfos()) { - yield return new ProjectedFileInfo(file.Name, file.Length, isDirectory: false); - } + if ((fileSystemInfo.Attributes & FileAttributes.ReparsePoint) == FileAttributes.ReparsePoint) + { + // While not 100% accurate on all platforms, for simplicity assume that if the the file has reparse data it's a symlink + yield return new ProjectedFileInfo( + fileSystemInfo.Name, + size: 0, + type: ProjectedFileInfo.FileType.SymLink); + } + else if ((fileSystemInfo.Attributes & FileAttributes.Directory) == FileAttributes.Directory) + { + yield return new ProjectedFileInfo( + fileSystemInfo.Name, + size: 0, + type: ProjectedFileInfo.FileType.Directory); + } + else + { + FileInfo fileInfo = fileSystemInfo as FileInfo; + yield return new ProjectedFileInfo( + fileInfo.Name, + fileInfo.Length, + ProjectedFileInfo.FileType.File); + } - foreach (DirectoryInfo subDirectory in dirInfo.GetDirectories()) - { - yield return new ProjectedFileInfo(subDirectory.Name, size: 0, isDirectory: true); } } protected FileSystemResult HydrateFile(string relativePath, int bufferSize, Func tryWriteBytes) { - string fullPathInMirror = Path.Combine(this.enlistment.MirrorRoot, relativePath); + string fullPathInMirror = this.GetFullPathInMirror(relativePath); if (!File.Exists(fullPathInMirror)) { return FileSystemResult.EFileNotFound; @@ -106,23 +129,47 @@ protected FileSystemResult HydrateFile(string relativePath, int bufferSize, Func return FileSystemResult.Success; } - private bool DirectoryExists(string fullParentPath, string directoryName, out string actualCaseName) + private bool FileOrDirectoryExists( + string fullParentPath, + string fileName, + out string actualCaseName, + out ProjectedFileInfo.FileType type) { - return this.NameExists(Directory.GetDirectories(fullParentPath), directoryName, out actualCaseName); - } + actualCaseName = null; + type = ProjectedFileInfo.FileType.Invalid; - private bool FileExists(string fullParentPath, string fileName, out string actualCaseName) - { - return this.NameExists(Directory.GetFiles(fullParentPath), fileName, out actualCaseName); - } + DirectoryInfo dirInfo = new DirectoryInfo(fullParentPath); + if (!dirInfo.Exists) + { + return false; + } - private bool NameExists(IEnumerable paths, string name, out string actualCaseName) - { - actualCaseName = - paths - .Select(path => Path.GetFileName(path)) - .FirstOrDefault(actualName => actualName.Equals(name, StringComparison.OrdinalIgnoreCase)); - return actualCaseName != null; + FileSystemInfo fileSystemInfo = + dirInfo + .GetFileSystemInfos() + .FirstOrDefault(fsInfo => fsInfo.Name.Equals(fileName, StringComparison.OrdinalIgnoreCase)); + + if (fileSystemInfo == null) + { + return false; + } + + actualCaseName = fileSystemInfo.Name; + + if ((fileSystemInfo.Attributes & FileAttributes.ReparsePoint) == FileAttributes.ReparsePoint) + { + type = ProjectedFileInfo.FileType.SymLink; + } + else if ((fileSystemInfo.Attributes & FileAttributes.Directory) == FileAttributes.Directory) + { + type = ProjectedFileInfo.FileType.Directory; + } + else + { + type = ProjectedFileInfo.FileType.File; + } + + return true; } } } diff --git a/MirrorProvider/MirrorProvider/ProjectedFileInfo.cs b/MirrorProvider/MirrorProvider/ProjectedFileInfo.cs index 310447b69..71269ad2e 100644 --- a/MirrorProvider/MirrorProvider/ProjectedFileInfo.cs +++ b/MirrorProvider/MirrorProvider/ProjectedFileInfo.cs @@ -2,15 +2,26 @@ { public class ProjectedFileInfo { - public ProjectedFileInfo(string name, long size, bool isDirectory) + public ProjectedFileInfo(string name, long size, FileType type) { this.Name = name; this.Size = size; - this.IsDirectory = isDirectory; + this.Type = type; + } + + public enum FileType + { + Invalid, + + File, + Directory, + SymLink + } public string Name { get; } public long Size { get; } - public bool IsDirectory { get; } + public FileType Type { get; } + public bool IsDirectory => this.Type == FileType.Directory; } } From 355ead878bac3fade5f213aadf239332ff396224 Mon Sep 17 00:00:00 2001 From: William Baker Date: Tue, 2 Oct 2018 13:00:05 -0700 Subject: [PATCH 193/272] Variable renaming --- .../GVFS.Platform.Windows/ActiveEnumeration.cs | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/GVFS/GVFS.Platform.Windows/ActiveEnumeration.cs b/GVFS/GVFS.Platform.Windows/ActiveEnumeration.cs index 55cb7c0ff..675c6d1bc 100644 --- a/GVFS/GVFS.Platform.Windows/ActiveEnumeration.cs +++ b/GVFS/GVFS.Platform.Windows/ActiveEnumeration.cs @@ -6,11 +6,11 @@ namespace GVFS.Platform.Windows { public class ActiveEnumeration { - private static FileNamePatternMatcher wildcardPatternMatcher = null; + private static FileNamePatternMatcher doesWildcardPatternMatch = null; // Use our own enumerator to avoid having to dispose anything private ProjectedFileInfoEnumerator fileInfoEnumerator; - private FileNamePatternMatcher patternMatcher; + private FileNamePatternMatcher doesPatternMatch; private string filterString = null; @@ -43,7 +43,7 @@ public ProjectedFileInfo Current /// FileNamePatternMatcher to be used by ActiveEnumeration public static void SetWildcardPatternMatcher(FileNamePatternMatcher patternMatcher) { - wildcardPatternMatcher = patternMatcher; + doesWildcardPatternMatch = patternMatcher; } /// @@ -109,7 +109,7 @@ public string GetFilterString() return this.filterString; } - private static bool NameMatchsNoWildcardFilter(string name, string filter) + private static bool NameMatchesNoWildcardFilter(string name, string filter) { return string.Equals(name, filter, System.StringComparison.OrdinalIgnoreCase); } @@ -119,7 +119,7 @@ private void SaveFilter(string filter) if (string.IsNullOrEmpty(filter)) { this.filterString = string.Empty; - this.patternMatcher = null; + this.doesPatternMatch = null; } else { @@ -127,11 +127,11 @@ private void SaveFilter(string filter) if (ProjFS.Utils.DoesNameContainWildCards(this.filterString)) { - this.patternMatcher = wildcardPatternMatcher; + this.doesPatternMatch = doesWildcardPatternMatch; } else { - this.patternMatcher = NameMatchsNoWildcardFilter; + this.doesPatternMatch = NameMatchesNoWildcardFilter; } if (this.IsCurrentValid && this.IsCurrentHidden()) @@ -143,12 +143,12 @@ private void SaveFilter(string filter) private bool IsCurrentHidden() { - if (this.patternMatcher == null) + if (this.doesPatternMatch == null) { return false; } - return !this.patternMatcher(this.Current.Name, this.GetFilterString()); + return !this.doesPatternMatch(this.Current.Name, this.GetFilterString()); } private void ResetEnumerator() From f97b5cdcacfb6ea019ba092a532a619fd50c37da Mon Sep 17 00:00:00 2001 From: Derrick Stolee Date: Mon, 1 Oct 2018 15:42:21 -0400 Subject: [PATCH 194/272] SharedCacheTests: add test for multi-enlistment checkout failures --- .../MultiEnlistmentTests/SharedCacheTests.cs | 27 +++++++++++++++++-- GVFS/GVFS/CommandLine/CloneVerb.cs | 4 +-- GVFS/GVFS/CommandLine/GVFSVerb.cs | 4 +-- 3 files changed, 29 insertions(+), 6 deletions(-) diff --git a/GVFS/GVFS.FunctionalTests/Tests/MultiEnlistmentTests/SharedCacheTests.cs b/GVFS/GVFS.FunctionalTests/Tests/MultiEnlistmentTests/SharedCacheTests.cs index de37e9070..2a55c4c5e 100644 --- a/GVFS/GVFS.FunctionalTests/Tests/MultiEnlistmentTests/SharedCacheTests.cs +++ b/GVFS/GVFS.FunctionalTests/Tests/MultiEnlistmentTests/SharedCacheTests.cs @@ -96,7 +96,7 @@ public void CloneCleansUpStaleMetadataLock() enlistment1.Status().ShouldContain("Mount status: Ready"); enlistment2.Status().ShouldContain("Mount status: Ready"); } - + [TestCase] public void ParallelReadsInASharedCache() { @@ -265,7 +265,30 @@ public void MountUsesNewLocalCacheKeyWhenLocalCacheDeleted() newObjectsRoot.ShouldContain(newCacheKey); newObjectsRoot.ShouldBeADirectory(this.fileSystem); - this.AlternatesFileShouldHaveGitObjectsRoot(enlistment); + this.AlternatesFileShouldHaveGitObjectsRoot(enlistment); + } + + [TestCase] + public void SecondCloneSucceedsWithMissingTrees() + { + string newCachePath = Path.Combine(this.localCacheParentPath, ".customGvfsCache2"); + GVFSFunctionalTestEnlistment enlistment1 = this.CreateNewEnlistment(localCacheRoot: newCachePath); + File.ReadAllText(Path.Combine(enlistment1.RepoRoot, WellKnownFile)); + + this.AlternatesFileShouldHaveGitObjectsRoot(enlistment1); + + string packDir = Path.Combine(enlistment1.GetObjectRoot(this.fileSystem), "pack"); + string[] packDirFiles = Directory.EnumerateFiles(packDir, "*", SearchOption.AllDirectories).ToArray(); + for (int i = 0; i < packDirFiles.Length; i++) + { + this.fileSystem.DeleteFile(Path.Combine(packDir, packDirFiles[i])); + } + + // Enumerate the root directory, populating the tip commit and root tree + this.fileSystem.EnumerateDirectory(enlistment1.RepoRoot); + + GVFSFunctionalTestEnlistment enlistment2 = this.CreateNewEnlistment(localCacheRoot: newCachePath); + this.fileSystem.EnumerateDirectory(enlistment2.RepoRoot); } // Override OnTearDownEnlistmentsDeleted rathern than using [TearDown] as the enlistments need to be unmounted before diff --git a/GVFS/GVFS/CommandLine/CloneVerb.cs b/GVFS/GVFS/CommandLine/CloneVerb.cs index 9078e688c..ef59eea86 100644 --- a/GVFS/GVFS/CommandLine/CloneVerb.cs +++ b/GVFS/GVFS/CommandLine/CloneVerb.cs @@ -572,7 +572,7 @@ private Result CreateClone( // It is possible to have the above TryDownloadCommit() fail because we // already have the commit and root tree we intend to check out, but // don't have a tree further down the working directory. If we fail - // checkout here, it may be due to not having these trees and the + // checkout here, its' because we don't have these trees and the // read-object hook is not available yet. Force downloading the commit // again and retry the checkout. @@ -583,7 +583,7 @@ private Result CreateClone( gitObjects, gitRepo, out errorMessage, - ignoreIfRootExists: false)) + checkLocalObjectCache: false)) { return new Result(errorMessage); } diff --git a/GVFS/GVFS/CommandLine/GVFSVerb.cs b/GVFS/GVFS/CommandLine/GVFSVerb.cs index 38ee3e1e6..cb69014d1 100644 --- a/GVFS/GVFS/CommandLine/GVFSVerb.cs +++ b/GVFS/GVFS/CommandLine/GVFSVerb.cs @@ -402,9 +402,9 @@ protected bool TryDownloadCommit( GVFSGitObjects gitObjects, GitRepo repo, out string error, - bool ignoreIfRootExists = true) + bool checkLocalObjectCache = true) { - if (!ignoreIfRootExists || !repo.CommitAndRootTreeExists(commitId)) + if (!checkLocalObjectCache || !repo.CommitAndRootTreeExists(commitId)) { if (!gitObjects.TryDownloadCommit(commitId)) { From 48f62a2535f5476a05b9b58013894281b0ac2560 Mon Sep 17 00:00:00 2001 From: Jessica Schumaker Date: Wed, 3 Oct 2018 12:18:20 -0400 Subject: [PATCH 195/272] Use correct repoUrl for credential This change allows us to reinitialize the GitProces with valid repository information information and pass it correctly to the credential call. --- .../Tests/EnlistmentPerTestCase/RepairTests.cs | 8 ++++++++ GVFS/GVFS/RepairJobs/GitConfigRepairJob.cs | 4 +++- 2 files changed, 11 insertions(+), 1 deletion(-) diff --git a/GVFS/GVFS.FunctionalTests/Tests/EnlistmentPerTestCase/RepairTests.cs b/GVFS/GVFS.FunctionalTests/Tests/EnlistmentPerTestCase/RepairTests.cs index 34da63c00..b53d2caf8 100644 --- a/GVFS/GVFS.FunctionalTests/Tests/EnlistmentPerTestCase/RepairTests.cs +++ b/GVFS/GVFS.FunctionalTests/Tests/EnlistmentPerTestCase/RepairTests.cs @@ -13,6 +13,14 @@ namespace GVFS.FunctionalTests.Tests.EnlistmentPerTestCase [Category(Categories.MacTODO.M4)] public class RepairTests : TestsWithEnlistmentPerTestCase { + [TestCase] + public void NoFixesNeeded() + { + this.Enlistment.UnmountGVFS(); + + this.Enlistment.Repair(); + } + [TestCase] public void FixesCorruptHeadSha() { diff --git a/GVFS/GVFS/RepairJobs/GitConfigRepairJob.cs b/GVFS/GVFS/RepairJobs/GitConfigRepairJob.cs index b8432ae19..58416e0cc 100644 --- a/GVFS/GVFS/RepairJobs/GitConfigRepairJob.cs +++ b/GVFS/GVFS/RepairJobs/GitConfigRepairJob.cs @@ -45,6 +45,7 @@ public override IssueType HasIssue(List messages) // At this point, we've confirmed that the repo url can be gotten, so we have to // reinitialize the GitProcess with a valid repo url for 'git credential fill' + string repoUrl = null; try { GVFSEnlistment enlistment = GVFSEnlistment.CreateFromDirectory( @@ -52,6 +53,7 @@ public override IssueType HasIssue(List messages) this.Enlistment.GitBinPath, this.Enlistment.GVFSHooksRoot); git = new GitProcess(enlistment); + repoUrl = enlistment.RepoUrl; } catch (InvalidRepoException) { @@ -61,7 +63,7 @@ public override IssueType HasIssue(List messages) string username; string password; - if (!git.TryGetCredentials(this.Tracer, this.Enlistment.RepoUrl, out username, out password)) + if (!git.TryGetCredentials(this.Tracer, repoUrl, out username, out password)) { messages.Add("Authentication failed. Run 'gvfs log' for more info."); messages.Add(".git\\config is valid and remote 'origin' is set, but may have a typo:"); From d2bbde892ea70b70c9e1b71fd9bebd15144df0e6 Mon Sep 17 00:00:00 2001 From: Derrick Stolee Date: Wed, 3 Oct 2018 13:04:08 -0400 Subject: [PATCH 196/272] Remove SharedCacheTests.SecondCloneSucceedsWithMissingTrees This test attempted to cover a rare scenario, but is flaky due to something holding a handle on the pack-files we are trying to delete. --- .../MultiEnlistmentTests/SharedCacheTests.cs | 23 ------------------- 1 file changed, 23 deletions(-) diff --git a/GVFS/GVFS.FunctionalTests/Tests/MultiEnlistmentTests/SharedCacheTests.cs b/GVFS/GVFS.FunctionalTests/Tests/MultiEnlistmentTests/SharedCacheTests.cs index 2a55c4c5e..7ec39205a 100644 --- a/GVFS/GVFS.FunctionalTests/Tests/MultiEnlistmentTests/SharedCacheTests.cs +++ b/GVFS/GVFS.FunctionalTests/Tests/MultiEnlistmentTests/SharedCacheTests.cs @@ -267,29 +267,6 @@ public void MountUsesNewLocalCacheKeyWhenLocalCacheDeleted() this.AlternatesFileShouldHaveGitObjectsRoot(enlistment); } - - [TestCase] - public void SecondCloneSucceedsWithMissingTrees() - { - string newCachePath = Path.Combine(this.localCacheParentPath, ".customGvfsCache2"); - GVFSFunctionalTestEnlistment enlistment1 = this.CreateNewEnlistment(localCacheRoot: newCachePath); - File.ReadAllText(Path.Combine(enlistment1.RepoRoot, WellKnownFile)); - - this.AlternatesFileShouldHaveGitObjectsRoot(enlistment1); - - string packDir = Path.Combine(enlistment1.GetObjectRoot(this.fileSystem), "pack"); - string[] packDirFiles = Directory.EnumerateFiles(packDir, "*", SearchOption.AllDirectories).ToArray(); - for (int i = 0; i < packDirFiles.Length; i++) - { - this.fileSystem.DeleteFile(Path.Combine(packDir, packDirFiles[i])); - } - - // Enumerate the root directory, populating the tip commit and root tree - this.fileSystem.EnumerateDirectory(enlistment1.RepoRoot); - - GVFSFunctionalTestEnlistment enlistment2 = this.CreateNewEnlistment(localCacheRoot: newCachePath); - this.fileSystem.EnumerateDirectory(enlistment2.RepoRoot); - } // Override OnTearDownEnlistmentsDeleted rathern than using [TearDown] as the enlistments need to be unmounted before // localCacheParentPath can be deleted (as the SQLite blob sizes database cannot be deleted while GVFS is mounted) From 0bf8b3f2709acceebe915fb0b1eebf1b639d77df Mon Sep 17 00:00:00 2001 From: Jessica Schumaker Date: Wed, 3 Oct 2018 15:06:02 -0400 Subject: [PATCH 197/272] Fix diagnose message for cache server If not cache server is present display "None" --- GVFS/GVFS.Common/GVFSEnlistment.cs | 3 +-- GVFS/GVFS/CommandLine/DiagnoseVerb.cs | 2 +- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/GVFS/GVFS.Common/GVFSEnlistment.cs b/GVFS/GVFS.Common/GVFSEnlistment.cs index 345bfd8d4..f897dbfa8 100644 --- a/GVFS/GVFS.Common/GVFSEnlistment.cs +++ b/GVFS/GVFS.Common/GVFSEnlistment.cs @@ -13,7 +13,6 @@ public partial class GVFSEnlistment : Enlistment public const string BlobSizesCacheName = "blobSizes"; private const string GitObjectCacheName = "gitObjects"; - private const string InvalidRepoUrl = "invalid://repoUrl"; private string gitVersion; private string gvfsVersion; @@ -90,7 +89,7 @@ public static GVFSEnlistment CreateWithoutRepoUrlFromDirectory(string directory, return null; } - return new GVFSEnlistment(enlistmentRoot, InvalidRepoUrl, gitBinRoot, gvfsHooksRoot); + return new GVFSEnlistment(enlistmentRoot, string.Empty, gitBinRoot, gvfsHooksRoot); } return null; diff --git a/GVFS/GVFS/CommandLine/DiagnoseVerb.cs b/GVFS/GVFS/CommandLine/DiagnoseVerb.cs index 138259e8c..1d4188340 100644 --- a/GVFS/GVFS/CommandLine/DiagnoseVerb.cs +++ b/GVFS/GVFS/CommandLine/DiagnoseVerb.cs @@ -53,7 +53,7 @@ protected override void Execute(GVFSEnlistment enlistment) this.WriteMessage(GVFSPlatform.Instance.GitInstallation.GetInstalledGitBinPath()); this.WriteMessage(string.Empty); this.WriteMessage("Enlistment root: " + enlistment.EnlistmentRoot); - this.WriteMessage("Cache Server: " + CacheServerResolver.GetUrlFromConfig(enlistment)); + this.WriteMessage("Cache Server: " + CacheServerResolver.GetCacheServerFromConfig(enlistment)); string localCacheRoot; string gitObjectsRoot; From e99c92f8fafa7d5a03e11fce1c480e7f2b04497d Mon Sep 17 00:00:00 2001 From: William Baker Date: Wed, 3 Oct 2018 12:32:50 -0700 Subject: [PATCH 198/272] Mac: Prevent double hydration and directory expansion --- ProjFS.Mac/PrjFSLib/PrjFSLib.cpp | 280 ++++++++++++++++++------------- 1 file changed, 168 insertions(+), 112 deletions(-) diff --git a/ProjFS.Mac/PrjFSLib/PrjFSLib.cpp b/ProjFS.Mac/PrjFSLib/PrjFSLib.cpp index 82e273fb4..a901cfed2 100644 --- a/ProjFS.Mac/PrjFSLib/PrjFSLib.cpp +++ b/ProjFS.Mac/PrjFSLib/PrjFSLib.cpp @@ -9,8 +9,9 @@ #include #include #include -#include +#include #include +#include #include #include #include @@ -35,14 +36,16 @@ using std::hex; using std::is_pod; using std::lock_guard; using std::make_pair; +using std::make_shared; +using std::map; using std::move; using std::mutex; using std::oct; using std::pair; using std::queue; using std::set; +using std::shared_ptr; using std::string; -using std::unordered_map; typedef lock_guard mutex_lock; @@ -69,7 +72,7 @@ static void CombinePaths(const char* root, const char* relative, char (&combined static errno_t SendKernelMessageResponse(uint64_t messageId, MessageType responseType); static errno_t RegisterVirtualizationRootPath(const char* path); -static void HandleKernelRequest(Message requestSpec, void* messageMemory); +static void HandleKernelRequest(void* messageMemory, uint32_t messageSize); static PrjFS_Result HandleEnumerateDirectoryRequest(const MessageHeader* request, const char* path); static PrjFS_Result HandleRecursivelyEnumerateDirectoryRequest(const MessageHeader* request, const char* path); static PrjFS_Result HandleHydrateFileRequest(const MessageHeader* request, const char* path); @@ -94,10 +97,27 @@ static PrjFS_Callbacks s_callbacks; static dispatch_queue_t s_messageQueueDispatchQueue; static dispatch_queue_t s_kernelRequestHandlingConcurrentQueue; -// Map of relative path -> set of pending message IDs for that path, plus mutex to protect it. -static unordered_map> s_PendingRequestMessageIDs; -static mutex s_PendingRequestMessageMutex; +struct CaseInsensitiveStringCompare +{ + bool operator() (const std::string& lhs, const std::string& rhs) const + { + return strcasecmp(lhs.c_str(), rhs.c_str()) < 0; + } +}; + +struct MutexAndUseCount +{ + shared_ptr mutex; + int useCount; +}; +// Map of relative path -> MutexAndUseCount for that path, plus mutex to protect the map itself. +typedef map PathToMutexMap; +PathToMutexMap s_inProgressExpansions; +mutex s_inProgressExpansionsMutex; + +static shared_ptr CheckoutPathMutex(const string& fullPath); +static void ReturnPathMutex(const string& fullPath, const shared_ptr& mutex); // The full API is defined in the header, but only the minimal set of functions needed // for the initial MirrorProvider implementation are listed here. Calling any other function @@ -192,37 +212,11 @@ PrjFS_Result PrjFS_StartVirtualizationInstance( cerr << "Unexpected result dequeueing message - result 0x" << hex << result << " dequeued " << dequeuedSize << "/" << messageSize << " bytes\n"; abort(); } - - Message message = ParseMessageMemory(messageMemory, messageSize); - - // At the moment, we expect all messages to include a path - assert(message.path != nullptr); - - // Ensure we don't run more than one request handler at once for the same file - { - mutex_lock lock(s_PendingRequestMessageMutex); - typedef unordered_map>::iterator PendingMessageIterator; - PendingMessageIterator file_messages_found = s_PendingRequestMessageIDs.find(message.path); - if (file_messages_found == s_PendingRequestMessageIDs.end()) - { - // Not handling this file/dir yet - pair inserted = - s_PendingRequestMessageIDs.insert(make_pair(string(message.path), set{ message.messageHeader->messageId })); - assert(inserted.second); - } - else - { - // Already a handler running for this path, don't handle it again. - file_messages_found->second.insert(message.messageHeader->messageId); - continue; - } - } - dispatch_async( s_kernelRequestHandlingConcurrentQueue, ^{ - HandleKernelRequest(message, messageMemory); + HandleKernelRequest(messageMemory, messageSize); }); } }); @@ -436,6 +430,7 @@ PrjFS_Result PrjFS_UpdatePlaceholderFileIfNeeded( return result; } + // TODO(Mac): Ensure that races with hydration are handled properly return PrjFS_WritePlaceholderFile(relativePath, providerId, contentId, fileSize, fileMode); } @@ -482,6 +477,7 @@ PrjFS_Result PrjFS_DeleteFile( return PrjFS_Result_EInvalidArgs; } + // TODO(Mac): Ensure that races with hydration are handled properly // TODO(Mac): Ensure file is not full before proceeding char fullPath[PrjFSMaxPath]; @@ -554,10 +550,15 @@ static Message ParseMessageMemory(const void* messageMemory, uint32_t size) return Message { header, path }; } -static void HandleKernelRequest(Message request, void* messageMemory) +static void HandleKernelRequest(void* messageMemory, uint32_t messageSize) { PrjFS_Result result = PrjFS_Result_EIOError; + Message request = ParseMessageMemory(messageMemory, messageSize); + + // At the moment, we expect all messages to include a path + assert(request.path != nullptr); + const MessageHeader* requestHeader = request.messageHeader; switch (requestHeader->messageType) { @@ -621,21 +622,8 @@ static void HandleKernelRequest(Message request, void* messageMemory) PrjFS_Result_Success == result ? MessageType_Response_Success : MessageType_Response_Fail; - - set messageIDs; - - { - mutex_lock lock(s_PendingRequestMessageMutex); - unordered_map>::iterator fileMessageIDsFound = s_PendingRequestMessageIDs.find(request.path); - assert(fileMessageIDsFound != s_PendingRequestMessageIDs.end()); - messageIDs = move(fileMessageIDsFound->second); - s_PendingRequestMessageIDs.erase(fileMessageIDsFound); - } - - for (uint64_t messageID : messageIDs) - { - SendKernelMessageResponse(messageID, responseType); - } + + SendKernelMessageResponse(requestHeader->messageId, responseType); } free(messageMemory); @@ -647,26 +635,44 @@ static PrjFS_Result HandleEnumerateDirectoryRequest(const MessageHeader* request cout << "PrjFSLib.HandleEnumerateDirectoryRequest: " << path << endl; #endif - PrjFS_Result callbackResult = s_callbacks.EnumerateDirectory( - 0 /* commandId */, - path, - request->pid, - request->procname); + char fullPath[PrjFSMaxPath]; + CombinePaths(s_virtualizationRootFullPath.c_str(), path, fullPath); + if (!IsBitSetInFileFlags(fullPath, FileFlags_IsEmpty)) + { + return PrjFS_Result_Success; + } - if (PrjFS_Result_Success == callbackResult) + PrjFS_Result result; + shared_ptr expansionMutex = CheckoutPathMutex(fullPath); { - char fullPath[PrjFSMaxPath]; - CombinePaths(s_virtualizationRootFullPath.c_str(), path, fullPath); + mutex_lock lock(*expansionMutex); + if (!IsBitSetInFileFlags(fullPath, FileFlags_IsEmpty)) + { + result = PrjFS_Result_Success; + goto CleanupAndReturn; + } + + result = s_callbacks.EnumerateDirectory( + 0 /* commandId */, + path, + request->pid, + request->procname); - if (!SetBitInFileFlags(fullPath, FileFlags_IsEmpty, false)) + if (PrjFS_Result_Success == result) { - // TODO(Mac): how should we handle this scenario where the provider thinks it succeeded, but we were unable to - // update placeholder metadata? - return PrjFS_Result_EIOError; + if (!SetBitInFileFlags(fullPath, FileFlags_IsEmpty, false)) + { + // TODO(Mac): how should we handle this scenario where the provider thinks it succeeded, but we were unable to + // update placeholder metadata? + result = PrjFS_Result_EIOError; + } } } - - return callbackResult; + +CleanupAndReturn: + ReturnPathMutex(fullPath, expansionMutex); + + return result; } static PrjFS_Result HandleRecursivelyEnumerateDirectoryRequest(const MessageHeader* request, const char* path) @@ -689,13 +695,10 @@ static PrjFS_Result HandleRecursivelyEnumerateDirectoryRequest(const MessageHead CombinePaths(s_virtualizationRootFullPath.c_str(), directoryRelativePath.c_str(), pathBuffer); - if (IsBitSetInFileFlags(pathBuffer, FileFlags_IsEmpty)) + PrjFS_Result result = HandleEnumerateDirectoryRequest(request, directoryRelativePath.c_str()); + if (result != PrjFS_Result_Success) { - PrjFS_Result result = HandleEnumerateDirectoryRequest(request, directoryRelativePath.c_str()); - if (result != PrjFS_Result_Success) - { - goto CleanupAndReturn; - } + goto CleanupAndReturn; } DIR* directory = opendir(pathBuffer); @@ -744,59 +747,81 @@ static PrjFS_Result HandleHydrateFileRequest(const MessageHeader* request, const return PrjFS_Result_EIOError; } - PrjFS_FileHandle fileHandle; - - // Mode "rb+" means: - // - The file must already exist - // - The handle is opened for reading and writing - // - We are allowed to seek to somewhere other than end of stream for writing - fileHandle.file = fopen(fullPath, "rb+"); - if (nullptr == fileHandle.file) - { - return PrjFS_Result_EIOError; - } - - // Seek back to the beginning so the provider can overwrite the empty contents - if (fseek(fileHandle.file, 0, 0)) + if (!IsBitSetInFileFlags(fullPath, FileFlags_IsEmpty)) { - fclose(fileHandle.file); - return PrjFS_Result_EIOError; + return PrjFS_Result_Success; } - PrjFS_Result callbackResult = s_callbacks.GetFileStream( - 0 /* comandId */, - path, - xattrData.providerId, - xattrData.contentId, - request->pid, - request->procname, - &fileHandle); - - // TODO: once we support async callbacks, we'll need to save off the fileHandle if the result is Pending + PrjFS_Result result; + PrjFS_FileHandle fileHandle; - if (fclose(fileHandle.file)) - { - // TODO: under what conditions can fclose fail? How do we recover? - return PrjFS_Result_EIOError; - } + shared_ptr hydrationMutex = CheckoutPathMutex(fullPath); - if (PrjFS_Result_Success == callbackResult) { - // TODO: validate that the total bytes written match the size that was reported on the placeholder in the first place - // Potential bugs if we don't: - // * The provider writes fewer bytes than expected. The hydrated is left with extra padding up to the original reported size. - // * The provider writes more bytes than expected. The write succeeds, but whatever tool originally opened the file may have already - // allocated the originally reported size, and now the contents appear truncated. + mutex_lock lock(*hydrationMutex); + if (!IsBitSetInFileFlags(fullPath, FileFlags_IsEmpty)) + { + result = PrjFS_Result_Success; + goto CleanupAndReturn; + } - if (!SetBitInFileFlags(fullPath, FileFlags_IsEmpty, false)) + // Mode "rb+" means: + // - The file must already exist + // - The handle is opened for reading and writing + // - We are allowed to seek to somewhere other than end of stream for writing + fileHandle.file = fopen(fullPath, "rb+"); + if (nullptr == fileHandle.file) { - // TODO: how should we handle this scenario where the provider thinks it succeeded, but we were unable to - // update placeholder metadata? - return PrjFS_Result_EIOError; + result = PrjFS_Result_EIOError; + goto CleanupAndReturn; + } + + // Seek back to the beginning so the provider can overwrite the empty contents + if (fseek(fileHandle.file, 0, 0)) + { + fclose(fileHandle.file); + result = PrjFS_Result_EIOError; + goto CleanupAndReturn; + } + + result = s_callbacks.GetFileStream( + 0 /* comandId */, + path, + xattrData.providerId, + xattrData.contentId, + request->pid, + request->procname, + &fileHandle); + + // TODO: once we support async callbacks, we'll need to save off the fileHandle if the result is Pending + + if (fclose(fileHandle.file)) + { + // TODO: under what conditions can fclose fail? How do we recover? + result = PrjFS_Result_EIOError; + goto CleanupAndReturn; + } + + if (PrjFS_Result_Success == result) + { + // TODO: validate that the total bytes written match the size that was reported on the placeholder in the first place + // Potential bugs if we don't: + // * The provider writes fewer bytes than expected. The hydrated is left with extra padding up to the original reported size. + // * The provider writes more bytes than expected. The write succeeds, but whatever tool originally opened the file may have already + // allocated the originally reported size, and now the contents appear truncated. + + if (!SetBitInFileFlags(fullPath, FileFlags_IsEmpty, false)) + { + // TODO: how should we handle this scenario where the provider thinks it succeeded, but we were unable to + // update placeholder metadata? + result = PrjFS_Result_EIOError; + } } } - - return callbackResult; + +CleanupAndReturn: + ReturnPathMutex(fullPath, hydrationMutex); + return result; } static PrjFS_Result HandleFileNotification( @@ -1031,3 +1056,34 @@ static const char* NotificationTypeToString(PrjFS_NotificationType notificationT } } #endif + +static shared_ptr CheckoutPathMutex(const string& fullPath) +{ + mutex_lock lock(s_inProgressExpansionsMutex); + PathToMutexMap::iterator iter = s_inProgressExpansions.find(fullPath); + if (iter == s_inProgressExpansions.end()) + { + pair newEntry = s_inProgressExpansions.insert( + PathToMutexMap::value_type(fullPath, { make_shared(), 1 })); + assert(newEntry.second); + return newEntry.first->second.mutex; + } + else + { + iter->second.useCount++; + return iter->second.mutex; + } +} + +static void ReturnPathMutex(const string& fullPath, const shared_ptr& mutex) +{ + mutex_lock lock(s_inProgressExpansionsMutex); + PathToMutexMap::iterator iter = s_inProgressExpansions.find(fullPath); + assert(iter != s_inProgressExpansions.end()); + assert(iter->second.mutex.get() == mutex.get()); + iter->second.useCount--; + if (iter->second.useCount == 0) + { + s_inProgressExpansions.erase(iter); + } +} From c226d42e03c47cba09ebc49e64af83f3777c8c62 Mon Sep 17 00:00:00 2001 From: Al Ameen Shah Date: Tue, 18 Sep 2018 12:01:29 -0400 Subject: [PATCH 199/272] 1. GVFS.Service checks for upgrade availability periodically. When an new release is available it will be downloaded in the background. 2. GVFS Git Hooks will nag user to install the upgrade. 3. GVFS Upgrade [--confirm] verb will enable user to manually trigger installation of the upgrade release. 4. Unit test cases 5. Functional test for upgrade reminder messaging --- GVFS.sln | 10 + GVFS/GVFS.Common/DiskLayoutUpgrade.cs | 2 +- GVFS/GVFS.Common/FileBasedLock.cs | 2 +- GVFS/GVFS.Common/FileSystem/IKernelDriver.cs | 1 + GVFS/GVFS.Common/GVFSConstants.cs | 16 +- GVFS/GVFS.Common/GVFSPlatform.cs | 7 +- GVFS/GVFS.Common/Git/GitProcess.cs | 26 + GVFS/GVFS.Common/Git/GitVersion.cs | 17 +- GVFS/GVFS.Common/InstallerPreRunChecker.cs | 212 ++++++++ GVFS/GVFS.Common/ProcessHelper.cs | 9 +- GVFS/GVFS.Common/ProductUpgrader.Shared.cs | 69 +++ GVFS/GVFS.Common/ProductUpgrader.cs | 477 ++++++++++++++++++ .../GVFSUpgradeReminderTests.cs | 88 ++++ GVFS/GVFS.Hooks/GVFS.Hooks.Mac.csproj | 1 + GVFS/GVFS.Hooks/GVFS.Hooks.Windows.csproj | 5 +- GVFS/GVFS.Hooks/Program.cs | 8 +- GVFS/GVFS.Installer/Setup.iss | 19 +- GVFS/GVFS.Platform.Mac/MacPlatform.cs | 5 + GVFS/GVFS.Platform.Mac/ProjFSKext.cs | 8 +- GVFS/GVFS.Platform.Windows/ProjFSFilter.cs | 49 +- GVFS/GVFS.Platform.Windows/WindowsPlatform.cs | 9 + GVFS/GVFS.Service/GVFS.Service.csproj | 1 + GVFS/GVFS.Service/GvfsService.cs | 11 +- GVFS/GVFS.Service/ProductUpgradeTimer.cs | 83 +++ GVFS/GVFS.SignFiles/GVFS.SignFiles.csproj | 1 + .../GVFS.UnitTests.Windows.csproj | 12 + .../Mock/MockInstallerPreRunChecker.cs | 129 +++++ .../Windows/Mock/MockProcessLauncher.cs | 45 ++ .../Windows/Mock/MockProductUpgrader.cs | 241 +++++++++ .../Windows/Mock/MockTextWriter.cs | 47 ++ .../Windows/Upgrader/ProductUpgraderTests.cs | 135 +++++ .../Upgrader/UpgradeOrchestratorTests.cs | 250 +++++++++ .../Windows/Upgrader/UpgradeTests.cs | 128 +++++ .../Windows/Upgrader/UpgradeVerbTests.cs | 247 +++++++++ .../Mock/Common/MockPlatform.cs | 5 + GVFS/GVFS.UnitTests/Mock/Common/MockTracer.cs | 9 +- GVFS/GVFS.Upgrader/App.config | 6 + GVFS/GVFS.Upgrader/GVFS.Upgrader.csproj | 122 +++++ GVFS/GVFS.Upgrader/Program.cs | 16 + GVFS/GVFS.Upgrader/Properties/AssemblyInfo.cs | 22 + GVFS/GVFS.Upgrader/UpgradeOrchestrator.cs | 389 ++++++++++++++ GVFS/GVFS.Upgrader/packages.config | 6 + GVFS/GVFS/CommandLine/DiagnoseVerb.cs | 20 +- GVFS/GVFS/CommandLine/GVFSVerb.cs | 5 +- GVFS/GVFS/CommandLine/LogVerb.cs | 5 +- GVFS/GVFS/CommandLine/UpgradeVerb.cs | 325 ++++++++++++ GVFS/GVFS/GVFS.Windows.csproj | 3 +- GVFS/GVFS/Program.cs | 13 +- Scripts/UninstallGVFS.bat | 2 + 49 files changed, 3281 insertions(+), 37 deletions(-) create mode 100644 GVFS/GVFS.Common/InstallerPreRunChecker.cs create mode 100644 GVFS/GVFS.Common/ProductUpgrader.Shared.cs create mode 100644 GVFS/GVFS.Common/ProductUpgrader.cs create mode 100644 GVFS/GVFS.FunctionalTests/Tests/EnlistmentPerFixture/GVFSUpgradeReminderTests.cs create mode 100644 GVFS/GVFS.Service/ProductUpgradeTimer.cs create mode 100644 GVFS/GVFS.UnitTests.Windows/Windows/Mock/MockInstallerPreRunChecker.cs create mode 100644 GVFS/GVFS.UnitTests.Windows/Windows/Mock/MockProcessLauncher.cs create mode 100644 GVFS/GVFS.UnitTests.Windows/Windows/Mock/MockProductUpgrader.cs create mode 100644 GVFS/GVFS.UnitTests.Windows/Windows/Mock/MockTextWriter.cs create mode 100644 GVFS/GVFS.UnitTests.Windows/Windows/Upgrader/ProductUpgraderTests.cs create mode 100644 GVFS/GVFS.UnitTests.Windows/Windows/Upgrader/UpgradeOrchestratorTests.cs create mode 100644 GVFS/GVFS.UnitTests.Windows/Windows/Upgrader/UpgradeTests.cs create mode 100644 GVFS/GVFS.UnitTests.Windows/Windows/Upgrader/UpgradeVerbTests.cs create mode 100644 GVFS/GVFS.Upgrader/App.config create mode 100644 GVFS/GVFS.Upgrader/GVFS.Upgrader.csproj create mode 100644 GVFS/GVFS.Upgrader/Program.cs create mode 100644 GVFS/GVFS.Upgrader/Properties/AssemblyInfo.cs create mode 100644 GVFS/GVFS.Upgrader/UpgradeOrchestrator.cs create mode 100644 GVFS/GVFS.Upgrader/packages.config create mode 100644 GVFS/GVFS/CommandLine/UpgradeVerb.cs diff --git a/GVFS.sln b/GVFS.sln index e5378e750..e0c6ba22d 100644 --- a/GVFS.sln +++ b/GVFS.sln @@ -125,6 +125,7 @@ Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "GVFS.Mac", "GVFS\GVFS\GVFS. EndProject Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "GVFS.Windows", "GVFS\GVFS\GVFS.Windows.csproj", "{32220664-594C-4425-B9A0-88E0BE2F3D2A}" ProjectSection(ProjectDependencies) = postProject + {AECEC217-2499-403D-B0BB-2962B9BE5970} = {AECEC217-2499-403D-B0BB-2962B9BE5970} {2D23AB54-541F-4ABC-8DCA-08C199E97ABB} = {2D23AB54-541F-4ABC-8DCA-08C199E97ABB} {798DE293-6EDA-4DC4-9395-BE7A71C563E3} = {798DE293-6EDA-4DC4-9395-BE7A71C563E3} {5A6656D5-81C7-472C-9DC8-32D071CB2258} = {5A6656D5-81C7-472C-9DC8-32D071CB2258} @@ -167,6 +168,8 @@ Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "GVFS.FunctionalTests.LockHo EndProject Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "GVFS.Hooks.Mac", "GVFS\GVFS.Hooks\GVFS.Hooks.Mac.csproj", "{4CC2A90D-D240-4382-B4BF-5E175515E492}" EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "GVFS.Upgrader", "GVFS\GVFS.Upgrader\GVFS.Upgrader.csproj", "{AECEC217-2499-403D-B0BB-2962B9BE5970}" +EndProject Global GlobalSection(SolutionConfigurationPlatforms) = preSolution Debug.Mac|x64 = Debug.Mac|x64 @@ -369,6 +372,12 @@ Global {4CC2A90D-D240-4382-B4BF-5E175515E492}.Release.Mac|x64.Build.0 = Release|x64 {4CC2A90D-D240-4382-B4BF-5E175515E492}.Release.Windows|x64.ActiveCfg = Release|x64 {4CC2A90D-D240-4382-B4BF-5E175515E492}.Release.Windows|x64.Build.0 = Release|x64 + {AECEC217-2499-403D-B0BB-2962B9BE5970}.Debug.Mac|x64.ActiveCfg = Debug|x64 + {AECEC217-2499-403D-B0BB-2962B9BE5970}.Debug.Windows|x64.ActiveCfg = Debug|x64 + {AECEC217-2499-403D-B0BB-2962B9BE5970}.Debug.Windows|x64.Build.0 = Debug|x64 + {AECEC217-2499-403D-B0BB-2962B9BE5970}.Release.Mac|x64.ActiveCfg = Release|x64 + {AECEC217-2499-403D-B0BB-2962B9BE5970}.Release.Windows|x64.ActiveCfg = Release|x64 + {AECEC217-2499-403D-B0BB-2962B9BE5970}.Release.Windows|x64.Build.0 = Release|x64 EndGlobalSection GlobalSection(SolutionProperties) = preSolution HideSolutionNode = FALSE @@ -403,6 +412,7 @@ Global {BD7C5776-82F2-40C6-AF01-B3CC1E2D83AF} = {C41F10F9-1163-4CFA-A465-EA728F75E9FA} {FA273F69-5762-43D8-AEA1-B4F08090D624} = {C41F10F9-1163-4CFA-A465-EA728F75E9FA} {4CC2A90D-D240-4382-B4BF-5E175515E492} = {2EF2EC94-3A68-4ED7-9A58-B7057ADBA01C} + {AECEC217-2499-403D-B0BB-2962B9BE5970} = {2EF2EC94-3A68-4ED7-9A58-B7057ADBA01C} EndGlobalSection GlobalSection(ExtensibilityGlobals) = postSolution SolutionGuid = {A025908B-DAB1-46CB-83A3-56F3B863D8FA} diff --git a/GVFS/GVFS.Common/DiskLayoutUpgrade.cs b/GVFS/GVFS.Common/DiskLayoutUpgrade.cs index f34e9ad86..5ec4115fb 100644 --- a/GVFS/GVFS.Common/DiskLayoutUpgrade.cs +++ b/GVFS/GVFS.Common/DiskLayoutUpgrade.cs @@ -327,7 +327,7 @@ private static void StartLogFile(string enlistmentRoot, JsonTracer tracer) tracer.AddLogFileEventListener( GVFSEnlistment.GetNewGVFSLogFileName( Path.Combine(enlistmentRoot, GVFSConstants.DotGVFS.LogPath), - GVFSConstants.LogFileTypes.Upgrade), + GVFSConstants.LogFileTypes.UpgradeDiskLayout), EventLevel.Informational, Keywords.Any); diff --git a/GVFS/GVFS.Common/FileBasedLock.cs b/GVFS/GVFS.Common/FileBasedLock.cs index 560ad091f..9f709f2cf 100644 --- a/GVFS/GVFS.Common/FileBasedLock.cs +++ b/GVFS/GVFS.Common/FileBasedLock.cs @@ -1,4 +1,4 @@ -using GVFS.Common.FileSystem; +using GVFS.Common.FileSystem; using GVFS.Common.Tracing; using System; diff --git a/GVFS/GVFS.Common/FileSystem/IKernelDriver.cs b/GVFS/GVFS.Common/FileSystem/IKernelDriver.cs index 35fbdd1c1..f223caf46 100644 --- a/GVFS/GVFS.Common/FileSystem/IKernelDriver.cs +++ b/GVFS/GVFS.Common/FileSystem/IKernelDriver.cs @@ -11,5 +11,6 @@ public interface IKernelDriver string FlushDriverLogs(); bool TryPrepareFolderForCallbacks(string folderPath, out string error, out Exception exception); bool IsReady(JsonTracer tracer, string enlistmentRoot, out string error); + bool IsGVFSUpgradeSupported(); } } diff --git a/GVFS/GVFS.Common/GVFSConstants.cs b/GVFS/GVFS.Common/GVFSConstants.cs index ece894ef8..a88200be6 100644 --- a/GVFS/GVFS.Common/GVFSConstants.cs +++ b/GVFS/GVFS.Common/GVFSConstants.cs @@ -1,4 +1,4 @@ -using System.IO; +using System.IO; namespace GVFS.Common { @@ -33,6 +33,7 @@ public static class GitConfig public const string HooksPrefix = GitConfig.GVFSPrefix + "clone.default-"; public const string GVFSTelemetryId = GitConfig.GVFSPrefix + "telemetry-id"; public const string HooksExtension = ".hooks"; + public const string UpgradeRing = GVFSPrefix + "upgradering"; } public static class GitStatusCache @@ -72,6 +73,7 @@ public static class SpecialGitFiles public static class LogFileTypes { public const string MountPrefix = "mount"; + public const string UpgradePrefix = "upgrade"; public const string Clone = "clone"; public const string Dehydrate = "dehydrate"; @@ -80,7 +82,9 @@ public static class LogFileTypes public const string Prefetch = "prefetch"; public const string Repair = "repair"; public const string Service = "service"; - public const string Upgrade = MountPrefix + "_upgrade"; + public const string UpgradeVerb = UpgradePrefix + "_verb"; + public const string UpgradeProcess = UpgradePrefix + "_process"; + public const string UpgradeDiskLayout = MountPrefix + "_upgrade"; } public static class DotGVFS @@ -215,5 +219,13 @@ public static class Unmount public const string SkipLock = "skip-wait-for-lock"; } } + + public static class UpgradeVerbMessages + { + public const string NoneRingConsoleAlert = "Upgrade ring set to None. No upgrade check was performed."; + public const string SetUpgradeRingCommand = "To set or change upgrade ring, run `git config --global gvfs.upgradering [\"Fast\"|\"Slow\"|\"None\"] from an elevated command prompt."; + public const string UpgradeAvailable = "A newer version of GVFS is available."; + public const string UpgradeInstallAdvice = "Run `gvfs upgrade --confirm` from an elevated command prompt to install."; + } } } diff --git a/GVFS/GVFS.Common/GVFSPlatform.cs b/GVFS/GVFS.Common/GVFSPlatform.cs index 2297a75d9..db124104b 100644 --- a/GVFS/GVFS.Common/GVFSPlatform.cs +++ b/GVFS/GVFS.Common/GVFSPlatform.cs @@ -37,7 +37,7 @@ public static void Register(GVFSPlatform platform) public abstract void StartBackgroundProcess(string programName, string[] args); public abstract bool IsProcessActive(int processId); - + public abstract void IsServiceInstalledAndRunning(string name, out bool installed, out bool running); public abstract string GetNamedPipeName(string enlistmentRoot); public abstract NamedPipeServerStream CreatePipeByName(string pipeName); @@ -111,6 +111,11 @@ public string MountExecutableName { get { return "GVFS.Mount" + this.ExecutableExtension; } } + + public string GVFSUpgraderExecutableName + { + get { return "GVFS.Upgrader" + this.ExecutableExtension; } + } } } } diff --git a/GVFS/GVFS.Common/Git/GitProcess.cs b/GVFS/GVFS.Common/Git/GitProcess.cs index 5c3bba3cd..1ce9236e1 100644 --- a/GVFS/GVFS.Common/Git/GitProcess.cs +++ b/GVFS/GVFS.Common/Git/GitProcess.cs @@ -92,6 +92,32 @@ public static Result GetFromSystemConfig(string gitBinPath, string settingName) return new GitProcess(gitBinPath, workingDirectoryRoot: null, gvfsHooksRoot: null).InvokeGitOutsideEnlistment("config --system " + settingName); } + public static bool TryGetVersion(out GitVersion gitVersion, out string error) + { + try + { + string gitPath = GVFSPlatform.Instance.GitInstallation.GetInstalledGitBinPath(); + GitProcess gitProcess = new GitProcess(gitPath, null, null); + Result result = gitProcess.InvokeGitOutsideEnlistment("--version"); + string version = result.Output; + + if (!GitVersion.TryParseGitVersionCommandResult(version, out gitVersion)) + { + error = "Unable to determine installed git version. " + version; + return false; + } + + error = null; + return true; + } + catch (NotSupportedException exception) + { + error = exception.ToString(); + gitVersion = null; + return false; + } + } + public virtual void RevokeCredential(string repoUrl) { this.InvokeGitOutsideEnlistment( diff --git a/GVFS/GVFS.Common/Git/GitVersion.cs b/GVFS/GVFS.Common/Git/GitVersion.cs index a9f1e4115..97a1bdebc 100644 --- a/GVFS/GVFS.Common/Git/GitVersion.cs +++ b/GVFS/GVFS.Common/Git/GitVersion.cs @@ -1,4 +1,4 @@ -using System; +using System; namespace GVFS.Common.Git { @@ -21,6 +21,21 @@ public GitVersion(int major, int minor, int build, string platform, int revision public int Revision { get; private set; } public int MinorRevision { get; private set; } + public static bool TryParseGitVersionCommandResult(string input, out GitVersion version) + { + // git version output is of the form + // git version 2.17.0.gvfs.1.preview.3 + + const string GitVersionExpectedPrefix = "git version "; + + if (input.StartsWith(GitVersionExpectedPrefix)) + { + input = input.Substring(GitVersionExpectedPrefix.Length); + } + + return TryParseVersion(input, out version); + } + public static bool TryParseInstallerName(string input, out GitVersion version) { // Installer name is of the form diff --git a/GVFS/GVFS.Common/InstallerPreRunChecker.cs b/GVFS/GVFS.Common/InstallerPreRunChecker.cs new file mode 100644 index 000000000..e92c12742 --- /dev/null +++ b/GVFS/GVFS.Common/InstallerPreRunChecker.cs @@ -0,0 +1,212 @@ +using GVFS.Common; +using GVFS.Common.Git; +using GVFS.Common.Tracing; +using System; +using System.Collections.Generic; +using System.Diagnostics; +using System.IO; +using System.Linq; +using System.Threading; + +namespace GVFS.Upgrader +{ + public class InstallerPreRunChecker + { + private static readonly HashSet BlockingProcessSet = new HashSet { "GVFS", "GVFS.Mount", "git", "ssh-agent", "bash", "wish", "git-bash" }; + + private ITracer tracer; + + public InstallerPreRunChecker(ITracer tracer) + { + this.tracer = tracer; + this.CommandToRerun = string.Empty; + } + + public string CommandToRerun { get; set; } + + public bool TryRunPreUpgradeChecks(out string consoleError) + { + this.tracer.RelatedInfo("Checking if GVFS upgrade can be run on this machine."); + + if (this.IsUnattended()) + { + consoleError = "`gvfs upgrade` is not supported in unattended mode"; + this.tracer.RelatedError($"{nameof(TryRunPreUpgradeChecks)}: {consoleError}"); + return false; + } + + if (this.IsDevelopmentVersion()) + { + consoleError = "Cannot run upgrade when development version of GVFS is installed."; + this.tracer.RelatedError($"{nameof(TryRunPreUpgradeChecks)}: {consoleError}"); + return false; + } + + if (!this.IsGVFSUpgradeAllowed(out consoleError)) + { + this.tracer.RelatedError($"{nameof(TryRunPreUpgradeChecks)}: {consoleError}"); + return false; + } + + this.tracer.RelatedInfo("Successfully finished pre upgrade checks. Okay to run GVFS upgrade."); + + consoleError = null; + return true; + } + + // TODO: Move repo mount calls to GVFS.Upgrader project. + public bool TryMountAllGVFSRepos(out string consoleError) + { + return this.TryRunGVFSWithArgs("service --mount-all", out consoleError); + } + + public bool TryUnmountAllGVFSRepos(out string consoleError) + { + consoleError = null; + + this.tracer.RelatedInfo("Unmounting any mounted GVFS repositories."); + + if (!this.TryRunGVFSWithArgs("service --unmount-all", out consoleError)) + { + this.tracer.RelatedError($"{nameof(TryUnmountAllGVFSRepos)}: {consoleError}"); + return false; + } + + // While checking for blocking processes like GVFS.Mount immediately after un-mounting, + // then sometimes GVFS.Mount shows up as running. But if the check is done after waiting + // for some time, then eventually GVFS.Mount goes away. The retry loop below is to help + // account for this delay between the time un-mount call returns and when GVFS.Mount + // actually quits. + this.tracer.RelatedInfo("Checking if GVFS or dependent processes are running."); + int retryCount = 10; + List processList = null; + while (retryCount > 0) + { + if (!this.IsBlockingProcessRunning(out processList)) + { + break; + } + + Thread.Sleep(TimeSpan.FromMilliseconds(250)); + retryCount--; + } + + if (processList.Count > 0) + { + consoleError = string.Join( + Environment.NewLine, + "Blocking processes are running.", + $"Run `{this.CommandToRerun}` again after quitting these processes - " + string.Join(", ", processList.ToArray())); + this.tracer.RelatedError($"{nameof(TryUnmountAllGVFSRepos)}: {consoleError}"); + return false; + } + + this.tracer.RelatedInfo("Successfully unmounted repositories."); + + return true; + } + + protected virtual bool IsElevated() + { + return GVFSPlatform.Instance.IsElevated(); + } + + protected virtual bool IsGVFSUpgradeSupported() + { + return GVFSPlatform.Instance.KernelDriver.IsGVFSUpgradeSupported(); + } + + protected virtual bool IsServiceInstalledAndNotRunning() + { + GVFSPlatform.Instance.IsServiceInstalledAndRunning(GVFSConstants.Service.ServiceName, out bool isInstalled, out bool isRunning); + + return isInstalled && !isRunning; + } + + protected virtual bool IsUnattended() + { + return GVFSEnlistment.IsUnattended(this.tracer); + } + + protected virtual bool IsDevelopmentVersion() + { + return ProcessHelper.IsDevelopmentVersion(); + } + + protected virtual bool IsBlockingProcessRunning(out List processes) + { + int currentProcessId = Process.GetCurrentProcess().Id; + Process[] allProcesses = Process.GetProcesses(); + HashSet matchingNames = new HashSet(); + + foreach (Process process in allProcesses) + { + if (process.Id == currentProcessId || !BlockingProcessSet.Contains(process.ProcessName)) + { + continue; + } + + matchingNames.Add(process.ProcessName); + } + + processes = matchingNames.ToList(); + return processes.Count > 0; + } + + protected virtual bool TryRunGVFSWithArgs(string args, out string consoleError) + { + consoleError = null; + + string gvfsPath = Path.Combine( + ProcessHelper.WhereDirectory(GVFSPlatform.Instance.Constants.GVFSExecutableName), + GVFSPlatform.Instance.Constants.GVFSExecutableName); + + ProcessResult processResult = ProcessHelper.Run(gvfsPath, args); + if (processResult.ExitCode == 0) + { + return true; + } + else + { + string output = string.IsNullOrEmpty(processResult.Output) ? string.Empty : processResult.Output; + string errorString = string.IsNullOrEmpty(processResult.Errors) ? "GVFS error" : processResult.Errors; + consoleError = string.Format("{0}. {1}", errorString, output); + return false; + } + } + + private bool IsGVFSUpgradeAllowed(out string consoleError) + { + consoleError = null; + + if (!this.IsElevated()) + { + consoleError = string.Join( + Environment.NewLine, + "The installer needs to be run from an elevated command prompt.", + $"Run `{this.CommandToRerun}` again from an elevated command prompt."); + return false; + } + + if (!this.IsGVFSUpgradeSupported()) + { + consoleError = string.Join( + Environment.NewLine, + "ProjFS configuration does not support `gvfs upgrade`.", + "Check your team's documentation for how to upgrade."); + return false; + } + + if (this.IsServiceInstalledAndNotRunning()) + { + consoleError = string.Join( + Environment.NewLine, + "GVFS Service is not running.", + $"Run `sc start GVFS.Service` and run `{this.CommandToRerun}` again."); + return false; + } + + return true; + } + } +} \ No newline at end of file diff --git a/GVFS/GVFS.Common/ProcessHelper.cs b/GVFS/GVFS.Common/ProcessHelper.cs index a949a9796..9250faaf5 100644 --- a/GVFS/GVFS.Common/ProcessHelper.cs +++ b/GVFS/GVFS.Common/ProcessHelper.cs @@ -1,4 +1,4 @@ -using Microsoft.Win32.SafeHandles; +using Microsoft.Win32.SafeHandles; using System; using System.Diagnostics; using System.IO; @@ -49,6 +49,13 @@ public static string GetCurrentProcessVersion() return fileVersionInfo.ProductVersion; } + public static bool IsDevelopmentVersion() + { + Version currentVersion = new Version(ProcessHelper.GetCurrentProcessVersion()); + + return currentVersion.Major == 0; + } + public static string WhereDirectory(string processName) { ProcessResult result = ProcessHelper.Run("where", processName); diff --git a/GVFS/GVFS.Common/ProductUpgrader.Shared.cs b/GVFS/GVFS.Common/ProductUpgrader.Shared.cs new file mode 100644 index 000000000..e8a995efc --- /dev/null +++ b/GVFS/GVFS.Common/ProductUpgrader.Shared.cs @@ -0,0 +1,69 @@ +using System; +using System.Diagnostics; +using System.IO; + +namespace GVFS.Common +{ + public partial class ProductUpgrader + { + public const string UpgradeDirectoryName = "GVFS.Upgrade"; + public const string LogDirectory = "Logs"; + + private const string RootDirectory = UpgradeDirectoryName; + private const string DownloadDirectory = "Downloads"; + private const string GVFSInstallerFileNamePrefix = "SetupGVFS"; + + public static bool IsLocalUpgradeAvailable() + { + string downloadDirectory = GetAssetDownloadsPath(); + if (Directory.Exists(downloadDirectory)) + { + string[] installers = Directory.GetFiles( + GetAssetDownloadsPath(), + $"{GVFSInstallerFileNamePrefix}*.*", + SearchOption.TopDirectoryOnly); + return installers.Length > 0; + } + + return false; + } + + public static string GetUpgradesDirectoryPath() + { + return Paths.GetServiceDataRoot(RootDirectory); + } + + public static string GetLogDirectoryPath() + { + return Path.Combine(Paths.GetServiceDataRoot(RootDirectory), LogDirectory); + } + + private static bool TryCreateDirectory(string path, out Exception exception) + { + try + { + Directory.CreateDirectory(path); + } + catch (IOException e) + { + exception = e; + return false; + } + catch (UnauthorizedAccessException e) + { + exception = e; + return false; + } + + exception = null; + return true; + } + + private static string GetAssetDownloadsPath() + { + return Path.Combine( + Paths.GetServiceDataRoot(RootDirectory), + DownloadDirectory); + } + } +} diff --git a/GVFS/GVFS.Common/ProductUpgrader.cs b/GVFS/GVFS.Common/ProductUpgrader.cs new file mode 100644 index 000000000..96f8dc683 --- /dev/null +++ b/GVFS/GVFS.Common/ProductUpgrader.cs @@ -0,0 +1,477 @@ +using GVFS.Common.FileSystem; +using GVFS.Common.Git; +using GVFS.Common.Tracing; +using System; +using System.Collections.Generic; +using System.IO; +using System.Net; +using System.Net.Http; +using System.Net.Http.Headers; +using System.Runtime.Serialization; +using System.Runtime.Serialization.Json; + +namespace GVFS.Common +{ + public partial class ProductUpgrader + { + private const string GitHubReleaseURL = @"https://api.github.com/repos/microsoft/vfsforgit/releases"; + private const string JSONMediaType = @"application/vnd.github.v3+json"; + private const string UserAgent = @"GVFS_Auto_Upgrader"; + private const string CommonInstallerArgs = "/VERYSILENT /CLOSEAPPLICATIONS /SUPPRESSMSGBOXES /NORESTART"; + private const string GVFSInstallerArgs = CommonInstallerArgs + " /MOUNTREPOS=false"; + private const string GitInstallerArgs = CommonInstallerArgs + " /ALLOWDOWNGRADE=1"; + private const string GitAssetNamePrefix = "Git"; + private const string GVFSAssetNamePrefix = "GVFS"; + private const string GitInstallerFileNamePrefix = "Git-"; + private const int RepoMountFailureExitCode = 17; + private const string ToolsDirectory = "Tools"; + private static readonly string UpgraderToolName = GVFSPlatform.Instance.Constants.GVFSUpgraderExecutableName; + private static readonly string UpgraderToolConfigFile = UpgraderToolName + ".config"; + private static readonly string[] UpgraderToolAndLibs = + { + UpgraderToolName, + UpgraderToolConfigFile, + "GVFS.Common.dll", + "GVFS.Platform.Windows.dll", + "Microsoft.Diagnostics.Tracing.EventSource.dll", + "netstandard.dll", + "System.Net.Http.dll", + "Newtonsoft.Json.dll" + }; + + private Version installedVersion; + private Release newestRelease; + private PhysicalFileSystem fileSystem; + private ITracer tracer; + + public ProductUpgrader( + string currentVersion, + ITracer tracer) + { + this.installedVersion = new Version(currentVersion); + this.fileSystem = new PhysicalFileSystem(); + this.Ring = RingType.Invalid; + this.tracer = tracer; + + string upgradesDirectoryPath = GetUpgradesDirectoryPath(); + this.fileSystem.CreateDirectory(upgradesDirectoryPath); + } + + public enum RingType + { + // The values here should be ascending. + // (Fast should be greater than Slow, + // Slow should be greater than None, None greater than Invalid.) + // This is required for the correct implementation of Ring based + // upgrade logic. + Invalid = 0, + None = 10, + Slow = 20, + Fast = 30, + } + + public RingType Ring { get; protected set; } + + public bool IsNoneRing() + { + return this.TryLoadRingConfig(out string _) && this.Ring == RingType.None; + } + + public bool TryGetNewerVersion( + out Version newVersion, + out string errorMessage) + { + List releases; + + newVersion = null; + if (this.Ring == RingType.Invalid && !this.TryLoadRingConfig(out errorMessage)) + { + return false; + } + + if (this.TryFetchReleases(out releases, out errorMessage)) + { + foreach (Release nextRelease in releases) + { + Version releaseVersion; + + if (nextRelease.Ring <= this.Ring && + nextRelease.TryParseVersion(out releaseVersion) && + releaseVersion > this.installedVersion) + { + newVersion = releaseVersion; + this.newestRelease = nextRelease; + break; + } + } + + return true; + } + + return false; + } + + public bool TryGetGitVersion(out GitVersion gitVersion, out string error) + { + gitVersion = null; + error = null; + + foreach (Asset asset in this.newestRelease.Assets) + { + if (asset.Name.StartsWith(GitInstallerFileNamePrefix) && + GitVersion.TryParseInstallerName(asset.Name, out gitVersion)) + { + return true; + } + } + + error = "Could not find Git version info in newest release"; + + return false; + } + + public bool TryDownloadNewestVersion(out string errorMessage) + { + errorMessage = null; + + foreach (Asset asset in this.newestRelease.Assets) + { + if (!this.TryDownloadAsset(asset, out errorMessage)) + { + return false; + } + } + + return true; + } + + public bool TryRunGitInstaller(out bool installationSucceeded, out string error) + { + error = null; + installationSucceeded = false; + + int exitCode = 0; + bool launched = this.TryRunInstallerForAsset(GitAssetNamePrefix, out exitCode, out error); + installationSucceeded = exitCode == 0; + + return launched; + } + + public bool TryRunGVFSInstaller(out bool installationSucceeded, out string error) + { + error = null; + installationSucceeded = false; + + int exitCode = 0; + bool launched = this.TryRunInstallerForAsset(GVFSAssetNamePrefix, out exitCode, out error); + installationSucceeded = exitCode == 0 || exitCode == RepoMountFailureExitCode; + + return launched; + } + + // TrySetupToolsDirectory - + // Copies GVFS Upgrader tool and its dependencies to a temporary location in ProgramData. + // Reason why this is needed - When GVFS.Upgrader.exe is run from C:\ProgramFiles\GVFS folder + // upgrade installer that is downloaded and run will fail. This is because it cannot overwrite + // C:\ProgramFiles\GVFS\GVFS.Upgrader.exe that is running. Moving GVFS.Upgrader.exe along with + // its dependencies to a temporary location inside ProgramData and running GVFS.Upgrader.exe + // from this temporary location helps avoid this problem. + public virtual bool TrySetupToolsDirectory(out string upgraderToolPath, out string error) + { + string rootDirectoryPath = ProductUpgrader.GetUpgradesDirectoryPath(); + string toolsDirectoryPath = Path.Combine(rootDirectoryPath, ToolsDirectory); + Exception exception; + if (TryCreateDirectory(toolsDirectoryPath, out exception)) + { + string currentPath = ProcessHelper.GetCurrentProcessLocation(); + error = null; + foreach (string name in UpgraderToolAndLibs) + { + string toolPath = Path.Combine(currentPath, name); + string destinationPath = Path.Combine(toolsDirectoryPath, name); + try + { + File.Copy(toolPath, destinationPath, overwrite: true); + } + catch (UnauthorizedAccessException e) + { + error = string.Join( + Environment.NewLine, + "File copy error - " + e.Message, + $"Make sure you have write permissions to directory {rootDirectoryPath} and run `gvfs upgrade --confirm` again."); + this.TraceException(e, nameof(this.TrySetupToolsDirectory), $"Error copying {toolPath} to {destinationPath}."); + break; + } + catch (IOException e) + { + error = "File copy error - " + e.Message; + this.TraceException(e, nameof(this.TrySetupToolsDirectory), $"Error copying {toolPath} to {destinationPath}."); + break; + } + } + + upgraderToolPath = string.IsNullOrEmpty(error) ? Path.Combine(toolsDirectoryPath, UpgraderToolName) : null; + return string.IsNullOrEmpty(error); + } + + upgraderToolPath = null; + error = exception.Message; + this.TraceException(exception, nameof(this.TrySetupToolsDirectory), $"Error creating upgrade tools directory {toolsDirectoryPath}."); + return false; + } + + public bool TryCleanup(out string error) + { + error = string.Empty; + if (this.newestRelease == null) + { + return true; + } + + foreach (Asset asset in this.newestRelease.Assets) + { + Exception exception; + if (!this.TryDeleteDownloadedAsset(asset, out exception)) + { + error += $"Could not delete {asset.LocalPath}. {exception.ToString()}." + Environment.NewLine; + } + } + + if (!string.IsNullOrEmpty(error)) + { + error.TrimEnd(Environment.NewLine.ToCharArray()); + return false; + } + + error = null; + return true; + } + + protected virtual bool TryDeleteDownloadedAsset(Asset asset, out Exception exception) + { + return this.fileSystem.TryDeleteFile(asset.LocalPath, out exception); + } + + protected virtual bool TryLoadRingConfig(out string error) + { + string errorAdvisory = "Run `git config --global gvfs.upgradering [\"Fast\"|\"Slow\"|\"None\"]` and run `gvfs upgrade [--confirm]` again."; + string gitPath = GVFSPlatform.Instance.GitInstallation.GetInstalledGitBinPath(); + GitProcess.Result result = GitProcess.GetFromGlobalConfig(gitPath, GVFSConstants.GitConfig.UpgradeRing); + if (!result.HasErrors && !string.IsNullOrEmpty(result.Output)) + { + string ringConfig = result.Output.TrimEnd('\r', '\n'); + RingType ringType; + + if (Enum.TryParse(ringConfig, ignoreCase: true, result: out ringType) && + Enum.IsDefined(typeof(RingType), ringType) && + ringType != RingType.Invalid) + { + this.Ring = ringType; + error = null; + return true; + } + else + { + error = "Invalid upgrade ring `" + ringConfig + "` specified in Git config."; + } + } + else + { + error = string.IsNullOrEmpty(result.Errors) ? "Unable to determine upgrade ring." : result.Errors; + } + + error += Environment.NewLine + errorAdvisory; + this.Ring = RingType.Invalid; + return false; + } + + protected virtual bool TryDownloadAsset(Asset asset, out string errorMessage) + { + errorMessage = null; + + string downloadPath = GetAssetDownloadsPath(); + Exception exception; + if (!ProductUpgrader.TryCreateDirectory(downloadPath, out exception)) + { + errorMessage = exception.Message; + this.TraceException(exception, nameof(this.TryDownloadAsset), $"Error creating download directory {downloadPath}."); + return false; + } + + string localPath = Path.Combine(downloadPath, asset.Name); + WebClient webClient = new WebClient(); + + try + { + webClient.DownloadFile(asset.DownloadURL, localPath); + asset.LocalPath = localPath; + } + catch (WebException webException) + { + errorMessage = "Download error: " + exception.Message; + this.TraceException(webException, nameof(this.TryDownloadAsset), $"Error downloading asset {asset.Name}."); + return false; + } + + return true; + } + + protected virtual bool TryFetchReleases(out List releases, out string errorMessage) + { + HttpClient client = new HttpClient(); + + client.DefaultRequestHeaders.Accept.Clear(); + client.DefaultRequestHeaders.Accept.Add(new MediaTypeWithQualityHeaderValue(JSONMediaType)); + client.DefaultRequestHeaders.Add("User-Agent", UserAgent); + + releases = null; + errorMessage = null; + + try + { + Stream result = client.GetStreamAsync(GitHubReleaseURL).GetAwaiter().GetResult(); + + DataContractJsonSerializer serializer = new DataContractJsonSerializer(typeof(List)); + releases = serializer.ReadObject(result) as List; + return true; + } + catch (HttpRequestException exception) + { + errorMessage = string.Format("Network error: could not connect to GitHub. {0}", exception.Message); + this.TraceException(exception, nameof(this.TryFetchReleases), $"Error fetching release info."); + } + catch (SerializationException exception) + { + errorMessage = string.Format("Parse error: could not parse releases info from GitHub. {0}", exception.Message); + this.TraceException(exception, nameof(this.TryFetchReleases), $"Error parsing release info."); + } + + return false; + } + + protected virtual void RunInstaller(string path, string args, out int exitCode, out string error) + { + ProcessResult processResult = ProcessHelper.Run(path, args); + + exitCode = processResult.ExitCode; + error = processResult.Errors; + } + + private bool TryRunInstallerForAsset(string name, out int installerExitCode, out string error) + { + error = null; + installerExitCode = 0; + + bool installerIsRun = false; + string path; + string installerArgs; + if (this.TryGetLocalInstallerPath(name, out path, out installerArgs)) + { + string logFilePath = GVFSEnlistment.GetNewLogFileName(GetLogDirectoryPath(), Path.GetFileNameWithoutExtension(path)); + string args = installerArgs + " /Log=" + logFilePath; + this.RunInstaller(path, args, out installerExitCode, out error); + + if (installerExitCode != 0 && string.IsNullOrEmpty(error)) + { + error = name + " installer failed. Error log: " + logFilePath; + } + + installerIsRun = true; + } + else + { + error = "Could not find downloaded installer for " + name; + } + + return installerIsRun; + } + + private void TraceException(Exception exception, string method, string message) + { + EventMetadata metadata = new EventMetadata(); + metadata.Add("Method", method); + metadata.Add("Exception", exception.ToString()); + this.tracer.RelatedError(metadata, message, Keywords.Telemetry); + } + + private bool TryGetLocalInstallerPath(string name, out string path, out string args) + { + foreach (Asset asset in this.newestRelease.Assets) + { + string extension = Path.GetExtension(asset.Name); + if (string.Equals(extension, ".exe", StringComparison.OrdinalIgnoreCase)) + { + path = asset.LocalPath; + if (name == GitAssetNamePrefix && asset.Name.StartsWith(GitInstallerFileNamePrefix, StringComparison.OrdinalIgnoreCase)) + { + args = GitInstallerArgs; + return true; + } + + if (name == GVFSAssetNamePrefix && asset.Name.StartsWith(GVFSInstallerFileNamePrefix, StringComparison.OrdinalIgnoreCase)) + { + args = GVFSInstallerArgs; + return true; + } + } + } + + path = null; + args = null; + return false; + } + + [DataContract(Name = "asset")] + protected class Asset + { + [DataMember(Name = "name")] + public string Name { get; set; } + + [DataMember(Name = "size")] + public long Size { get; set; } + + [DataMember(Name = "browser_download_url")] + public Uri DownloadURL { get; set; } + + [IgnoreDataMember] + public string LocalPath { get; set; } + } + + [DataContract(Name = "release")] + protected class Release + { + [DataMember(Name = "name")] + public string Name { get; set; } + + [DataMember(Name = "tag_name")] + public string Tag { get; set; } + + [DataMember(Name = "prerelease")] + public bool PreRelease { get; set; } + + [DataMember(Name = "assets")] + public List Assets { get; set; } + + [IgnoreDataMember] + public RingType Ring + { + get + { + return this.PreRelease == true ? RingType.Fast : RingType.Slow; + } + } + + public bool TryParseVersion(out Version version) + { + version = null; + + if (this.Tag.StartsWith("v", StringComparison.CurrentCultureIgnoreCase)) + { + return Version.TryParse(this.Tag.Substring(1), out version); + } + + return false; + } + } + } +} \ No newline at end of file diff --git a/GVFS/GVFS.FunctionalTests/Tests/EnlistmentPerFixture/GVFSUpgradeReminderTests.cs b/GVFS/GVFS.FunctionalTests/Tests/EnlistmentPerFixture/GVFSUpgradeReminderTests.cs new file mode 100644 index 000000000..837f21d27 --- /dev/null +++ b/GVFS/GVFS.FunctionalTests/Tests/EnlistmentPerFixture/GVFSUpgradeReminderTests.cs @@ -0,0 +1,88 @@ +using GVFS.FunctionalTests.FileSystemRunners; +using GVFS.FunctionalTests.Tools; +using GVFS.Tests.Should; +using NUnit.Framework; +using System; +using System.IO; +using System.Linq; + +namespace GVFS.FunctionalTests.Tests.EnlistmentPerFixture +{ + [TestFixture] + [NonParallelizable] + [Category(Categories.FullSuiteOnly)] + [Category(Categories.WindowsOnly)] + public class UpgradeReminderTests : TestsWithEnlistmentPerFixture + { + private const string GVFSInstallerName = "SetupGVFS.1.0.18234.1.exe"; + private const string GitInstallerName = "Git-2.17.1.gvfs.2.5.g2962052-64-bit.exe"; + + private string upgradeDirectory; + private FileSystemRunner fileSystem; + + public UpgradeReminderTests() + { + this.fileSystem = new SystemIORunner(); + this.upgradeDirectory = Path.Combine( + Environment.GetFolderPath(Environment.SpecialFolder.CommonApplicationData, Environment.SpecialFolderOption.Create), + "GVFS", + "GVFS.Upgrade", + "Downloads"); + } + + [TestCase] + public void NoNagWhenUpgradeNotAvailable() + { + this.EmptyDownloadDirectory(); + + ProcessResult result = GitHelpers.InvokeGitAgainstGVFSRepo( + this.Enlistment.RepoRoot, + "status"); + + string.IsNullOrEmpty(result.Errors).ShouldBeTrue(); + } + + [TestCase] + public void NagWhenUpgradeAvailable() + { + this.CreateUpgradeInstallers(); + + ProcessResult result = GitHelpers.InvokeGitAgainstGVFSRepo( + this.Enlistment.RepoRoot, + "status"); + + result.Errors.ShouldContain(new string[] + { + "A newer version of GVFS is available.", + "Run `gvfs upgrade --confirm` from an elevated command prompt to install." + }); + + this.EmptyDownloadDirectory(); + } + + private void EmptyDownloadDirectory() + { + if (Directory.Exists(this.upgradeDirectory)) + { + Directory.Delete(this.upgradeDirectory, recursive: true); + } + + Directory.CreateDirectory(this.upgradeDirectory); + Directory.Exists(this.upgradeDirectory).ShouldBeTrue(); + Directory.EnumerateFiles(this.upgradeDirectory).Any().ShouldBeFalse(); + } + + private void CreateUpgradeInstallers() + { + string gvfsInstallerPath = Path.Combine(this.upgradeDirectory, GVFSInstallerName); + string gitInstallerPath = Path.Combine(this.upgradeDirectory, GitInstallerName); + + this.EmptyDownloadDirectory(); + + this.fileSystem.CreateEmptyFile(gvfsInstallerPath); + this.fileSystem.CreateEmptyFile(gitInstallerPath); + this.fileSystem.FileExists(gvfsInstallerPath).ShouldBeTrue(); + this.fileSystem.FileExists(gitInstallerPath).ShouldBeTrue(); + } + } +} diff --git a/GVFS/GVFS.Hooks/GVFS.Hooks.Mac.csproj b/GVFS/GVFS.Hooks/GVFS.Hooks.Mac.csproj index dccf8bdd3..4cdc8852b 100644 --- a/GVFS/GVFS.Hooks/GVFS.Hooks.Mac.csproj +++ b/GVFS/GVFS.Hooks/GVFS.Hooks.Mac.csproj @@ -77,6 +77,7 @@ Common\ProcessResult.cs + Common\Tracing\EventLevel.cs diff --git a/GVFS/GVFS.Hooks/GVFS.Hooks.Windows.csproj b/GVFS/GVFS.Hooks/GVFS.Hooks.Windows.csproj index 05629c39e..e7dee24e3 100644 --- a/GVFS/GVFS.Hooks/GVFS.Hooks.Windows.csproj +++ b/GVFS/GVFS.Hooks/GVFS.Hooks.Windows.csproj @@ -1,4 +1,4 @@ - + @@ -101,6 +101,9 @@ Common\ProcessResult.cs + + Common\ProductUpgrader.Shared.cs + Common\Tracing\EventLevel.cs diff --git a/GVFS/GVFS.Hooks/Program.cs b/GVFS/GVFS.Hooks/Program.cs index 5ffaea8c0..e535b3452 100644 --- a/GVFS/GVFS.Hooks/Program.cs +++ b/GVFS/GVFS.Hooks/Program.cs @@ -1,4 +1,4 @@ -using GVFS.Common; +using GVFS.Common; using GVFS.Common.Git; using GVFS.Common.NamedPipes; using GVFS.Hooks.HooksPlatform; @@ -83,6 +83,12 @@ public static void Main(string[] args) private static void RunPreCommands(string[] args) { + if (ProductUpgrader.IsLocalUpgradeAvailable()) + { + Console.WriteLine(GVFSConstants.UpgradeVerbMessages.UpgradeAvailable); + Console.WriteLine(GVFSConstants.UpgradeVerbMessages.UpgradeInstallAdvice); + } + string command = GetGitCommand(args); switch (command) { diff --git a/GVFS/GVFS.Installer/Setup.iss b/GVFS/GVFS.Installer/Setup.iss index f2875b699..e01597ee2 100644 --- a/GVFS/GVFS.Installer/Setup.iss +++ b/GVFS/GVFS.Installer/Setup.iss @@ -14,6 +14,7 @@ #define GVFSMountDir BuildOutputDir + "\GVFS.Mount.Windows\bin\" + PlatformAndConfiguration #define ReadObjectDir BuildOutputDir + "\GVFS.ReadObjectHook.Windows\bin\" + PlatformAndConfiguration #define VirtualFileSystemDir BuildOutputDir + "\GVFS.VirtualFileSystemHook.Windows\bin\" + PlatformAndConfiguration +#define GVFSUpgraderDir BuildOutputDir + "\GVFS.Upgrader\bin\" + PlatformAndConfiguration #define MyAppName "GVFS" #define MyAppInstallerVersion GetFileVersion(GVFSDir + "\GVFS.exe") @@ -34,7 +35,7 @@ AppPublisher={#MyAppPublisher} AppPublisherURL={#MyAppPublisherURL} AppSupportURL={#MyAppURL} AppUpdatesURL={#MyAppURL} -AppCopyright=Copyright � Microsoft 2017 +AppCopyright=Copyright � Microsoft 2018 BackColor=clWhite BackSolid=yes DefaultDirName={pf}\{#MyAppName} @@ -93,6 +94,11 @@ DestDir: "{app}"; Flags: ignoreversion; Source:"{#GVFSMountDir}\GVFS.Mount.pdb" DestDir: "{app}"; Flags: ignoreversion; Source:"{#GVFSMountDir}\GVFS.Mount.exe" DestDir: "{app}"; Flags: ignoreversion; Source:"{#GVFSMountDir}\GVFS.Mount.exe.config" +; GVFS.Upgrader Files +DestDir: "{app}"; Flags: ignoreversion; Source:"{#GVFSUpgraderDir}\GVFS.Upgrader.pdb" +DestDir: "{app}"; Flags: ignoreversion; Source:"{#GVFSUpgraderDir}\GVFS.Upgrader.exe" +DestDir: "{app}"; Flags: ignoreversion; Source:"{#GVFSUpgraderDir}\GVFS.Upgrader.exe.config" + ; GVFS.ReadObjectHook files DestDir: "{app}"; Flags: ignoreversion; Source:"{#ReadObjectDir}\GVFS.ReadObjectHook.pdb" DestDir: "{app}"; Flags: ignoreversion; Source:"{#ReadObjectDir}\GVFS.ReadObjectHook.exe" @@ -159,6 +165,7 @@ DestDir: "{app}"; Flags: ignoreversion; Source:"{#ServiceDir}\GVFS.Service.exe"; [UninstallDelete] ; Deletes the entire installation directory, including files and subdirectories Type: filesandordirs; Name: "{app}"; +Type: filesandordirs; Name: "{commonappdata}\GVFS\GVFS.Upgrade"; [Registry] Root: HKLM; Subkey: "{#EnvironmentKey}"; \ @@ -623,7 +630,10 @@ begin end; ssPostInstall: begin - MountRepos(); + if ExpandConstant('{param:REMOUNTREPOS|true}') = 'true' then + begin + MountRepos(); + end end; end; end; @@ -650,7 +660,10 @@ begin Result := ''; if ConfirmUnmountAll() then begin - UnmountRepos(); + if ExpandConstant('{param:REMOUNTREPOS|true}') = 'true' then + begin + UnmountRepos(); + end end; if not EnsureGvfsNotRunning() then begin diff --git a/GVFS/GVFS.Platform.Mac/MacPlatform.cs b/GVFS/GVFS.Platform.Mac/MacPlatform.cs index 9be9a5ed9..6dade62c7 100644 --- a/GVFS/GVFS.Platform.Mac/MacPlatform.cs +++ b/GVFS/GVFS.Platform.Mac/MacPlatform.cs @@ -56,6 +56,11 @@ public override bool IsProcessActive(int processId) return MacPlatform.IsProcessActiveImplementation(processId); } + public override void IsServiceInstalledAndRunning(string name, out bool installed, out bool running) + { + throw new NotImplementedException(); + } + public override void StartBackgroundProcess(string programName, string[] args) { ProcessLauncher.StartBackgroundProcess(programName, args); diff --git a/GVFS/GVFS.Platform.Mac/ProjFSKext.cs b/GVFS/GVFS.Platform.Mac/ProjFSKext.cs index ede5623fc..01663e606 100644 --- a/GVFS/GVFS.Platform.Mac/ProjFSKext.cs +++ b/GVFS/GVFS.Platform.Mac/ProjFSKext.cs @@ -1,4 +1,4 @@ -using GVFS.Common; +using GVFS.Common; using GVFS.Common.FileSystem; using GVFS.Common.Tracing; using PrjFSLib.Mac; @@ -14,6 +14,12 @@ public class ProjFSKext : IKernelDriver public string DriverLogFolderName => throw new NotImplementedException(); + public bool IsGVFSUpgradeSupported() + { + // TODO(Mac) + return false; + } + public bool IsSupported(string normalizedEnlistmentRootPath, out string warning, out string error) { warning = null; diff --git a/GVFS/GVFS.Platform.Windows/ProjFSFilter.cs b/GVFS/GVFS.Platform.Windows/ProjFSFilter.cs index 3d7b4ce92..a5a0c3208 100644 --- a/GVFS/GVFS.Platform.Windows/ProjFSFilter.cs +++ b/GVFS/GVFS.Platform.Windows/ProjFSFilter.cs @@ -1,4 +1,4 @@ -using GVFS.Common; +using GVFS.Common; using GVFS.Common.FileSystem; using GVFS.Common.Tracing; using Microsoft.Win32; @@ -35,7 +35,16 @@ public class ProjFSFilter : IKernelDriver private const uint OkResult = 0; private const uint NameCollisionErrorResult = 0x801F0012; + private enum ProjFSInboxStatus + { + Invalid, + NotInbox = 2, + Enabled = 3, + Disabled = 4, + } + public bool EnumerationExpandsDirectories { get; } = false; + public string DriverLogFolderName { get; } = ProjFSFilter.ServiceName; public static bool TryAttach(ITracer tracer, string enlistmentRoot, out string errorMessage) @@ -272,6 +281,11 @@ public static bool IsNativeLibInstalled(ITracer tracer, PhysicalFileSystem fileS return existsInSystem32 || existsInAppDirectory; } + public bool IsGVFSUpgradeSupported() + { + return IsInboxAndEnabled(); + } + public bool IsSupported(string normalizedEnlistmentRootPath, out string warning, out string error) { warning = null; @@ -362,6 +376,12 @@ public bool IsReady(JsonTracer tracer, string enlistmentRoot, out string error) TryAttach(tracer, enlistmentRoot, out error); } + private static bool IsInboxAndEnabled() + { + ProcessResult getOptionalFeatureResult = GetProjFSOptionalFeatureStatus(); + return getOptionalFeatureResult.ExitCode == (int)ProjFSInboxStatus.Enabled; + } + private static bool TryGetIsInboxProjFSFinalAPI(ITracer tracer, out uint windowsBuildNumber, out bool isProjFSInbox) { isProjFSInbox = false; @@ -495,20 +515,13 @@ private static void GetNativeLibPaths(string gvfsAppDirectory, out string instal private static bool TryEnableProjFSOptionalFeature(ITracer tracer, PhysicalFileSystem fileSystem, out bool isProjFSFeatureAvailable) { EventMetadata metadata = CreateEventMetadata(); - - const int ProjFSNotAnOptionalFeature = 2; - const int ProjFSEnabled = 3; - const int ProjFSDisabled = 4; - - ProcessResult getOptionalFeatureResult = CallPowershellCommand( - "$var=(Get-WindowsOptionalFeature -Online -FeatureName " + OptionalFeatureName + "); if($var -eq $null){exit " + - ProjFSNotAnOptionalFeature + "}else{if($var.State -eq 'Enabled'){exit " + ProjFSEnabled + "}else{exit " + ProjFSDisabled + "}}"); + ProcessResult getOptionalFeatureResult = GetProjFSOptionalFeatureStatus(); isProjFSFeatureAvailable = true; bool projFSEnabled = false; switch (getOptionalFeatureResult.ExitCode) { - case ProjFSNotAnOptionalFeature: + case (int)ProjFSInboxStatus.NotInbox: metadata.Add("getOptionalFeatureResult.Output", getOptionalFeatureResult.Output); metadata.Add("getOptionalFeatureResult.Errors", getOptionalFeatureResult.Errors); tracer.RelatedWarning(metadata, $"{nameof(TryEnableProjFSOptionalFeature)}: {OptionalFeatureName} optional feature is missing"); @@ -516,7 +529,7 @@ private static bool TryEnableProjFSOptionalFeature(ITracer tracer, PhysicalFileS isProjFSFeatureAvailable = false; break; - case ProjFSEnabled: + case (int)ProjFSInboxStatus.Enabled: tracer.RelatedEvent( EventLevel.Informational, $"{nameof(TryEnableProjFSOptionalFeature)}_ClientProjFSAlreadyEnabled", @@ -525,7 +538,7 @@ private static bool TryEnableProjFSOptionalFeature(ITracer tracer, PhysicalFileS projFSEnabled = true; break; - case ProjFSDisabled: + case (int)ProjFSInboxStatus.Disabled: ProcessResult enableOptionalFeatureResult = CallPowershellCommand("try {Enable-WindowsOptionalFeature -Online -FeatureName " + OptionalFeatureName + " -NoRestart}catch{exit 1}"); metadata.Add("enableOptionalFeatureResult.Output", enableOptionalFeatureResult.Output.Trim().Replace("\r\n", ",")); metadata.Add("enableOptionalFeatureResult.Errors", enableOptionalFeatureResult.Errors); @@ -564,6 +577,18 @@ private static bool TryEnableProjFSOptionalFeature(ITracer tracer, PhysicalFileS return false; } + private static ProcessResult GetProjFSOptionalFeatureStatus() + { + return CallPowershellCommand( + "$var=(Get-WindowsOptionalFeature -Online -FeatureName " + OptionalFeatureName + "); if($var -eq $null){exit " + + (int)ProjFSInboxStatus.NotInbox + "}else{if($var.State -eq 'Enabled'){exit " + (int)ProjFSInboxStatus.Enabled + "}else{exit " + (int)ProjFSInboxStatus.Disabled + "}}"); + } + + private static ProcessResult CallPowershellCommand(string command) + { + return ProcessHelper.Run("powershell.exe", "-NonInteractive -NoProfile -Command \"& { " + command + " }\""); + } + private static EventMetadata CreateEventMetadata(Exception e = null) { EventMetadata metadata = new EventMetadata(); diff --git a/GVFS/GVFS.Platform.Windows/WindowsPlatform.cs b/GVFS/GVFS.Platform.Windows/WindowsPlatform.cs index aa8c3953b..0880ce9a4 100644 --- a/GVFS/GVFS.Platform.Windows/WindowsPlatform.cs +++ b/GVFS/GVFS.Platform.Windows/WindowsPlatform.cs @@ -12,6 +12,7 @@ using System.Linq; using System.Security.AccessControl; using System.Security.Principal; +using System.ServiceProcess; using System.Text; namespace GVFS.Platform.Windows @@ -170,6 +171,14 @@ public override bool IsProcessActive(int processId) return WindowsPlatform.IsProcessActiveImplementation(processId); } + public override void IsServiceInstalledAndRunning(string name, out bool installed, out bool running) + { + ServiceController service = ServiceController.GetServices().FirstOrDefault(s => s.ServiceName.Equals(name, StringComparison.Ordinal)); + + installed = service != null; + running = service != null ? service.Status == ServiceControllerStatus.Running : false; + } + public override string GetNamedPipeName(string enlistmentRoot) { return WindowsPlatform.GetNamedPipeNameImplementation(enlistmentRoot); diff --git a/GVFS/GVFS.Service/GVFS.Service.csproj b/GVFS/GVFS.Service/GVFS.Service.csproj index 32c7a4753..19975fabe 100644 --- a/GVFS/GVFS.Service/GVFS.Service.csproj +++ b/GVFS/GVFS.Service/GVFS.Service.csproj @@ -71,6 +71,7 @@ + diff --git a/GVFS/GVFS.Service/GvfsService.cs b/GVFS/GVFS.Service/GvfsService.cs index 0134cd244..4a51e988d 100644 --- a/GVFS/GVFS.Service/GvfsService.cs +++ b/GVFS/GVFS.Service/GvfsService.cs @@ -1,4 +1,4 @@ -using GVFS.Common; +using GVFS.Common; using GVFS.Common.FileSystem; using GVFS.Common.NamedPipes; using GVFS.Common.Tracing; @@ -6,6 +6,7 @@ using System; using System.IO; using System.Linq; +using System.Reflection; using System.Runtime.Serialization; using System.ServiceProcess; using System.Threading; @@ -23,12 +24,14 @@ public class GVFSService : ServiceBase private string serviceName; private string serviceDataLocation; private RepoRegistry repoRegistry; + private ProductUpgradeTimer productUpgradeTimer; public GVFSService(JsonTracer tracer) { this.tracer = tracer; this.serviceName = GVFSConstants.Service.ServiceName; this.CanHandleSessionChangeEvent = true; + this.productUpgradeTimer = new ProductUpgradeTimer(tracer); } public void Run() @@ -37,6 +40,7 @@ public void Run() { this.repoRegistry = new RepoRegistry(this.tracer, new PhysicalFileSystem(), this.serviceDataLocation); this.repoRegistry.Upgrade(); + this.productUpgradeTimer.Start(); string pipeName = this.serviceName + ".Pipe"; this.tracer.RelatedInfo("Starting pipe server with name: " + pipeName); @@ -68,6 +72,11 @@ public void StopRunning() try { + if (this.productUpgradeTimer != null) + { + this.productUpgradeTimer.Stop(); + } + if (this.tracer != null) { this.tracer.RelatedInfo("Stopping"); diff --git a/GVFS/GVFS.Service/ProductUpgradeTimer.cs b/GVFS/GVFS.Service/ProductUpgradeTimer.cs new file mode 100644 index 000000000..48c37d415 --- /dev/null +++ b/GVFS/GVFS.Service/ProductUpgradeTimer.cs @@ -0,0 +1,83 @@ +using GVFS.Common; +using GVFS.Common.Tracing; +using GVFS.Upgrader; +using System; +using System.Collections.Generic; +using System.Threading; + +namespace GVFS.Service +{ + public class ProductUpgradeTimer + { + private static readonly TimeSpan TimeInterval = TimeSpan.FromDays(1); + private JsonTracer tracer; + private Timer timer; + + public ProductUpgradeTimer(JsonTracer tracer) + { + this.tracer = tracer; + } + + public void Start() + { + Random random = new Random(); + TimeSpan startTime = TimeSpan.FromMinutes(random.Next(0, 60)); + + this.tracer.RelatedInfo($"Starting auto upgrade checks. Start time - {startTime.ToString()}"); + this.timer = new Timer( + this.TimerCallback, + null, + startTime, + TimeInterval); + } + + public void Stop() + { + this.tracer.RelatedInfo("Stopping auto upgrade checks"); + this.timer.Dispose(); + } + + private void TimerCallback(object unusedState) + { + string errorMessage = null; + + InstallerPreRunChecker prerunChecker = new InstallerPreRunChecker(this.tracer); + if (prerunChecker.TryRunPreUpgradeChecks(out string _) && !this.TryDownloadUpgrade(out errorMessage)) + { + this.tracer.RelatedError(errorMessage); + } + } + + private bool TryDownloadUpgrade(out string errorMessage) + { + this.tracer.RelatedInfo("Checking for product upgrades."); + + ProductUpgrader productUpgrader = new ProductUpgrader(ProcessHelper.GetCurrentProcessVersion(), this.tracer); + Version newerVersion = null; + string detailedError = null; + if (!productUpgrader.TryGetNewerVersion(out newerVersion, out detailedError)) + { + errorMessage = "Could not fetch new version info. " + detailedError; + return false; + } + + if (newerVersion == null) + { + // Already up-to-date + errorMessage = null; + return true; + } + + if (productUpgrader.TryDownloadNewestVersion(out detailedError)) + { + errorMessage = null; + return true; + } + else + { + errorMessage = "Could not download product upgrade. " + detailedError; + return false; + } + } + } +} diff --git a/GVFS/GVFS.SignFiles/GVFS.SignFiles.csproj b/GVFS/GVFS.SignFiles/GVFS.SignFiles.csproj index 7f14680f1..d3dacff46 100644 --- a/GVFS/GVFS.SignFiles/GVFS.SignFiles.csproj +++ b/GVFS/GVFS.SignFiles/GVFS.SignFiles.csproj @@ -47,6 +47,7 @@ $(BuildOutputDir)\GVFS.Service.UI\bin\$(Platform)\$(Configuration)\GVFS.Service.UI.exe; $(BuildOutputDir)\GVFS.Virtualization\bin\$(Platform)\$(Configuration)\netstandard2.0\GVFS.Virtualization.dll; $(BuildOutputDir)\GVFS.VirtualFileSystemHook.Windows\bin\$(Platform)\$(Configuration)\GVFS.VirtualFileSystemHook.exe; + $(BuildOutputDir)\GVFS.Upgrader\bin\$(Platform)\$(Configuration)\GVFS.Upgrader.exe; $(BuildOutputDir)\GVFS.Windows\bin\$(Platform)\$(Configuration)\GVFS.exe;"> Microsoft false diff --git a/GVFS/GVFS.UnitTests.Windows/GVFS.UnitTests.Windows.csproj b/GVFS/GVFS.UnitTests.Windows/GVFS.UnitTests.Windows.csproj index 6d9597a17..aaf571282 100644 --- a/GVFS/GVFS.UnitTests.Windows/GVFS.UnitTests.Windows.csproj +++ b/GVFS/GVFS.UnitTests.Windows/GVFS.UnitTests.Windows.csproj @@ -77,7 +77,15 @@ + + + + + + + + @@ -120,6 +128,10 @@ {72701bc3-5da9-4c7a-bf10-9e98c9fc8eac} GVFS.Tests + + {aecec217-2499-403d-b0bb-2962b9be5970} + GVFS.Upgrader + {F468B05A-95E5-46BC-8C67-B80A78527B7D} GVFS.Virtualization diff --git a/GVFS/GVFS.UnitTests.Windows/Windows/Mock/MockInstallerPreRunChecker.cs b/GVFS/GVFS.UnitTests.Windows/Windows/Mock/MockInstallerPreRunChecker.cs new file mode 100644 index 000000000..6b51f9a15 --- /dev/null +++ b/GVFS/GVFS.UnitTests.Windows/Windows/Mock/MockInstallerPreRunChecker.cs @@ -0,0 +1,129 @@ +using GVFS.Common; +using GVFS.Common.Git; +using GVFS.Common.Tracing; +using GVFS.UnitTests.Windows.Upgrader; +using GVFS.Upgrader; +using System; +using System.Collections.Generic; +using System.IO; + +namespace GVFS.UnitTests.Windows.Mock.Upgrader +{ + public class MockInstallerPrerunChecker : InstallerPreRunChecker + { + public const string GitUpgradeCheckError = "Unable to upgrade Git"; + + private FailOnCheckType failOnCheck; + + public MockInstallerPrerunChecker(ITracer tracer) : base(tracer) + { + } + + [Flags] + public enum FailOnCheckType + { + Invalid = 0, + ProjFSEnabled = 0x1, + IsElevated = 0x2, + BlockingProcessesRunning = 0x4, + UnattendedMode = 0x8, + IsDevelopmentVersion = 0x10, + IsGitUpgradeAllowed = 0x20, + UnMountRepos = 0x40, + RemountRepos = 0x80, + IsServiceInstalledAndNotRunning = 0x100 + } + + public List GVFSArgs { get; private set; } = new List(); + + public void SetReturnFalseOnCheck(FailOnCheckType prerunCheck) + { + this.failOnCheck |= prerunCheck; + } + + public void SetReturnTrueOnCheck(FailOnCheckType prerunCheck) + { + this.failOnCheck &= ~prerunCheck; + } + + public void Reset() + { + this.failOnCheck = FailOnCheckType.Invalid; + + this.SetReturnFalseOnCheck(MockInstallerPrerunChecker.FailOnCheckType.IsDevelopmentVersion); + this.SetReturnFalseOnCheck(MockInstallerPrerunChecker.FailOnCheckType.UnattendedMode); + this.SetReturnFalseOnCheck(MockInstallerPrerunChecker.FailOnCheckType.BlockingProcessesRunning); + this.SetReturnFalseOnCheck(MockInstallerPrerunChecker.FailOnCheckType.IsServiceInstalledAndNotRunning); + + this.GVFSArgs.Clear(); + } + + protected override bool IsServiceInstalledAndNotRunning() + { + return this.FakedResultOfCheck(FailOnCheckType.IsServiceInstalledAndNotRunning); + } + + protected override bool IsElevated() + { + return this.FakedResultOfCheck(FailOnCheckType.IsElevated); + } + + protected override bool IsGVFSUpgradeSupported() + { + return this.FakedResultOfCheck(FailOnCheckType.ProjFSEnabled); + } + + protected override bool IsUnattended() + { + return this.FakedResultOfCheck(FailOnCheckType.UnattendedMode); + } + + protected override bool IsDevelopmentVersion() + { + return this.FakedResultOfCheck(FailOnCheckType.IsDevelopmentVersion); + } + + protected override bool IsBlockingProcessRunning(out List processes) + { + processes = new List(); + + bool isRunning = this.FakedResultOfCheck(FailOnCheckType.BlockingProcessesRunning); + if (isRunning) + { + processes.Add("GVFS.Mount"); + processes.Add("git"); + } + + return isRunning; + } + + protected override bool TryRunGVFSWithArgs(string args, out string error) + { + this.GVFSArgs.Add(args); + + if (string.CompareOrdinal(args, "service --unmount-all") == 0) + { + bool result = this.FakedResultOfCheck(FailOnCheckType.UnMountRepos); + error = result == false ? "Unmount of some of the repositories failed." : null; + return result; + } + + if (string.CompareOrdinal(args, "service --mount-all") == 0) + { + bool result = this.FakedResultOfCheck(FailOnCheckType.RemountRepos); + error = result == false ? "Auto remount failed." : null; + return result; + } + + error = "Unknown GVFS command"; + return false; + } + + private bool FakedResultOfCheck(FailOnCheckType checkType) + { + bool result = this.failOnCheck.HasFlag(checkType) ? false : true; + + return result; + } + } +} diff --git a/GVFS/GVFS.UnitTests.Windows/Windows/Mock/MockProcessLauncher.cs b/GVFS/GVFS.UnitTests.Windows/Windows/Mock/MockProcessLauncher.cs new file mode 100644 index 000000000..dc3ddde55 --- /dev/null +++ b/GVFS/GVFS.UnitTests.Windows/Windows/Mock/MockProcessLauncher.cs @@ -0,0 +1,45 @@ +using System; +using static GVFS.CommandLine.UpgradeVerb; + +namespace GVFS.UnitTests.Windows.Upgrader +{ + public class MockProcessLauncher : ProcessLauncher + { + private int exitCode; + private bool hasExited; + private bool startResult; + + public MockProcessLauncher( + int exitCode, + bool hasExited, + bool startResult) : base() + { + this.exitCode = exitCode; + this.hasExited = hasExited; + this.startResult = startResult; + } + + public bool IsLaunched { get; private set; } + + public string LaunchPath { get; private set; } + + public override bool HasExited + { + get { return this.hasExited; } + } + + public override int ExitCode + { + get { return this.exitCode; } + } + + public override bool TryStart(string path, out Exception exception) + { + this.LaunchPath = path; + this.IsLaunched = true; + + exception = null; + return this.startResult; + } + } +} diff --git a/GVFS/GVFS.UnitTests.Windows/Windows/Mock/MockProductUpgrader.cs b/GVFS/GVFS.UnitTests.Windows/Windows/Mock/MockProductUpgrader.cs new file mode 100644 index 000000000..229994393 --- /dev/null +++ b/GVFS/GVFS.UnitTests.Windows/Windows/Mock/MockProductUpgrader.cs @@ -0,0 +1,241 @@ +using GVFS.Common; +using GVFS.Common.Tracing; +using GVFS.UnitTests.Windows.Upgrader; +using System; +using System.Collections.Generic; +using System.IO; + +namespace GVFS.UnitTests.Windows.Mock.Upgrader +{ + public class MockProductUpgrader : ProductUpgrader + { + private string expectedGVFSAssetName; + private string expectedGitAssetName; + private ActionType failActionTypes; + + public MockProductUpgrader( + string currentVersion, + ITracer tracer) : base(currentVersion, tracer) + { + this.DownloadedFiles = new List(); + this.InstallerArgs = new Dictionary>(); + } + + [Flags] + public enum ActionType + { + Invalid = 0, + FetchReleaseInfo = 0x1, + CopyTools = 0x2, + GitDownload = 0x4, + GVFSDownload = 0x8, + GitInstall = 0x10, + GVFSInstall = 0x20, + GVFSCleanup = 0x40, + GitCleanup = 0x80, + } + + public RingType LocalRingConfig { get; set; } + public List DownloadedFiles { get; private set; } + public Dictionary> InstallerArgs { get; private set; } + + private Release FakeUpgradeRelease { get; set; } + + public void SetFailOnAction(ActionType failureType) + { + this.failActionTypes |= failureType; + } + + public void SetSucceedOnAction(ActionType failureType) + { + this.failActionTypes &= ~failureType; + } + + public void ResetFailedAction() + { + this.failActionTypes = ActionType.Invalid; + } + + public void PretendNewReleaseAvailableAtRemote(string upgradeVersion, RingType remoteRing) + { + string assetDownloadURLPrefix = "https://github.com/Microsoft/VFSForGit/releases/download/v" + upgradeVersion; + + Release release = new Release(); + + release.Name = "GVFS " + upgradeVersion; + release.Tag = "v" + upgradeVersion; + release.PreRelease = remoteRing == RingType.Fast; + release.Assets = new List(); + + Random random = new Random(); + Asset gvfsAsset = new Asset(); + gvfsAsset.Name = "SetupGVFS." + upgradeVersion + ".exe"; + + // This is not cross-checked anywhere, random value is good. + gvfsAsset.Size = random.Next(int.MaxValue / 10, int.MaxValue / 2); + gvfsAsset.DownloadURL = new Uri(assetDownloadURLPrefix + "/SetupGVFS." + upgradeVersion + ".exe"); + release.Assets.Add(gvfsAsset); + + Asset gitAsset = new Asset(); + gitAsset.Name = "Git-2.17.1.gvfs.2.1.4.g4385455-64-bit.exe"; + gitAsset.Size = random.Next(int.MaxValue / 10, int.MaxValue / 2); + gitAsset.DownloadURL = new Uri(assetDownloadURLPrefix + "/Git-2.17.1.gvfs.2.1.4.g4385455-64-bit.exe"); + release.Assets.Add(gitAsset); + + this.expectedGVFSAssetName = gvfsAsset.Name; + this.expectedGitAssetName = gitAsset.Name; + this.FakeUpgradeRelease = release; + } + + public override bool TrySetupToolsDirectory(out string upgraderToolPath, out string error) + { + if (this.failActionTypes.HasFlag(ActionType.CopyTools)) + { + upgraderToolPath = null; + error = "Unable to copy upgrader tools"; + return false; + } + + upgraderToolPath = @"C:\ProgramData\GVFS\GVFS.Upgrade\Tools\GVFS.Upgrader.exe"; + error = null; + return true; + } + + protected override bool TryLoadRingConfig(out string error) + { + this.Ring = this.LocalRingConfig; + + if (this.LocalRingConfig == RingType.Invalid) + { + error = "Invalid upgrade ring `Invalid` specified in Git config."; + return false; + } + + error = null; + return true; + } + + protected override bool TryDownloadAsset(Asset asset, out string errorMessage) + { + bool validAsset = true; + if (this.expectedGVFSAssetName.Equals(asset.Name, StringComparison.OrdinalIgnoreCase)) + { + if (this.failActionTypes.HasFlag(ActionType.GVFSDownload)) + { + errorMessage = "Error downloading GVFS from GitHub"; + return false; + } + } + else if (this.expectedGitAssetName.Equals(asset.Name, StringComparison.OrdinalIgnoreCase)) + { + if (this.failActionTypes.HasFlag(ActionType.GitDownload)) + { + errorMessage = "Error downloading Git from GitHub"; + return false; + } + } + else + { + validAsset = false; + } + + if (validAsset) + { + string fakeDownloadDirectory = @"C:\ProgramData\GVFS\GVFS.Upgrade\Downloads"; + asset.LocalPath = Path.Combine(fakeDownloadDirectory, asset.Name); + this.DownloadedFiles.Add(asset.LocalPath); + + errorMessage = null; + return true; + } + + errorMessage = "Cannot download unknown asset."; + return false; + } + + protected override bool TryDeleteDownloadedAsset(Asset asset, out Exception exception) + { + if (this.expectedGVFSAssetName.Equals(asset.Name, StringComparison.OrdinalIgnoreCase)) + { + if (this.failActionTypes.HasFlag(ActionType.GVFSCleanup)) + { + exception = new Exception("Error deleting downloaded GVFS installer."); + return false; + } + + exception = null; + return true; + } + else if (this.expectedGitAssetName.Equals(asset.Name, StringComparison.OrdinalIgnoreCase)) + { + if (this.failActionTypes.HasFlag(ActionType.GitCleanup)) + { + exception = new Exception("Error deleting downloaded Git installer."); + return false; + } + + exception = null; + return true; + } + else + { + exception = new Exception("Unknown asset."); + return false; + } + } + + protected override bool TryFetchReleases(out List releases, out string errorMessage) + { + if (this.failActionTypes.HasFlag(ActionType.FetchReleaseInfo)) + { + releases = null; + errorMessage = "Error fetching upgrade release info."; + return false; + } + + releases = new List { this.FakeUpgradeRelease }; + errorMessage = null; + + return true; + } + + protected override void RunInstaller(string path, string args, out int exitCode, out string error) + { + string fileName = Path.GetFileName(path); + Dictionary installationInfo = new Dictionary(); + installationInfo.Add("Installer", fileName); + installationInfo.Add("Args", args); + + exitCode = 0; + error = null; + + if (fileName.Equals(this.expectedGitAssetName, StringComparison.OrdinalIgnoreCase)) + { + this.InstallerArgs.Add("Git", installationInfo); + if (this.failActionTypes.HasFlag(ActionType.GitInstall)) + { + exitCode = -1; + error = "Git installation failed"; + } + + return; + } + + if (fileName.Equals(this.expectedGVFSAssetName, StringComparison.OrdinalIgnoreCase)) + { + this.InstallerArgs.Add("GVFS", installationInfo); + if (this.failActionTypes.HasFlag(ActionType.GVFSInstall)) + { + exitCode = -1; + error = "GVFS installation failed"; + } + + return; + } + + exitCode = -1; + error = "Cannot launch unknown installer"; + return; + } + } +} diff --git a/GVFS/GVFS.UnitTests.Windows/Windows/Mock/MockTextWriter.cs b/GVFS/GVFS.UnitTests.Windows/Windows/Mock/MockTextWriter.cs new file mode 100644 index 000000000..e6e8c7fd9 --- /dev/null +++ b/GVFS/GVFS.UnitTests.Windows/Windows/Mock/MockTextWriter.cs @@ -0,0 +1,47 @@ +using System; +using System.Collections.Generic; +using System.IO; +using System.Text; + +namespace GVFS.UnitTests.Windows.Mock.Upgrader +{ + public class MockTextWriter : TextWriter + { + private StringBuilder stringBuilder; + + public MockTextWriter() : base() + { + this.AllLines = new List(); + this.stringBuilder = new StringBuilder(); + } + + public List AllLines { get; private set; } + + public override Encoding Encoding + { + get { return Encoding.Default; } + } + + public override void Write(char value) + { + if (value.Equals('\r')) + { + return; + } + + if (value.Equals('\n')) + { + this.AllLines.Add(this.stringBuilder.ToString()); + this.stringBuilder.Clear(); + return; + } + + this.stringBuilder.Append(value); + } + + public bool ContainsLine(string line) + { + return this.AllLines.Exists(x => x.Equals(line, StringComparison.Ordinal)); + } + } +} diff --git a/GVFS/GVFS.UnitTests.Windows/Windows/Upgrader/ProductUpgraderTests.cs b/GVFS/GVFS.UnitTests.Windows/Windows/Upgrader/ProductUpgraderTests.cs new file mode 100644 index 000000000..dec15f652 --- /dev/null +++ b/GVFS/GVFS.UnitTests.Windows/Windows/Upgrader/ProductUpgraderTests.cs @@ -0,0 +1,135 @@ +using GVFS.Common; +using GVFS.Tests.Should; +using NUnit.Framework; +using System; + +namespace GVFS.UnitTests.Windows.Upgrader +{ + [TestFixture] + public class ProductUpgraderTests : UpgradeTests + { + [SetUp] + public override void Setup() + { + base.Setup(); + } + + [TestCase] + public void UpgradeAvailableOnFastWhileOnLocalNoneRing() + { + this.SimulateUpgradeAvailable( + remoteRing: ProductUpgrader.RingType.Fast, + remoteVersion: UpgradeTests.NewerThanLocalVersion, + localRing: ProductUpgrader.RingType.None, + expectedReturn: true, + expectedUpgradeVersion: null); + } + + [TestCase] + public void UpgradeAvailableOnSlowWhileOnLocalNoneRing() + { + this.SimulateUpgradeAvailable( + remoteRing: ProductUpgrader.RingType.Slow, + remoteVersion: UpgradeTests.NewerThanLocalVersion, + localRing: ProductUpgrader.RingType.None, + expectedReturn: true, + expectedUpgradeVersion: null); + } + + [TestCase] + public void UpgradeAvailableOnFastWhileOnLocalSlowRing() + { + this.SimulateUpgradeAvailable( + remoteRing: ProductUpgrader.RingType.Fast, + remoteVersion: UpgradeTests.NewerThanLocalVersion, + localRing: ProductUpgrader.RingType.Slow, + expectedReturn: true, + expectedUpgradeVersion: null); + } + + [TestCase] + public void UpgradeAvailableOnSlowWhileOnLocalSlowRing() + { + this.SimulateUpgradeAvailable( + remoteRing: ProductUpgrader.RingType.Slow, + remoteVersion: UpgradeTests.NewerThanLocalVersion, + localRing: ProductUpgrader.RingType.Slow, + expectedReturn: true, + expectedUpgradeVersion: UpgradeTests.NewerThanLocalVersion); + } + + [TestCase] + public void UpgradeAvailableOnFastWhileOnLocalFastRing() + { + this.SimulateUpgradeAvailable( + remoteRing: ProductUpgrader.RingType.Fast, + remoteVersion: UpgradeTests.NewerThanLocalVersion, + localRing: ProductUpgrader.RingType.Fast, + expectedReturn: true, + expectedUpgradeVersion: UpgradeTests.NewerThanLocalVersion); + } + + [TestCase] + public void UpgradeAvailableOnSlowWhileOnLocalFastRing() + { + this.SimulateUpgradeAvailable( + remoteRing: ProductUpgrader.RingType.Slow, + remoteVersion: UpgradeTests.NewerThanLocalVersion, + localRing:ProductUpgrader.RingType.Fast, + expectedReturn: true, + expectedUpgradeVersion:UpgradeTests.NewerThanLocalVersion); + } + + public override void NoneLocalRing() + { + throw new NotImplementedException(); + } + + public override void InvalidUpgradeRing() + { + throw new NotImplementedException(); + } + + public override void FetchReleaseInfo() + { + throw new NotImplementedException(); + } + + protected override void RunUpgrade() + { + throw new NotImplementedException(); + } + + protected override ReturnCode ExitCode() + { + return ReturnCode.Success; + } + + private void SimulateUpgradeAvailable( + ProductUpgrader.RingType remoteRing, + string remoteVersion, + ProductUpgrader.RingType localRing, + bool expectedReturn, + string expectedUpgradeVersion) + { + this.Upgrader.LocalRingConfig = localRing; + this.Upgrader.PretendNewReleaseAvailableAtRemote( + remoteVersion, + remoteRing); + + Version newVersion; + string errorMessage; + this.Upgrader.TryGetNewerVersion(out newVersion, out errorMessage).ShouldEqual(expectedReturn); + + if (string.IsNullOrEmpty(expectedUpgradeVersion)) + { + newVersion.ShouldBeNull(); + } + else + { + newVersion.ShouldNotBeNull(); + newVersion.ShouldEqual(new Version(expectedUpgradeVersion)); + } + } + } +} \ No newline at end of file diff --git a/GVFS/GVFS.UnitTests.Windows/Windows/Upgrader/UpgradeOrchestratorTests.cs b/GVFS/GVFS.UnitTests.Windows/Windows/Upgrader/UpgradeOrchestratorTests.cs new file mode 100644 index 000000000..abced85c4 --- /dev/null +++ b/GVFS/GVFS.UnitTests.Windows/Windows/Upgrader/UpgradeOrchestratorTests.cs @@ -0,0 +1,250 @@ +using GVFS.Common; +using GVFS.Tests.Should; +using GVFS.UnitTests.Windows.Mock.Upgrader; +using GVFS.Upgrader; +using NUnit.Framework; +using System.Collections.Generic; + +namespace GVFS.UnitTests.Windows.Upgrader +{ + [TestFixture] + public class UpgradeOrchestratorTests : UpgradeTests + { + private UpgradeOrchestrator Orchestrator { get; set; } + + [SetUp] + public override void Setup() + { + base.Setup(); + + this.Orchestrator = new UpgradeOrchestrator( + this.Upgrader, + this.Tracer, + this.PrerunChecker, + input: null, + output: this.Output, + shouldExitOnError: false); + this.PrerunChecker.CommandToRerun = "gvfs upgrade --confirm"; + } + + [TestCase] + public void UpgradeNoError() + { + this.Orchestrator.Execute(); + this.Orchestrator.ExitCode.ShouldEqual(ReturnCode.Success); + this.Tracer.RelatedErrorEvents.ShouldBeEmpty(); + } + + [TestCase] + public void AutoUnmountError() + { + this.ConfigureRunAndVerify( + configure: () => + { + this.PrerunChecker.SetReturnFalseOnCheck(MockInstallerPrerunChecker.FailOnCheckType.UnMountRepos); + }, + expectedReturn: ReturnCode.GenericError, + expectedOutput: new List + { + "Unmount of some of the repositories failed." + }, + expectedErrors: new List + { + "Unmount of some of the repositories failed." + }); + } + + [TestCase] + public void AbortOnBlockingProcess() + { + this.ConfigureRunAndVerify( + configure: () => + { + this.PrerunChecker.SetReturnTrueOnCheck(MockInstallerPrerunChecker.FailOnCheckType.BlockingProcessesRunning); + }, + expectedReturn: ReturnCode.GenericError, + expectedOutput: new List + { + "ERROR: Blocking processes are running.", + $"Run `gvfs upgrade --confirm` again after quitting these processes - GVFS.Mount, git" + }, + expectedErrors: new List + { + $"Run `gvfs upgrade --confirm` again after quitting these processes - GVFS.Mount, git" + }); + } + + [TestCase] + public void GVFSDownloadError() + { + this.ConfigureRunAndVerify( + configure: () => + { + this.Upgrader.SetFailOnAction(MockProductUpgrader.ActionType.GVFSDownload); + }, + expectedReturn: ReturnCode.GenericError, + expectedOutput: new List + { + "Error downloading GVFS from GitHub" + }, + expectedErrors: new List + { + "Error downloading GVFS from GitHub" + }); + } + + [TestCase] + public void GitDownloadError() + { + this.ConfigureRunAndVerify( + configure: () => + { + this.Upgrader.SetFailOnAction(MockProductUpgrader.ActionType.GitDownload); + }, + expectedReturn: ReturnCode.GenericError, + expectedOutput: new List + { + "Error downloading Git from GitHub" + }, + expectedErrors: new List + { + "Error downloading Git from GitHub" + }); + } + + [TestCase] + public void GitInstallationArgs() + { + this.Orchestrator.Execute(); + + this.Orchestrator.ExitCode.ShouldEqual(ReturnCode.Success); + + Dictionary gitInstallerInfo; + this.Upgrader.InstallerArgs.ShouldBeNonEmpty(); + this.Upgrader.InstallerArgs.TryGetValue("Git", out gitInstallerInfo).ShouldBeTrue(); + + string args; + gitInstallerInfo.TryGetValue("Args", out args).ShouldBeTrue(); + args.ShouldContain(new string[] { "/VERYSILENT", "/CLOSEAPPLICATIONS", "/SUPPRESSMSGBOXES", "/NORESTART", "/Log" }); + } + + [TestCase] + public void GitInstallError() + { + this.ConfigureRunAndVerify( + configure: () => + { + this.Upgrader.SetFailOnAction(MockProductUpgrader.ActionType.GitInstall); + }, + expectedReturn: ReturnCode.GenericError, + expectedOutput: new List + { + "Git installation failed" + }, + expectedErrors: new List + { + "Git installation failed" + }); + } + + [TestCase] + public void GVFSInstallationArgs() + { + this.Orchestrator.Execute(); + + this.Orchestrator.ExitCode.ShouldEqual(ReturnCode.Success); + + Dictionary gitInstallerInfo; + this.Upgrader.InstallerArgs.ShouldBeNonEmpty(); + this.Upgrader.InstallerArgs.TryGetValue("GVFS", out gitInstallerInfo).ShouldBeTrue(); + + string args; + gitInstallerInfo.TryGetValue("Args", out args).ShouldBeTrue(); + args.ShouldContain(new string[] { "/VERYSILENT", "/CLOSEAPPLICATIONS", "/SUPPRESSMSGBOXES", "/NORESTART", "/Log", "/MOUNTREPOS=false" }); + } + + [TestCase] + public void GVFSInstallError() + { + this.ConfigureRunAndVerify( + configure: () => + { + this.Upgrader.SetFailOnAction(MockProductUpgrader.ActionType.GVFSInstall); + }, + expectedReturn: ReturnCode.GenericError, + expectedOutput: new List + { + "GVFS installation failed" + }, + expectedErrors: new List + { + "GVFS installation failed" + }); + } + + [TestCase] + public void GVFSCleanupError() + { + this.ConfigureRunAndVerify( + configure: () => + { + this.Upgrader.SetFailOnAction(MockProductUpgrader.ActionType.GVFSCleanup); + }, + expectedReturn: ReturnCode.Success, + expectedOutput: new List + { + }, + expectedErrors: new List + { + "Error deleting downloaded GVFS installer." + }); + } + + [TestCase] + public void GitCleanupError() + { + this.ConfigureRunAndVerify( + configure: () => + { + this.Upgrader.SetFailOnAction(MockProductUpgrader.ActionType.GitCleanup); + }, + expectedReturn: ReturnCode.Success, + expectedOutput: new List + { + }, + expectedErrors: new List + { + "Error deleting downloaded Git installer." + }); + } + + [TestCase] + public void RemountReposError() + { + this.ConfigureRunAndVerify( + configure: () => + { + this.PrerunChecker.SetReturnFalseOnCheck(MockInstallerPrerunChecker.FailOnCheckType.RemountRepos); + }, + expectedReturn: ReturnCode.Success, + expectedOutput: new List + { + "Auto remount failed." + }, + expectedErrors: new List + { + "Auto remount failed." + }); + } + + protected override void RunUpgrade() + { + this.Orchestrator.Execute(); + } + + protected override ReturnCode ExitCode() + { + return this.Orchestrator.ExitCode; + } + } +} \ No newline at end of file diff --git a/GVFS/GVFS.UnitTests.Windows/Windows/Upgrader/UpgradeTests.cs b/GVFS/GVFS.UnitTests.Windows/Windows/Upgrader/UpgradeTests.cs new file mode 100644 index 000000000..9f9e1b3d3 --- /dev/null +++ b/GVFS/GVFS.UnitTests.Windows/Windows/Upgrader/UpgradeTests.cs @@ -0,0 +1,128 @@ +using GVFS.Common; +using GVFS.Tests.Should; +using GVFS.UnitTests.Category; +using GVFS.UnitTests.Mock.Common; +using GVFS.UnitTests.Windows.Mock.Upgrader; +using NUnit.Framework; +using System; +using System.Collections.Generic; + +namespace GVFS.UnitTests.Windows.Upgrader +{ + public abstract class UpgradeTests + { + protected const string OlderThanLocalVersion = "1.0.17000.1"; + protected const string LocalGVFSVersion = "1.0.18115.1"; + protected const string NewerThanLocalVersion = "1.1.18115.1"; + + protected MockTracer Tracer { get; private set; } + protected MockTextWriter Output { get; private set; } + protected MockInstallerPrerunChecker PrerunChecker { get; private set; } + protected MockProductUpgrader Upgrader { get; private set; } + + public virtual void Setup() + { + this.Tracer = new MockTracer(); + this.Output = new MockTextWriter(); + this.PrerunChecker = new MockInstallerPrerunChecker(this.Tracer); + this.Upgrader = new MockProductUpgrader(LocalGVFSVersion, this.Tracer); + + this.PrerunChecker.Reset(); + this.Upgrader.PretendNewReleaseAvailableAtRemote( + upgradeVersion: NewerThanLocalVersion, + remoteRing: ProductUpgrader.RingType.Slow); + this.Upgrader.LocalRingConfig = ProductUpgrader.RingType.Slow; + } + + [TestCase] + public virtual void NoneLocalRing() + { + string message = "Upgrade ring set to None. No upgrade check was performed."; + this.ConfigureRunAndVerify( + configure: () => + { + this.Upgrader.LocalRingConfig = ProductUpgrader.RingType.None; + }, + expectedReturn: ReturnCode.Success, + expectedOutput: new List + { + message + }, + expectedErrors: new List + { + }); + } + + [TestCase] + public virtual void InvalidUpgradeRing() + { + string errorString = "Invalid upgrade ring `Invalid` specified in Git config."; + this.ConfigureRunAndVerify( + configure: () => + { + this.Upgrader.LocalRingConfig = GVFS.Common.ProductUpgrader.RingType.Invalid; + }, + expectedReturn: ReturnCode.GenericError, + expectedOutput: new List + { + errorString + }, + expectedErrors: new List + { + errorString + }); + } + + [TestCase] + [Category(CategoryConstants.ExceptionExpected)] + public virtual void FetchReleaseInfo() + { + string errorString = "Error fetching upgrade release info."; + this.ConfigureRunAndVerify( + configure: () => + { + this.Upgrader.SetFailOnAction(MockProductUpgrader.ActionType.FetchReleaseInfo); + }, + expectedReturn: ReturnCode.GenericError, + expectedOutput: new List + { + errorString + }, + expectedErrors: new List + { + errorString + }); + } + + protected abstract void RunUpgrade(); + + protected abstract ReturnCode ExitCode(); + + protected void ConfigureRunAndVerify( + Action configure, + ReturnCode expectedReturn, + List expectedOutput, + List expectedErrors) + { + configure(); + + this.RunUpgrade(); + + this.ExitCode().ShouldEqual(expectedReturn); + + if (expectedOutput != null) + { + this.Output.AllLines.ShouldContain( + expectedOutput, + (line, expectedLine) => { return line.Contains(expectedLine); }); + } + + if (expectedErrors != null) + { + this.Tracer.RelatedErrorEvents.ShouldContain( + expectedErrors, + (error, expectedError) => { return error.Contains(expectedError); }); + } + } + } +} diff --git a/GVFS/GVFS.UnitTests.Windows/Windows/Upgrader/UpgradeVerbTests.cs b/GVFS/GVFS.UnitTests.Windows/Windows/Upgrader/UpgradeVerbTests.cs new file mode 100644 index 000000000..bdade4207 --- /dev/null +++ b/GVFS/GVFS.UnitTests.Windows/Windows/Upgrader/UpgradeVerbTests.cs @@ -0,0 +1,247 @@ +using GVFS.CommandLine; +using GVFS.Common; +using GVFS.Tests.Should; +using GVFS.UnitTests.Category; +using GVFS.UnitTests.Windows.Mock.Upgrader; +using NUnit.Framework; +using System.Collections.Generic; + +namespace GVFS.UnitTests.Windows.Upgrader +{ + [TestFixture] + public class UpgradeVerbTests : UpgradeTests + { + private MockProcessLauncher ProcessWrapper { get; set; } + private UpgradeVerb UpgradeVerb { get; set; } + + [SetUp] + public override void Setup() + { + base.Setup(); + + this.ProcessWrapper = new MockProcessLauncher(exitCode: 0, hasExited: true, startResult: true); + this.UpgradeVerb = new UpgradeVerb( + this.Upgrader, + this.Tracer, + this.PrerunChecker, + this.ProcessWrapper, + this.Output); + this.UpgradeVerb.Confirmed = false; + this.PrerunChecker.CommandToRerun = "gvfs upgrade"; + } + + [TestCase] + public void UpgradeAvailabilityReporting() + { + this.ConfigureRunAndVerify( + configure: () => + { + this.Upgrader.PretendNewReleaseAvailableAtRemote( + upgradeVersion: NewerThanLocalVersion, + remoteRing: ProductUpgrader.RingType.Slow); + }, + expectedReturn: ReturnCode.Success, + expectedOutput: new List + { + "New GVFS version available: " + NewerThanLocalVersion, + "Run `gvfs upgrade --confirm` to install it" + }, + expectedErrors: null); + } + + [TestCase] + public void DowngradePrevention() + { + this.ConfigureRunAndVerify( + configure: () => + { + this.Upgrader.PretendNewReleaseAvailableAtRemote( + upgradeVersion: OlderThanLocalVersion, + remoteRing: ProductUpgrader.RingType.Slow); + }, + expectedReturn: ReturnCode.Success, + expectedOutput: new List + { + "Checking for GVFS upgrades...Succeeded", + "Great news, you're all caught up on upgrades in the Slow ring!" + }, + expectedErrors: null); + } + + [TestCase] + public void LaunchInstaller() + { + this.ConfigureRunAndVerify( + configure: () => + { + this.UpgradeVerb.Confirmed = true; + this.PrerunChecker.CommandToRerun = "gvfs upgrade --confirm"; + }, + expectedReturn: ReturnCode.Success, + expectedOutput: new List + { + "New GVFS version available: " + NewerThanLocalVersion, + "Launching upgrade tool...Succeeded" + }, + expectedErrors:null); + + this.ProcessWrapper.IsLaunched.ShouldBeTrue(); + } + + [TestCase] + [Category(CategoryConstants.ExceptionExpected)] + public override void NoneLocalRing() + { + base.NoneLocalRing(); + } + + [TestCase] + [Category(CategoryConstants.ExceptionExpected)] + public override void InvalidUpgradeRing() + { + base.InvalidUpgradeRing(); + } + + [TestCase] + [Category(CategoryConstants.ExceptionExpected)] + public void CopyTools() + { + this.ConfigureRunAndVerify( + configure: () => + { + this.Upgrader.SetFailOnAction(MockProductUpgrader.ActionType.CopyTools); + this.UpgradeVerb.Confirmed = true; + this.PrerunChecker.CommandToRerun = "gvfs upgrade --confirm"; + }, + expectedReturn: ReturnCode.GenericError, + expectedOutput: new List + { + "Could not launch upgrade tool. Unable to copy upgrader tools" + }, + expectedErrors: new List + { + "Could not launch upgrade tool. Unable to copy upgrader tools" + }); + } + + [TestCase] + public void ProjFSPreCheck() + { + this.ConfigureRunAndVerify( + configure: () => + { + this.UpgradeVerb.Confirmed = true; + this.PrerunChecker.SetReturnFalseOnCheck(MockInstallerPrerunChecker.FailOnCheckType.ProjFSEnabled); + }, + expectedReturn: ReturnCode.GenericError, + expectedOutput: new List + { + "ERROR: ProjFS configuration does not support `gvfs upgrade`.", + "Check your team's documentation for how to upgrade." + }, + expectedErrors: new List + { + "ProjFS configuration does not support `gvfs upgrade`." + }); + } + + [TestCase] + public void IsGVFSServiceRunningPreCheck() + { + this.ConfigureRunAndVerify( + configure: () => + { + this.UpgradeVerb.Confirmed = true; + this.PrerunChecker.SetReturnTrueOnCheck(MockInstallerPrerunChecker.FailOnCheckType.IsServiceInstalledAndNotRunning); + }, + expectedReturn: ReturnCode.GenericError, + expectedOutput: new List + { + "GVFS Service is not running.", + "Run `sc start GVFS.Service` and run `gvfs upgrade --confirm` again." + }, + expectedErrors: new List + { + "GVFS Service is not running." + }); + } + + [TestCase] + public void ElevatedRunPreCheck() + { + this.ConfigureRunAndVerify( + configure: () => + { + this.UpgradeVerb.Confirmed = true; + this.PrerunChecker.SetReturnFalseOnCheck(MockInstallerPrerunChecker.FailOnCheckType.IsElevated); + }, + expectedReturn: ReturnCode.GenericError, + expectedOutput: new List + { + "The installer needs to be run from an elevated command prompt.", + "Run `gvfs upgrade --confirm` again from an elevated command prompt." + }, + expectedErrors: new List + { + "The installer needs to be run from an elevated command prompt." + }); + } + + [TestCase] + public void UnAttendedModePreCheck() + { + this.ConfigureRunAndVerify( + configure: () => + { + this.UpgradeVerb.Confirmed = true; + this.PrerunChecker.SetReturnTrueOnCheck(MockInstallerPrerunChecker.FailOnCheckType.UnattendedMode); + }, + expectedReturn: ReturnCode.GenericError, + expectedOutput: new List + { + "`gvfs upgrade` is not supported in unattended mode" + }, + expectedErrors: new List + { + "`gvfs upgrade` is not supported in unattended mode" + }); + } + + [TestCase] + public void DeveloperMachinePreCheck() + { + this.ConfigureRunAndVerify( + configure: () => + { + this.UpgradeVerb.Confirmed = true; + this.PrerunChecker.SetReturnTrueOnCheck(MockInstallerPrerunChecker.FailOnCheckType.IsDevelopmentVersion); + }, + expectedReturn: ReturnCode.GenericError, + expectedOutput: new List + { + "Cannot run upgrade when development version of GVFS is installed." + }, + expectedErrors: new List + { + "Cannot run upgrade when development version of GVFS is installed." + }); + } + + protected override void RunUpgrade() + { + try + { + this.UpgradeVerb.Execute(); + } + catch (GVFSVerb.VerbAbortedException) + { + // ignore. exceptions are expected while simulating some failures. + } + } + + protected override ReturnCode ExitCode() + { + return this.UpgradeVerb.ReturnCode; + } + } +} \ No newline at end of file diff --git a/GVFS/GVFS.UnitTests/Mock/Common/MockPlatform.cs b/GVFS/GVFS.UnitTests/Mock/Common/MockPlatform.cs index b0cd6d43f..4fb290f50 100644 --- a/GVFS/GVFS.UnitTests/Mock/Common/MockPlatform.cs +++ b/GVFS/GVFS.UnitTests/Mock/Common/MockPlatform.cs @@ -90,6 +90,11 @@ public override bool IsProcessActive(int processId) throw new NotSupportedException(); } + public override void IsServiceInstalledAndRunning(string name, out bool installed, out bool running) + { + throw new NotSupportedException(); + } + public override bool TryGetGVFSEnlistmentRoot(string directory, out string enlistmentRoot, out string errorMessage) { throw new NotSupportedException(); diff --git a/GVFS/GVFS.UnitTests/Mock/Common/MockTracer.cs b/GVFS/GVFS.UnitTests/Mock/Common/MockTracer.cs index 17ecb7a93..d3bb8a5d9 100644 --- a/GVFS/GVFS.UnitTests/Mock/Common/MockTracer.cs +++ b/GVFS/GVFS.UnitTests/Mock/Common/MockTracer.cs @@ -1,4 +1,4 @@ -using GVFS.Common.Tracing; +using GVFS.Common.Tracing; using Newtonsoft.Json; using System; using System.Collections.Generic; @@ -15,12 +15,14 @@ public MockTracer() this.waitEvent = new AutoResetEvent(false); this.RelatedInfoEvents = new List(); this.RelatedWarningEvents = new List(); + this.RelatedErrorEvents = new List(); } public string WaitRelatedEventName { get; set; } public List RelatedInfoEvents { get; } public List RelatedWarningEvents { get; } + public List RelatedErrorEvents { get; } public void WaitForRelatedEvent() { @@ -71,18 +73,23 @@ public void RelatedWarning(string format, params object[] args) public void RelatedError(EventMetadata metadata, string message) { + metadata[TracingConstants.MessageKey.ErrorMessage] = message; + this.RelatedErrorEvents.Add(JsonConvert.SerializeObject(metadata)); } public void RelatedError(EventMetadata metadata, string message, Keywords keyword) { + this.RelatedError(metadata, message); } public void RelatedError(string message) { + this.RelatedErrorEvents.Add(message); } public void RelatedError(string format, params object[] args) { + this.RelatedErrorEvents.Add(string.Format(format, args)); } public ITracer StartActivity(string activityName, EventLevel level) diff --git a/GVFS/GVFS.Upgrader/App.config b/GVFS/GVFS.Upgrader/App.config new file mode 100644 index 000000000..00bfd114a --- /dev/null +++ b/GVFS/GVFS.Upgrader/App.config @@ -0,0 +1,6 @@ + + + + + + \ No newline at end of file diff --git a/GVFS/GVFS.Upgrader/GVFS.Upgrader.csproj b/GVFS/GVFS.Upgrader/GVFS.Upgrader.csproj new file mode 100644 index 000000000..8fb2f91ac --- /dev/null +++ b/GVFS/GVFS.Upgrader/GVFS.Upgrader.csproj @@ -0,0 +1,122 @@ + + + + + Debug + AnyCPU + {AECEC217-2499-403D-B0BB-2962B9BE5970} + Exe + GVFS.Upgrader + GVFS.Upgrader + v4.6.1 + 512 + true + publish\ + true + Disk + false + Foreground + 7 + Days + false + false + true + 0 + 1.0.0.%2a + false + false + true + + + + + true + ..\..\..\BuildOutput\GVFS.Upgrader\bin\x64\Debug\ + DEBUG;TRACE + true + full + x64 + prompt + MinimumRecommendedRules.ruleset + true + + + ..\..\..\BuildOutput\GVFS.Upgrader\bin\x64\Release\ + TRACE + true + true + pdbonly + x64 + prompt + MinimumRecommendedRules.ruleset + true + + + + ..\..\..\packages\Newtonsoft.Json.11.0.2\lib\net45\Newtonsoft.Json.dll + + + + + + + + + + + + + + CommonAssemblyVersion.cs + + + PlatformLoader.Windows.cs + + + + + + + + + Designer + + + + + False + Microsoft .NET Framework 4.6.1 %28x86 and x64%29 + true + + + False + .NET Framework 3.5 SP1 + false + + + + + + {374bf1e5-0b2d-4d4a-bd5e-4212299def09} + GVFS.Common + + + {4ce404e7-d3fc-471c-993c-64615861ea63} + GVFS.Platform.Windows + + + {f468b05a-95e5-46bc-8c67-b80a78527b7d} + GVFS.Virtualization + + + + + + + This project references NuGet package(s) that are missing on this computer. Use NuGet Package Restore to download them. For more information, see http://go.microsoft.com/fwlink/?LinkID=322105. The missing file is {0}. + + + + + + \ No newline at end of file diff --git a/GVFS/GVFS.Upgrader/Program.cs b/GVFS/GVFS.Upgrader/Program.cs new file mode 100644 index 000000000..9f61701b2 --- /dev/null +++ b/GVFS/GVFS.Upgrader/Program.cs @@ -0,0 +1,16 @@ +using GVFS.PlatformLoader; + +namespace GVFS.Upgrader +{ + public class Program + { + public static void Main(string[] args) + { + GVFSPlatformLoader.Initialize(); + + UpgradeOrchestrator upgrader = new UpgradeOrchestrator(); + + upgrader.Execute(); + } + } +} diff --git a/GVFS/GVFS.Upgrader/Properties/AssemblyInfo.cs b/GVFS/GVFS.Upgrader/Properties/AssemblyInfo.cs new file mode 100644 index 000000000..4fddbd3e3 --- /dev/null +++ b/GVFS/GVFS.Upgrader/Properties/AssemblyInfo.cs @@ -0,0 +1,22 @@ +using System.Reflection; +using System.Runtime.InteropServices; + +// General Information about an assembly is controlled through the following +// set of attributes. Change these attribute values to modify the information +// associated with an assembly. +[assembly: AssemblyTitle("GVFS.Upgrader")] +[assembly: AssemblyDescription("")] +[assembly: AssemblyConfiguration("")] +[assembly: AssemblyCompany("")] +[assembly: AssemblyProduct("GVFS.Upgrader")] +[assembly: AssemblyCopyright("Copyright © Microsoft 2018")] +[assembly: AssemblyTrademark("")] +[assembly: AssemblyCulture("")] + +// Setting ComVisible to false makes the types in this assembly not visible +// to COM components. If you need to access a type in this assembly from +// COM, set the ComVisible attribute to true on that type. +[assembly: ComVisible(false)] + +// The following GUID is for the ID of the typelib if this project is exposed to COM +[assembly: Guid("aecec217-2499-403d-b0bb-2962b9be5970")] diff --git a/GVFS/GVFS.Upgrader/UpgradeOrchestrator.cs b/GVFS/GVFS.Upgrader/UpgradeOrchestrator.cs new file mode 100644 index 000000000..18780cf3d --- /dev/null +++ b/GVFS/GVFS.Upgrader/UpgradeOrchestrator.cs @@ -0,0 +1,389 @@ +using GVFS.Common; +using GVFS.Common.Git; +using GVFS.Common.Tracing; +using System; +using System.IO; + +namespace GVFS.Upgrader +{ + public class UpgradeOrchestrator + { + private const EventLevel DefaultEventLevel = EventLevel.Informational; + + private ProductUpgrader upgrader; + private ITracer tracer; + private InstallerPreRunChecker preRunChecker; + private TextWriter output; + private TextReader input; + private bool remount; + private bool shouldExitOnError; + + public UpgradeOrchestrator( + ProductUpgrader upgrader, + ITracer tracer, + InstallerPreRunChecker preRunChecker, + TextReader input, + TextWriter output, + bool shouldExitOnError) + { + this.upgrader = upgrader; + this.tracer = tracer; + this.preRunChecker = preRunChecker; + this.output = output; + this.input = input; + this.remount = false; + this.shouldExitOnError = shouldExitOnError; + this.ExitCode = ReturnCode.Success; + } + + public UpgradeOrchestrator() + { + string logFilePath = GVFSEnlistment.GetNewGVFSLogFileName( + ProductUpgrader.GetLogDirectoryPath(), + GVFSConstants.LogFileTypes.UpgradeProcess); + JsonTracer jsonTracer = new JsonTracer(GVFSConstants.GVFSEtwProviderName, "UpgradeProcess"); + jsonTracer.AddLogFileEventListener( + logFilePath, + DefaultEventLevel, + Keywords.Any); + + this.tracer = jsonTracer; + this.preRunChecker = new InstallerPreRunChecker(this.tracer); + this.upgrader = new ProductUpgrader(ProcessHelper.GetCurrentProcessVersion(), this.tracer); + this.output = Console.Out; + this.input = Console.In; + this.remount = false; + this.shouldExitOnError = false; + this.ExitCode = ReturnCode.Success; + } + + public ReturnCode ExitCode { get; private set; } + + public void Execute() + { + string error = null; + + if (this.upgrader.IsNoneRing()) + { + this.output.WriteLine(GVFSConstants.UpgradeVerbMessages.NoneRingConsoleAlert); + this.output.WriteLine(GVFSConstants.UpgradeVerbMessages.SetUpgradeRingCommand); + } + else + { + try + { + Version newVersion = null; + if (!this.TryRunUpgradeInstall(out newVersion, out error)) + { + this.ExitCode = ReturnCode.GenericError; + } + } + finally + { + string remountError = null; + if (!this.TryRemountRepositories(out remountError)) + { + remountError = Environment.NewLine + "WARNING: " + remountError; + this.output.WriteLine(remountError); + this.ExitCode = ReturnCode.Success; + } + + this.DeletedDownloadedAssets(); + } + } + + if (this.ExitCode == ReturnCode.GenericError) + { + error = Environment.NewLine + "ERROR: " + error; + this.output.WriteLine(error); + } + + if (this.input == Console.In) + { + this.output.WriteLine("Press Enter to exit."); + this.input.ReadLine(); + } + + if (this.shouldExitOnError) + { + Environment.Exit((int)this.ExitCode); + } + } + + private bool LaunchInsideSpinner(Func method, string message) + { + return ConsoleHelper.ShowStatusWhileRunning( + method, + message, + this.output, + this.output == Console.Out && !GVFSPlatform.Instance.IsConsoleOutputRedirectedToFile(), + null); + } + + private bool TryRunUpgradeInstall(out Version newVersion, out string consoleError) + { + newVersion = null; + + Version newGVFSVersion = null; + GitVersion newGitVersion = null; + string errorMessage = null; + if (!this.LaunchInsideSpinner( + () => + { + if (!this.TryCheckIfUpgradeAvailable(out newGVFSVersion, out errorMessage) || + !this.TryGetNewGitVersion(out newGitVersion, out errorMessage)) + { + return false; + } + + this.LogInstalledVersionInfo(); + this.LogVersionInfo(newGVFSVersion, newGitVersion, "Available Version"); + + this.preRunChecker.CommandToRerun = "gvfs upgrade --confirm"; + if (!this.preRunChecker.TryRunPreUpgradeChecks(out errorMessage)) + { + return false; + } + + if (!this.TryDownloadUpgrade(newGVFSVersion, out errorMessage)) + { + return false; + } + + return true; + }, + "Downloading")) + { + consoleError = errorMessage; + return false; + } + + if (!this.LaunchInsideSpinner( + () => + { + if (!this.preRunChecker.TryUnmountAllGVFSRepos(out errorMessage)) + { + return false; + } + + this.remount = true; + + return true; + }, + "Unmounting repositories")) + { + consoleError = errorMessage; + return false; + } + + if (!this.LaunchInsideSpinner( + () => + { + if (!this.TryInstallGitUpgrade(newGitVersion, out errorMessage)) + { + return false; + } + + return true; + }, + $"Installing Git version: {newGitVersion}")) + { + consoleError = errorMessage; + return false; + } + + if (!this.LaunchInsideSpinner( + () => + { + if (!this.TryInstallGVFSUpgrade(newGVFSVersion, out errorMessage)) + { + return false; + } + + return true; + }, + $"Installing VFSForGit version: {newGVFSVersion}")) + { + consoleError = errorMessage; + return false; + } + + this.LogVersionInfo(newGVFSVersion, newGitVersion, "Newly Installed Version"); + + newVersion = newGVFSVersion; + consoleError = null; + return true; + } + + private bool TryRemountRepositories(out string consoleError) + { + string errorMessage = string.Empty; + if (this.remount && !this.LaunchInsideSpinner( + () => + { + string remountError; + if (!this.preRunChecker.TryMountAllGVFSRepos(out remountError)) + { + EventMetadata metadata = new EventMetadata(); + metadata.Add("Upgrade Step", nameof(this.TryRemountRepositories)); + metadata.Add("Remount Error", remountError); + this.tracer.RelatedError(metadata, $"{nameof(this.preRunChecker.TryMountAllGVFSRepos)} failed."); + errorMessage += remountError; + return false; + } + + return true; + }, + "Mounting repositories")) + { + consoleError = errorMessage; + return false; + } + + consoleError = null; + return true; + } + + private void DeletedDownloadedAssets() + { + string downloadsCleanupError; + if (!this.upgrader.TryCleanup(out downloadsCleanupError)) + { + EventMetadata metadata = new EventMetadata(); + metadata.Add("Upgrade Step", nameof(this.DeletedDownloadedAssets)); + metadata.Add("Download cleanup error", downloadsCleanupError); + this.tracer.RelatedError(metadata, $"{nameof(this.DeletedDownloadedAssets)} failed."); + } + } + + private bool TryGetNewGitVersion(out GitVersion gitVersion, out string consoleError) + { + gitVersion = null; + + this.tracer.RelatedInfo("Reading Git version from release info"); + + if (!this.upgrader.TryGetGitVersion(out gitVersion, out consoleError)) + { + EventMetadata metadata = new EventMetadata(); + metadata.Add("Upgrade Step", nameof(this.TryGetNewGitVersion)); + this.tracer.RelatedError(metadata, $"{nameof(this.upgrader.TryGetGitVersion)} failed. {consoleError}"); + return false; + } + + this.tracer.RelatedInfo("Successfully read Git version {0}", gitVersion); + + return true; + } + + private bool TryCheckIfUpgradeAvailable(out Version newestVersion, out string consoleError) + { + newestVersion = null; + + this.tracer.RelatedInfo("Checking upgrade server for new releases"); + + if (!this.upgrader.TryGetNewerVersion(out newestVersion, out consoleError)) + { + EventMetadata metadata = new EventMetadata(); + metadata.Add("Upgrade Step", nameof(this.TryCheckIfUpgradeAvailable)); + this.tracer.RelatedError(metadata, $"{nameof(this.upgrader.TryGetNewerVersion)} failed. {consoleError}"); + return false; + } + + if (newestVersion == null) + { + consoleError = "No upgrades available in ring: " + this.upgrader.Ring; + this.tracer.RelatedInfo("No new upgrade releases available"); + return false; + } + + this.tracer.RelatedInfo("Successfully checked for new release. {0}", newestVersion); + + return true; + } + + private bool TryDownloadUpgrade(Version version, out string consoleError) + { + this.tracer.RelatedInfo("Downloading version: " + version.ToString()); + + if (!this.upgrader.TryDownloadNewestVersion(out consoleError)) + { + EventMetadata metadata = new EventMetadata(); + metadata.Add("Upgrade Step", nameof(this.TryDownloadUpgrade)); + this.tracer.RelatedError(metadata, $"{nameof(this.upgrader.TryDownloadNewestVersion)} failed. {consoleError}"); + return false; + } + + this.tracer.RelatedInfo("Successfully downloaded version: " + version.ToString()); + + return true; + } + + private bool TryInstallGitUpgrade(GitVersion version, out string consoleError) + { + this.tracer.RelatedInfo("Installing Git version: " + version.ToString()); + + bool installSuccess = false; + if (!this.upgrader.TryRunGitInstaller(out installSuccess, out consoleError) || + !installSuccess) + { + EventMetadata metadata = new EventMetadata(); + metadata.Add("Upgrade Step", nameof(this.TryInstallGitUpgrade)); + this.tracer.RelatedError(metadata, $"{nameof(this.upgrader.TryRunGitInstaller)} failed. {consoleError}"); + return false; + } + + this.tracer.RelatedInfo("Successfully installed Git version: " + version.ToString()); + + return installSuccess; + } + + private bool TryInstallGVFSUpgrade(Version version, out string consoleError) + { + this.tracer.RelatedInfo("Installing GVFS version: " + version.ToString()); + + bool installSuccess = false; + if (!this.upgrader.TryRunGVFSInstaller(out installSuccess, out consoleError) || + !installSuccess) + { + EventMetadata metadata = new EventMetadata(); + metadata.Add("Upgrade Step", nameof(this.TryInstallGVFSUpgrade)); + this.tracer.RelatedError(metadata, $"{nameof(this.upgrader.TryRunGVFSInstaller)} failed. {consoleError}"); + return false; + } + + this.tracer.RelatedInfo("Successfully installed GVFS version: " + version.ToString()); + + return installSuccess; + } + + private void LogVersionInfo( + Version gvfsVersion, + GitVersion gitVersion, + string message) + { + EventMetadata metadata = new EventMetadata(); + metadata.Add(nameof(gvfsVersion), gvfsVersion.ToString()); + metadata.Add(nameof(gitVersion), gitVersion.ToString()); + + this.tracer.RelatedEvent(EventLevel.Informational, message, metadata); + } + + private void LogInstalledVersionInfo() + { + EventMetadata metadata = new EventMetadata(); + string installedGVFSVersion = ProcessHelper.GetCurrentProcessVersion(); + metadata.Add(nameof(installedGVFSVersion), installedGVFSVersion); + + GitVersion installedGitVersion = null; + string error = null; + if (GitProcess.TryGetVersion( + out installedGitVersion, + out error)) + { + metadata.Add(nameof(installedGitVersion), installedGitVersion.ToString()); + } + + this.tracer.RelatedEvent(EventLevel.Informational, "Installed Version", metadata); + } + } +} \ No newline at end of file diff --git a/GVFS/GVFS.Upgrader/packages.config b/GVFS/GVFS.Upgrader/packages.config new file mode 100644 index 000000000..88e238819 --- /dev/null +++ b/GVFS/GVFS.Upgrader/packages.config @@ -0,0 +1,6 @@ + + + + + + \ No newline at end of file diff --git a/GVFS/GVFS/CommandLine/DiagnoseVerb.cs b/GVFS/GVFS/CommandLine/DiagnoseVerb.cs index 138259e8c..b2e932f96 100644 --- a/GVFS/GVFS/CommandLine/DiagnoseVerb.cs +++ b/GVFS/GVFS/CommandLine/DiagnoseVerb.cs @@ -1,4 +1,4 @@ -using CommandLine; +using CommandLine; using GVFS.Common; using GVFS.Common.FileSystem; using GVFS.Common.Git; @@ -113,6 +113,14 @@ protected override void Execute(GVFSEnlistment enlistment) this.ServiceName, copySubFolders: true); + // upgrader + this.CopyAllFiles( + ProductUpgrader.GetUpgradesDirectoryPath(), + archiveFolderPath, + ProductUpgrader.LogDirectory, + copySubFolders: true, + targetFolderName: ProductUpgrader.UpgradeDirectoryName); + return true; }, "Copying logs"); @@ -160,10 +168,16 @@ private void RecordVersionInformation() this.diagnosticLogFileWriter.WriteLine(information); } - private void CopyAllFiles(string sourceRoot, string targetRoot, string folderName, bool copySubFolders, bool hideErrorsFromStdout = false) + private void CopyAllFiles( + string sourceRoot, + string targetRoot, + string folderName, + bool copySubFolders, + bool hideErrorsFromStdout = false, + string targetFolderName = null) { string sourceFolder = Path.Combine(sourceRoot, folderName); - string targetFolder = Path.Combine(targetRoot, folderName); + string targetFolder = Path.Combine(targetRoot, targetFolderName ?? folderName); try { diff --git a/GVFS/GVFS/CommandLine/GVFSVerb.cs b/GVFS/GVFS/CommandLine/GVFSVerb.cs index 54f3fe204..eb4370621 100644 --- a/GVFS/GVFS/CommandLine/GVFSVerb.cs +++ b/GVFS/GVFS/CommandLine/GVFSVerb.cs @@ -1,4 +1,4 @@ -using CommandLine; +using CommandLine; using GVFS.Common; using GVFS.Common.FileSystem; using GVFS.Common.Git; @@ -642,8 +642,7 @@ private void CheckGitVersion(ITracer tracer, GVFSEnlistment enlistment, out stri this.ReportErrorAndExit(tracer, "Error: Invalid version of git {0}. Must use gvfs version.", version); } - Version gvfsVersion = new Version(ProcessHelper.GetCurrentProcessVersion()); - if (gvfsVersion.Major == 0) + if (ProcessHelper.IsDevelopmentVersion()) { if (gitVersion.IsLessThan(GVFSConstants.SupportedGitVersion)) { diff --git a/GVFS/GVFS/CommandLine/LogVerb.cs b/GVFS/GVFS/CommandLine/LogVerb.cs index da93d7614..d1eb5c9c4 100644 --- a/GVFS/GVFS/CommandLine/LogVerb.cs +++ b/GVFS/GVFS/CommandLine/LogVerb.cs @@ -1,4 +1,4 @@ -using CommandLine; +using CommandLine; using GVFS.Common; using System.IO; using System.Linq; @@ -60,6 +60,9 @@ public override void Execute() string serviceLogsRoot = Paths.GetServiceLogsPath(this.ServiceName); this.DisplayMostRecent(serviceLogsRoot, GVFSConstants.LogFileTypes.Service); + + string autoUpgradeLogsRoot = Paths.GetServiceLogsPath(ProductUpgrader.UpgradeDirectoryName); + this.DisplayMostRecent(autoUpgradeLogsRoot, GVFSConstants.LogFileTypes.UpgradePrefix); } else { diff --git a/GVFS/GVFS/CommandLine/UpgradeVerb.cs b/GVFS/GVFS/CommandLine/UpgradeVerb.cs new file mode 100644 index 000000000..e914842a8 --- /dev/null +++ b/GVFS/GVFS/CommandLine/UpgradeVerb.cs @@ -0,0 +1,325 @@ +using CommandLine; +using GVFS.Common; +using GVFS.Common.Git; +using GVFS.Common.Tracing; +using GVFS.Upgrader; +using System; +using System.Diagnostics; +using System.IO; + +namespace GVFS.CommandLine +{ + [Verb(UpgradeVerbName, HelpText = "Checks if a new GVFS release is available.")] + public class UpgradeVerb : GVFSVerb + { + private const string UpgradeVerbName = "upgrade"; + private ITracer tracer; + private ProductUpgrader upgrader; + private InstallerPreRunChecker prerunChecker; + private ProcessLauncher processWrapper; + + public UpgradeVerb( + ProductUpgrader upgrader, + ITracer tracer, + InstallerPreRunChecker prerunChecker, + ProcessLauncher processWrapper, + TextWriter output) + { + this.upgrader = upgrader; + this.tracer = tracer; + this.prerunChecker = prerunChecker; + this.processWrapper = processWrapper; + this.Output = output; + } + + public UpgradeVerb() + { + JsonTracer jsonTracer = new JsonTracer(GVFSConstants.GVFSEtwProviderName, "UpgradeVerb"); + string logFilePath = GVFSEnlistment.GetNewGVFSLogFileName( + ProductUpgrader.GetLogDirectoryPath(), + GVFSConstants.LogFileTypes.UpgradeVerb); + jsonTracer.AddLogFileEventListener(logFilePath, EventLevel.Informational, Keywords.Any); + + this.tracer = jsonTracer; + this.prerunChecker = new InstallerPreRunChecker(this.tracer); + this.processWrapper = new ProcessLauncher(); + this.Output = Console.Out; + this.upgrader = new ProductUpgrader(ProcessHelper.GetCurrentProcessVersion(), this.tracer); + } + + [Option( + "confirm", + Default = false, + Required = false, + HelpText = "Pass in this flag to actually install the newest release")] + public bool Confirmed { get; set; } + + public override string EnlistmentRootPathParameter { get; set; } + + protected override string VerbName + { + get { return UpgradeVerbName; } + } + + public override void Execute() + { + ReturnCode exitCode = ReturnCode.Success; + if (!this.TryRunProductUpgrade()) + { + exitCode = ReturnCode.GenericError; + this.ReportErrorAndExit(this.tracer, exitCode, string.Empty); + } + } + + private bool TryRunProductUpgrade() + { + string errorOutputFormat = Environment.NewLine + "ERROR: {0}"; + string error = null; + Version newestVersion = null; + bool isInstallable = false; + + if (this.upgrader.IsNoneRing()) + { + this.tracer.RelatedInfo($"{nameof(this.TryRunProductUpgrade)}: {GVFSConstants.UpgradeVerbMessages.NoneRingConsoleAlert}"); + this.ReportInfoToConsole(GVFSConstants.UpgradeVerbMessages.NoneRingConsoleAlert); + this.ReportInfoToConsole(GVFSConstants.UpgradeVerbMessages.SetUpgradeRingCommand); + return true; + } + + if (!this.TryRunUpgradeChecks(out newestVersion, out isInstallable, out error)) + { + this.Output.WriteLine(errorOutputFormat, error); + this.tracer.RelatedError($"{nameof(this.TryRunProductUpgrade)}: Upgrade checks failed. {error}"); + return false; + } + + if (newestVersion == null) + { + this.ReportInfoToConsole($"Great news, you're all caught up on upgrades in the {this.upgrader.Ring} ring!"); + return true; + } + + this.ReportInfoToConsole("New GVFS version available: {0}", newestVersion.ToString()); + + if (!this.Confirmed && isInstallable) + { + this.ReportInfoToConsole("Run `gvfs upgrade --confirm` to install it"); + return true; + } + + if (!isInstallable) + { + this.tracer.RelatedError($"{nameof(this.TryRunProductUpgrade)}: {error}"); + this.Output.WriteLine(errorOutputFormat, error); + return false; + } + + if (!this.TryRunInstaller(out error)) + { + this.tracer.RelatedError($"{nameof(this.TryRunProductUpgrade)}: Could not launch upgrade tool. {error}"); + this.Output.WriteLine(errorOutputFormat, "Could not launch upgrade tool. " + error); + return false; + } + + return true; + } + + private bool TryRunUpgradeChecks( + out Version latestVersion, + out bool isUpgradeInstallable, + out string consoleError) + { + bool upgradeCheckSuccess = false; + bool upgradeIsInstallable = false; + string errorMessage = null; + Version version = null; + + this.ShowStatusWhileRunning( + () => + { + upgradeCheckSuccess = this.TryCheckUpgradeAvailable(out version, out errorMessage); + if (upgradeCheckSuccess && version != null) + { + upgradeIsInstallable = true; + + if (!this.TryCheckUpgradeInstallable(out errorMessage)) + { + upgradeIsInstallable = false; + } + } + + return upgradeCheckSuccess; + }, + "Checking for GVFS upgrades", + suppressGvfsLogMessage: true); + + latestVersion = version; + isUpgradeInstallable = upgradeIsInstallable; + consoleError = errorMessage; + + return upgradeCheckSuccess; + } + + private bool TryRunInstaller(out string consoleError) + { + string upgraderPath = null; + string errorMessage = null; + + bool preUpgradeSuccess = this.ShowStatusWhileRunning( + () => + { + if (this.TryCopyUpgradeTool(out upgraderPath, out errorMessage) && + this.TryLaunchUpgradeTool(upgraderPath, out errorMessage)) + { + return true; + } + + return false; + }, + "Launching upgrade tool", + suppressGvfsLogMessage: true); + + if (!preUpgradeSuccess) + { + consoleError = errorMessage; + return false; + } + + consoleError = null; + return true; + } + + private bool TryCopyUpgradeTool(out string upgraderExePath, out string consoleError) + { + upgraderExePath = null; + + this.tracer.RelatedInfo("Copying upgrade tool"); + + if (!this.upgrader.TrySetupToolsDirectory(out upgraderExePath, out consoleError)) + { + return false; + } + + this.tracer.RelatedInfo($"Successfully Copied upgrade tool to {upgraderExePath}"); + + return true; + } + + private bool TryLaunchUpgradeTool(string path, out string consoleError) + { + this.tracer.RelatedInfo("Launching upgrade tool"); + + Exception exception; + if (!this.processWrapper.TryStart(path, out exception)) + { + if (exception != null) + { + consoleError = exception.Message; + this.tracer.RelatedError($"Error launching upgrade tool. {exception.ToString()}"); + } + else + { + consoleError = $"Error launching upgrade tool"; + } + + return false; + } + + this.tracer.RelatedInfo("Successfully launched upgrade tool."); + + consoleError = null; + return true; + } + + private bool TryCheckUpgradeAvailable( + out Version latestVersion, + out string consoleError) + { + latestVersion = null; + consoleError = null; + + this.tracer.RelatedInfo("Checking server for available upgrades."); + + bool checkSucceeded = false; + Version version = null; + + checkSucceeded = this.upgrader.TryGetNewerVersion(out version, out consoleError); + if (!checkSucceeded) + { + return false; + } + + latestVersion = version; + + this.tracer.RelatedInfo("Successfully checked server for GVFS upgrades."); + + return true; + } + + private bool TryCheckUpgradeInstallable(out string consoleError) + { + consoleError = null; + + this.tracer.RelatedInfo("Checking if upgrade is installable on this machine."); + + this.prerunChecker.CommandToRerun = this.Confirmed ? "gvfs upgrade --confirm" : "gvfs upgrade"; + + if (!this.prerunChecker.TryRunPreUpgradeChecks( + out consoleError)) + { + return false; + } + + this.tracer.RelatedInfo("Upgrade is installable."); + + return true; + } + + private void ReportInfoToConsole(string message, params object[] args) + { + this.Output.WriteLine(message, args); + } + + public class ProcessLauncher + { + public ProcessLauncher() + { + this.Process = new Process(); + } + + public Process Process { get; private set; } + + public virtual bool HasExited + { + get { return this.Process.HasExited; } + } + + public virtual int ExitCode + { + get { return this.Process.ExitCode; } + } + + public virtual bool TryStart(string path, out Exception exception) + { + this.Process.StartInfo = new ProcessStartInfo(path) + { + UseShellExecute = true, + WindowStyle = ProcessWindowStyle.Normal + }; + + exception = null; + + try + { + return this.Process.Start(); + } + catch (Exception ex) + { + exception = ex; + } + + return false; + } + } + } +} \ No newline at end of file diff --git a/GVFS/GVFS/GVFS.Windows.csproj b/GVFS/GVFS/GVFS.Windows.csproj index 306a8073c..d8de69db9 100644 --- a/GVFS/GVFS/GVFS.Windows.csproj +++ b/GVFS/GVFS/GVFS.Windows.csproj @@ -1,4 +1,4 @@ - + @@ -80,6 +80,7 @@ + diff --git a/GVFS/GVFS/Program.cs b/GVFS/GVFS/Program.cs index f906114dc..43061a95a 100644 --- a/GVFS/GVFS/Program.cs +++ b/GVFS/GVFS/Program.cs @@ -1,4 +1,4 @@ -using CommandLine; +using CommandLine; using GVFS.CommandLine; using GVFS.Common; using GVFS.PlatformLoader; @@ -27,6 +27,7 @@ public static void Main(string[] args) typeof(ServiceVerb), typeof(StatusVerb), typeof(UnmountVerb), + typeof(UpgradeVerb) }; int consoleWidth = 80; @@ -52,7 +53,7 @@ public static void Main(string[] args) settings.CaseSensitive = false; settings.EnableDashDash = true; settings.IgnoreUnknownArguments = false; - settings.HelpWriter = Console.Error; + settings.HelpWriter = Console.Error; settings.MaximumDisplayWidth = consoleWidth; }) .ParseArguments(args, verbTypes) @@ -80,6 +81,14 @@ public static void Main(string[] args) service.Execute(); Environment.Exit((int)ReturnCode.Success); }) + .WithParsed( + upgrade => + { + // The upgrade verb doesn't operate on a repo, so it doesn't use the enlistment + // path at all. + upgrade.Execute(); + Environment.Exit((int)ReturnCode.Success); + }) .WithParsed( verb => { diff --git a/Scripts/UninstallGVFS.bat b/Scripts/UninstallGVFS.bat index c22ccab38..27ecc7a9e 100644 --- a/Scripts/UninstallGVFS.bat +++ b/Scripts/UninstallGVFS.bat @@ -17,4 +17,6 @@ for /F "delims=" %%f in ('dir "c:\Program Files\GVFS\unins*.exe" /B /S /O:-D') d :deleteGVFS rmdir /q/s "c:\Program Files\GVFS" +rmdir /q/s "C:\ProgramData\GVFS\GVFS.Upgrade" + :end From d0668d481dd4074e55204ae6b4761c87df8d415a Mon Sep 17 00:00:00 2001 From: Al Ameen Shah Date: Wed, 19 Sep 2018 15:33:34 -0400 Subject: [PATCH 200/272] Incorporating suggestions from test feedback. - Display nag only 10% of the times a git command is run - Removed "Run gvfs upgrade..." from the reminder notification message. - Use ITracer.StartActivity for method tracing. - Service Verb redirects list of repos that failed to mount to stderr. - Upgrader tool reads stderr and displays the list on Console. - New console message displayed after launching installer advising user to not run gvfs, git commands until installation has finished. - Added a new Console message to let user know upgrade completed successfully - inserted blank line before final instruction to run `gvfs upgrade --confirm` command - Early exit when no upgrade or invalid upgade ring is set - Upgrade verb crash fix on Mac. - Early exit when upgrade is not installable and `gvfs upgrade --confirm` is run --- GVFS/GVFS.Common/GVFSConstants.cs | 10 +- GVFS/GVFS.Common/InstallerPreRunChecker.cs | 106 ++++---- GVFS/GVFS.Common/ProductUpgrader.cs | 21 +- .../GVFSUpgradeReminderTests.cs | 27 +- GVFS/GVFS.Hooks/Program.cs | 28 ++- .../Mock/MockInstallerPreRunChecker.cs | 4 +- .../Windows/Mock/MockProductUpgrader.cs | 2 +- .../Windows/Upgrader/UpgradeTests.cs | 2 +- .../Windows/Upgrader/UpgradeVerbTests.cs | 2 +- GVFS/GVFS.Upgrader/UpgradeOrchestrator.cs | 163 +++++++----- GVFS/GVFS/CommandLine/ServiceVerb.cs | 23 +- GVFS/GVFS/CommandLine/UpgradeVerb.cs | 234 +++++++++++------- start | 1 + 13 files changed, 386 insertions(+), 237 deletions(-) create mode 100644 start diff --git a/GVFS/GVFS.Common/GVFSConstants.cs b/GVFS/GVFS.Common/GVFSConstants.cs index a88200be6..b9f6c56d2 100644 --- a/GVFS/GVFS.Common/GVFSConstants.cs +++ b/GVFS/GVFS.Common/GVFSConstants.cs @@ -222,10 +222,12 @@ public static class Unmount public static class UpgradeVerbMessages { - public const string NoneRingConsoleAlert = "Upgrade ring set to None. No upgrade check was performed."; - public const string SetUpgradeRingCommand = "To set or change upgrade ring, run `git config --global gvfs.upgradering [\"Fast\"|\"Slow\"|\"None\"] from an elevated command prompt."; - public const string UpgradeAvailable = "A newer version of GVFS is available."; - public const string UpgradeInstallAdvice = "Run `gvfs upgrade --confirm` from an elevated command prompt to install."; + public const string NoneRingConsoleAlert = "Upgrade ring set to \"None\". No upgrade check was performed."; + public const string InvalidRingConsoleAlert = "Upgrade ring is not set. No upgrade check was performed."; + public const string SetUpgradeRingCommand = "To set or change upgrade ring, run `git config --global gvfs.upgradering [\"Fast\"|\"Slow\"|\"None\"] from a command prompt."; + public const string ReminderNotification = "A new version of GVFS is available. Run `gvfs upgrade` to start the upgrade."; + public const string UnmountRepoWarning = "The upgrade process will unmount all GVFS repositories for several minutes and remount them when it is complete. Ensure you are at a stopping point prior to upgrading."; + public const string UpgradeInstallAdvice = "When you are ready, run `gvfs upgrade --confirm` from an elevated command prompt."; } } } diff --git a/GVFS/GVFS.Common/InstallerPreRunChecker.cs b/GVFS/GVFS.Common/InstallerPreRunChecker.cs index e92c12742..aa5f919e5 100644 --- a/GVFS/GVFS.Common/InstallerPreRunChecker.cs +++ b/GVFS/GVFS.Common/InstallerPreRunChecker.cs @@ -26,29 +26,30 @@ public InstallerPreRunChecker(ITracer tracer) public bool TryRunPreUpgradeChecks(out string consoleError) { - this.tracer.RelatedInfo("Checking if GVFS upgrade can be run on this machine."); - - if (this.IsUnattended()) + using (ITracer activity = this.tracer.StartActivity(nameof(this.TryRunPreUpgradeChecks), EventLevel.Informational)) { - consoleError = "`gvfs upgrade` is not supported in unattended mode"; - this.tracer.RelatedError($"{nameof(TryRunPreUpgradeChecks)}: {consoleError}"); - return false; - } + if (this.IsUnattended()) + { + consoleError = "`gvfs upgrade` is not supported in unattended mode"; + this.tracer.RelatedError($"{nameof(TryRunPreUpgradeChecks)}: {consoleError}"); + return false; + } - if (this.IsDevelopmentVersion()) - { - consoleError = "Cannot run upgrade when development version of GVFS is installed."; - this.tracer.RelatedError($"{nameof(TryRunPreUpgradeChecks)}: {consoleError}"); - return false; - } + if (this.IsDevelopmentVersion()) + { + consoleError = "Cannot run upgrade when development version of GVFS is installed."; + this.tracer.RelatedError($"{nameof(TryRunPreUpgradeChecks)}: {consoleError}"); + return false; + } - if (!this.IsGVFSUpgradeAllowed(out consoleError)) - { - this.tracer.RelatedError($"{nameof(TryRunPreUpgradeChecks)}: {consoleError}"); - return false; - } + if (!this.IsGVFSUpgradeAllowed(out consoleError)) + { + this.tracer.RelatedError($"{nameof(TryRunPreUpgradeChecks)}: {consoleError}"); + return false; + } - this.tracer.RelatedInfo("Successfully finished pre upgrade checks. Okay to run GVFS upgrade."); + activity.RelatedInfo("Successfully finished pre upgrade checks. Okay to run GVFS upgrade."); + } consoleError = null; return true; @@ -57,7 +58,7 @@ public bool TryRunPreUpgradeChecks(out string consoleError) // TODO: Move repo mount calls to GVFS.Upgrader project. public bool TryMountAllGVFSRepos(out string consoleError) { - return this.TryRunGVFSWithArgs("service --mount-all", out consoleError); + return this.TryRunGVFSWithArgs("service --mount-all --log-mount-failure-in-stderr", out consoleError); } public bool TryUnmountAllGVFSRepos(out string consoleError) @@ -66,43 +67,46 @@ public bool TryUnmountAllGVFSRepos(out string consoleError) this.tracer.RelatedInfo("Unmounting any mounted GVFS repositories."); - if (!this.TryRunGVFSWithArgs("service --unmount-all", out consoleError)) + using (ITracer activity = this.tracer.StartActivity(nameof(this.TryUnmountAllGVFSRepos), EventLevel.Informational)) { - this.tracer.RelatedError($"{nameof(TryUnmountAllGVFSRepos)}: {consoleError}"); - return false; - } + if (!this.TryRunGVFSWithArgs("service --unmount-all --log-mount-failure-in-stderr", out consoleError)) + { + this.tracer.RelatedError($"{nameof(TryUnmountAllGVFSRepos)}: {consoleError}"); + return false; + } - // While checking for blocking processes like GVFS.Mount immediately after un-mounting, - // then sometimes GVFS.Mount shows up as running. But if the check is done after waiting - // for some time, then eventually GVFS.Mount goes away. The retry loop below is to help - // account for this delay between the time un-mount call returns and when GVFS.Mount - // actually quits. - this.tracer.RelatedInfo("Checking if GVFS or dependent processes are running."); - int retryCount = 10; - List processList = null; - while (retryCount > 0) - { - if (!this.IsBlockingProcessRunning(out processList)) + // While checking for blocking processes like GVFS.Mount immediately after un-mounting, + // then sometimes GVFS.Mount shows up as running. But if the check is done after waiting + // for some time, then eventually GVFS.Mount goes away. The retry loop below is to help + // account for this delay between the time un-mount call returns and when GVFS.Mount + // actually quits. + this.tracer.RelatedInfo("Checking if GVFS or dependent processes are running."); + int retryCount = 10; + List processList = null; + while (retryCount > 0) { - break; + if (!this.IsBlockingProcessRunning(out processList)) + { + break; + } + + Thread.Sleep(TimeSpan.FromMilliseconds(250)); + retryCount--; } - Thread.Sleep(TimeSpan.FromMilliseconds(250)); - retryCount--; - } + if (processList.Count > 0) + { + consoleError = string.Join( + Environment.NewLine, + "Blocking processes are running.", + $"Run `{this.CommandToRerun}` again after quitting these processes - " + string.Join(", ", processList.ToArray())); + this.tracer.RelatedError($"{nameof(TryUnmountAllGVFSRepos)}: {consoleError}"); + return false; + } - if (processList.Count > 0) - { - consoleError = string.Join( - Environment.NewLine, - "Blocking processes are running.", - $"Run `{this.CommandToRerun}` again after quitting these processes - " + string.Join(", ", processList.ToArray())); - this.tracer.RelatedError($"{nameof(TryUnmountAllGVFSRepos)}: {consoleError}"); - return false; + activity.RelatedInfo("Successfully unmounted repositories."); } - this.tracer.RelatedInfo("Successfully unmounted repositories."); - return true; } @@ -168,9 +172,7 @@ protected virtual bool TryRunGVFSWithArgs(string args, out string consoleError) } else { - string output = string.IsNullOrEmpty(processResult.Output) ? string.Empty : processResult.Output; - string errorString = string.IsNullOrEmpty(processResult.Errors) ? "GVFS error" : processResult.Errors; - consoleError = string.Format("{0}. {1}", errorString, output); + consoleError = string.IsNullOrEmpty(processResult.Errors) ? $"GVFS {args} failed." : processResult.Errors; return false; } } diff --git a/GVFS/GVFS.Common/ProductUpgrader.cs b/GVFS/GVFS.Common/ProductUpgrader.cs index 96f8dc683..b8b411a0c 100644 --- a/GVFS/GVFS.Common/ProductUpgrader.cs +++ b/GVFS/GVFS.Common/ProductUpgrader.cs @@ -71,12 +71,7 @@ public enum RingType } public RingType Ring { get; protected set; } - - public bool IsNoneRing() - { - return this.TryLoadRingConfig(out string _) && this.Ring == RingType.None; - } - + public bool TryGetNewerVersion( out Version newVersion, out string errorMessage) @@ -246,13 +241,8 @@ public bool TryCleanup(out string error) error = null; return true; } - - protected virtual bool TryDeleteDownloadedAsset(Asset asset, out Exception exception) - { - return this.fileSystem.TryDeleteFile(asset.LocalPath, out exception); - } - - protected virtual bool TryLoadRingConfig(out string error) + + public virtual bool TryLoadRingConfig(out string error) { string errorAdvisory = "Run `git config --global gvfs.upgradering [\"Fast\"|\"Slow\"|\"None\"]` and run `gvfs upgrade [--confirm]` again."; string gitPath = GVFSPlatform.Instance.GitInstallation.GetInstalledGitBinPath(); @@ -285,6 +275,11 @@ protected virtual bool TryLoadRingConfig(out string error) return false; } + protected virtual bool TryDeleteDownloadedAsset(Asset asset, out Exception exception) + { + return this.fileSystem.TryDeleteFile(asset.LocalPath, out exception); + } + protected virtual bool TryDownloadAsset(Asset asset, out string errorMessage) { errorMessage = null; diff --git a/GVFS/GVFS.FunctionalTests/Tests/EnlistmentPerFixture/GVFSUpgradeReminderTests.cs b/GVFS/GVFS.FunctionalTests/Tests/EnlistmentPerFixture/GVFSUpgradeReminderTests.cs index 837f21d27..376e35ea5 100644 --- a/GVFS/GVFS.FunctionalTests/Tests/EnlistmentPerFixture/GVFSUpgradeReminderTests.cs +++ b/GVFS/GVFS.FunctionalTests/Tests/EnlistmentPerFixture/GVFSUpgradeReminderTests.cs @@ -31,30 +31,41 @@ public UpgradeReminderTests() } [TestCase] - public void NoNagWhenUpgradeNotAvailable() + public void NoReminderWhenUpgradeNotAvailable() { this.EmptyDownloadDirectory(); - ProcessResult result = GitHelpers.InvokeGitAgainstGVFSRepo( + for (int count = 0; count < 50; count++) + { + ProcessResult result = GitHelpers.InvokeGitAgainstGVFSRepo( this.Enlistment.RepoRoot, "status"); - string.IsNullOrEmpty(result.Errors).ShouldBeTrue(); + string.IsNullOrEmpty(result.Errors).ShouldBeTrue(); + } } [TestCase] - public void NagWhenUpgradeAvailable() + public void RemindWhenUpgradeAvailable() { this.CreateUpgradeInstallers(); - ProcessResult result = GitHelpers.InvokeGitAgainstGVFSRepo( + string errors = string.Empty; + for (int count = 0; count < 50; count++) + { + ProcessResult result = GitHelpers.InvokeGitAgainstGVFSRepo( this.Enlistment.RepoRoot, "status"); - result.Errors.ShouldContain(new string[] + if (!string.IsNullOrEmpty(result.Errors)) + { + errors += result.Errors; + } + } + + errors.ShouldContain(new string[] { - "A newer version of GVFS is available.", - "Run `gvfs upgrade --confirm` from an elevated command prompt to install." + "A new version of GVFS is available." }); this.EmptyDownloadDirectory(); diff --git a/GVFS/GVFS.Hooks/Program.cs b/GVFS/GVFS.Hooks/Program.cs index e535b3452..94df57679 100644 --- a/GVFS/GVFS.Hooks/Program.cs +++ b/GVFS/GVFS.Hooks/Program.cs @@ -22,6 +22,7 @@ public class Program private static Dictionary specialArgValues = new Dictionary(); private static string enlistmentRoot; private static string enlistmentPipename; + private static Random random = new Random(); private delegate void LockRequestDelegate(bool unattended, string[] args, int pid, NamedPipeClient pipeClient); @@ -68,6 +69,7 @@ public static void Main(string[] args) RunLockRequest(args, unattended, ReleaseGVFSLock); } + RunPostCommands(args); break; default: @@ -83,12 +85,6 @@ public static void Main(string[] args) private static void RunPreCommands(string[] args) { - if (ProductUpgrader.IsLocalUpgradeAvailable()) - { - Console.WriteLine(GVFSConstants.UpgradeVerbMessages.UpgradeAvailable); - Console.WriteLine(GVFSConstants.UpgradeVerbMessages.UpgradeInstallAdvice); - } - string command = GetGitCommand(args); switch (command) { @@ -99,6 +95,26 @@ private static void RunPreCommands(string[] args) } } + private static void RunPostCommands(string[] args) + { + RemindUpgradeAvailable(); + } + + private static void RemindUpgradeAvailable() + { + // The idea is to generate a random number between 0 and 100. To make + // sure that the reminder is displayed only 10% of the times a git + // command is run, check that the random number is between 0 and 10, + // which will have a probability of 10/100 == 10%. + int reminderFrequency = 10; + int randomValue = random.Next(0, 100); + + if (randomValue <= reminderFrequency && ProductUpgrader.IsLocalUpgradeAvailable()) + { + Console.WriteLine(Environment.NewLine + GVFSConstants.UpgradeVerbMessages.ReminderNotification); + } + } + private static void ExitWithError(params string[] messages) { foreach (string message in messages) diff --git a/GVFS/GVFS.UnitTests.Windows/Windows/Mock/MockInstallerPreRunChecker.cs b/GVFS/GVFS.UnitTests.Windows/Windows/Mock/MockInstallerPreRunChecker.cs index 6b51f9a15..6dc432e4a 100644 --- a/GVFS/GVFS.UnitTests.Windows/Windows/Mock/MockInstallerPreRunChecker.cs +++ b/GVFS/GVFS.UnitTests.Windows/Windows/Mock/MockInstallerPreRunChecker.cs @@ -101,14 +101,14 @@ protected override bool TryRunGVFSWithArgs(string args, out string error) { this.GVFSArgs.Add(args); - if (string.CompareOrdinal(args, "service --unmount-all") == 0) + if (string.CompareOrdinal(args, "service --unmount-all --log-mount-failure-in-stderr") == 0) { bool result = this.FakedResultOfCheck(FailOnCheckType.UnMountRepos); error = result == false ? "Unmount of some of the repositories failed." : null; return result; } - if (string.CompareOrdinal(args, "service --mount-all") == 0) + if (string.CompareOrdinal(args, "service --mount-all --log-mount-failure-in-stderr") == 0) { bool result = this.FakedResultOfCheck(FailOnCheckType.RemountRepos); error = result == false ? "Auto remount failed." : null; diff --git a/GVFS/GVFS.UnitTests.Windows/Windows/Mock/MockProductUpgrader.cs b/GVFS/GVFS.UnitTests.Windows/Windows/Mock/MockProductUpgrader.cs index 229994393..3b8c30633 100644 --- a/GVFS/GVFS.UnitTests.Windows/Windows/Mock/MockProductUpgrader.cs +++ b/GVFS/GVFS.UnitTests.Windows/Windows/Mock/MockProductUpgrader.cs @@ -101,7 +101,7 @@ public override bool TrySetupToolsDirectory(out string upgraderToolPath, out str return true; } - protected override bool TryLoadRingConfig(out string error) + public override bool TryLoadRingConfig(out string error) { this.Ring = this.LocalRingConfig; diff --git a/GVFS/GVFS.UnitTests.Windows/Windows/Upgrader/UpgradeTests.cs b/GVFS/GVFS.UnitTests.Windows/Windows/Upgrader/UpgradeTests.cs index 9f9e1b3d3..4ac28563b 100644 --- a/GVFS/GVFS.UnitTests.Windows/Windows/Upgrader/UpgradeTests.cs +++ b/GVFS/GVFS.UnitTests.Windows/Windows/Upgrader/UpgradeTests.cs @@ -37,7 +37,7 @@ public virtual void Setup() [TestCase] public virtual void NoneLocalRing() { - string message = "Upgrade ring set to None. No upgrade check was performed."; + string message = "Upgrade ring set to \"None\". No upgrade check was performed."; this.ConfigureRunAndVerify( configure: () => { diff --git a/GVFS/GVFS.UnitTests.Windows/Windows/Upgrader/UpgradeVerbTests.cs b/GVFS/GVFS.UnitTests.Windows/Windows/Upgrader/UpgradeVerbTests.cs index bdade4207..d76ef303c 100644 --- a/GVFS/GVFS.UnitTests.Windows/Windows/Upgrader/UpgradeVerbTests.cs +++ b/GVFS/GVFS.UnitTests.Windows/Windows/Upgrader/UpgradeVerbTests.cs @@ -44,7 +44,7 @@ public void UpgradeAvailabilityReporting() expectedOutput: new List { "New GVFS version available: " + NewerThanLocalVersion, - "Run `gvfs upgrade --confirm` to install it" + "When you are ready, run `gvfs upgrade --confirm` from an elevated command prompt." }, expectedErrors: null); } diff --git a/GVFS/GVFS.Upgrader/UpgradeOrchestrator.cs b/GVFS/GVFS.Upgrader/UpgradeOrchestrator.cs index 18780cf3d..a5e51dae0 100644 --- a/GVFS/GVFS.Upgrader/UpgradeOrchestrator.cs +++ b/GVFS/GVFS.Upgrader/UpgradeOrchestrator.cs @@ -63,10 +63,19 @@ public void Execute() { string error = null; - if (this.upgrader.IsNoneRing()) + ProductUpgrader.RingType ring = ProductUpgrader.RingType.Invalid; + + if (!this.TryLoadUpgradeRing(out ring, out error) || ring == ProductUpgrader.RingType.None) { - this.output.WriteLine(GVFSConstants.UpgradeVerbMessages.NoneRingConsoleAlert); - this.output.WriteLine(GVFSConstants.UpgradeVerbMessages.SetUpgradeRingCommand); + string message = ring == ProductUpgrader.RingType.None ? + GVFSConstants.UpgradeVerbMessages.NoneRingConsoleAlert : + GVFSConstants.UpgradeVerbMessages.InvalidRingConsoleAlert; + this.output.WriteLine(message); + + if (ring == ProductUpgrader.RingType.None) + { + this.output.WriteLine(GVFSConstants.UpgradeVerbMessages.SetUpgradeRingCommand); + } } else { @@ -91,12 +100,16 @@ public void Execute() this.DeletedDownloadedAssets(); } } - + if (this.ExitCode == ReturnCode.GenericError) { error = Environment.NewLine + "ERROR: " + error; this.output.WriteLine(error); } + else + { + this.output.WriteLine(Environment.NewLine + "Upgrade completed successfully!"); + } if (this.input == Console.In) { @@ -120,6 +133,27 @@ private bool LaunchInsideSpinner(Func method, string message) null); } + private bool TryLoadUpgradeRing(out ProductUpgrader.RingType ring, out string consoleError) + { + bool loaded = false; + if (!this.upgrader.TryLoadRingConfig(out consoleError)) + { + EventMetadata metadata = new EventMetadata(); + metadata.Add("Upgrade Step", nameof(this.TryLoadUpgradeRing)); + metadata.Add("Load Error", consoleError); + this.tracer.RelatedError(metadata, $"{nameof(this.TryLoadUpgradeRing)} failed."); + this.ExitCode = ReturnCode.GenericError; + } + else + { + consoleError = null; + loaded = true; + } + + ring = this.upgrader.Ring; + return loaded; + } + private bool TryRunUpgradeInstall(out Version newVersion, out string consoleError) { newVersion = null; @@ -202,7 +236,7 @@ private bool TryRunUpgradeInstall(out Version newVersion, out string consoleErro return true; }, - $"Installing VFSForGit version: {newGVFSVersion}")) + $"Installing GVFS version: {newGVFSVersion}")) { consoleError = errorMessage; return false; @@ -260,17 +294,18 @@ private bool TryGetNewGitVersion(out GitVersion gitVersion, out string consoleEr { gitVersion = null; - this.tracer.RelatedInfo("Reading Git version from release info"); - - if (!this.upgrader.TryGetGitVersion(out gitVersion, out consoleError)) + using (ITracer activity = this.tracer.StartActivity(nameof(this.TryGetNewGitVersion), EventLevel.Informational)) { - EventMetadata metadata = new EventMetadata(); - metadata.Add("Upgrade Step", nameof(this.TryGetNewGitVersion)); - this.tracer.RelatedError(metadata, $"{nameof(this.upgrader.TryGetGitVersion)} failed. {consoleError}"); - return false; + if (!this.upgrader.TryGetGitVersion(out gitVersion, out consoleError)) + { + EventMetadata metadata = new EventMetadata(); + metadata.Add("Upgrade Step", nameof(this.TryGetNewGitVersion)); + this.tracer.RelatedError(metadata, $"{nameof(this.upgrader.TryGetGitVersion)} failed. {consoleError}"); + return false; + } + + activity.RelatedInfo("Successfully read Git version {0}", gitVersion); } - - this.tracer.RelatedInfo("Successfully read Git version {0}", gitVersion); return true; } @@ -279,79 +314,89 @@ private bool TryCheckIfUpgradeAvailable(out Version newestVersion, out string co { newestVersion = null; - this.tracer.RelatedInfo("Checking upgrade server for new releases"); - - if (!this.upgrader.TryGetNewerVersion(out newestVersion, out consoleError)) + using (ITracer activity = this.tracer.StartActivity(nameof(this.TryCheckIfUpgradeAvailable), EventLevel.Informational)) { - EventMetadata metadata = new EventMetadata(); - metadata.Add("Upgrade Step", nameof(this.TryCheckIfUpgradeAvailable)); - this.tracer.RelatedError(metadata, $"{nameof(this.upgrader.TryGetNewerVersion)} failed. {consoleError}"); - return false; - } + if (!this.upgrader.TryGetNewerVersion(out newestVersion, out consoleError)) + { + EventMetadata metadata = new EventMetadata(); + metadata.Add("Upgrade Step", nameof(this.TryCheckIfUpgradeAvailable)); + this.tracer.RelatedError(metadata, $"{nameof(this.upgrader.TryGetNewerVersion)} failed. {consoleError}"); + return false; + } - if (newestVersion == null) - { - consoleError = "No upgrades available in ring: " + this.upgrader.Ring; - this.tracer.RelatedInfo("No new upgrade releases available"); - return false; - } + if (newestVersion == null) + { + consoleError = "No upgrades available in ring: " + this.upgrader.Ring; + this.tracer.RelatedInfo("No new upgrade releases available"); + return false; + } - this.tracer.RelatedInfo("Successfully checked for new release. {0}", newestVersion); + activity.RelatedInfo("Successfully checked for new release. {0}", newestVersion); + } return true; } private bool TryDownloadUpgrade(Version version, out string consoleError) { - this.tracer.RelatedInfo("Downloading version: " + version.ToString()); - - if (!this.upgrader.TryDownloadNewestVersion(out consoleError)) + using (ITracer activity = this.tracer.StartActivity( + $"{nameof(this.TryDownloadUpgrade)}({version.ToString()})", + EventLevel.Informational)) { - EventMetadata metadata = new EventMetadata(); - metadata.Add("Upgrade Step", nameof(this.TryDownloadUpgrade)); - this.tracer.RelatedError(metadata, $"{nameof(this.upgrader.TryDownloadNewestVersion)} failed. {consoleError}"); - return false; - } + if (!this.upgrader.TryDownloadNewestVersion(out consoleError)) + { + EventMetadata metadata = new EventMetadata(); + metadata.Add("Upgrade Step", nameof(this.TryDownloadUpgrade)); + this.tracer.RelatedError(metadata, $"{nameof(this.upgrader.TryDownloadNewestVersion)} failed. {consoleError}"); + return false; + } - this.tracer.RelatedInfo("Successfully downloaded version: " + version.ToString()); + activity.RelatedInfo("Successfully downloaded version: " + version.ToString()); + } return true; } private bool TryInstallGitUpgrade(GitVersion version, out string consoleError) { - this.tracer.RelatedInfo("Installing Git version: " + version.ToString()); - bool installSuccess = false; - if (!this.upgrader.TryRunGitInstaller(out installSuccess, out consoleError) || - !installSuccess) - { - EventMetadata metadata = new EventMetadata(); - metadata.Add("Upgrade Step", nameof(this.TryInstallGitUpgrade)); - this.tracer.RelatedError(metadata, $"{nameof(this.upgrader.TryRunGitInstaller)} failed. {consoleError}"); - return false; - } + using (ITracer activity = this.tracer.StartActivity( + $"{nameof(this.TryInstallGitUpgrade)}({version.ToString()})", + EventLevel.Informational)) + { + if (!this.upgrader.TryRunGitInstaller(out installSuccess, out consoleError) || + !installSuccess) + { + EventMetadata metadata = new EventMetadata(); + metadata.Add("Upgrade Step", nameof(this.TryInstallGitUpgrade)); + this.tracer.RelatedError(metadata, $"{nameof(this.upgrader.TryRunGitInstaller)} failed. {consoleError}"); + return false; + } - this.tracer.RelatedInfo("Successfully installed Git version: " + version.ToString()); + activity.RelatedInfo("Successfully installed Git version: " + version.ToString()); + } return installSuccess; } private bool TryInstallGVFSUpgrade(Version version, out string consoleError) { - this.tracer.RelatedInfo("Installing GVFS version: " + version.ToString()); - bool installSuccess = false; - if (!this.upgrader.TryRunGVFSInstaller(out installSuccess, out consoleError) || - !installSuccess) + using (ITracer activity = this.tracer.StartActivity( + $"{nameof(this.TryInstallGVFSUpgrade)}({version.ToString()})", + EventLevel.Informational)) { - EventMetadata metadata = new EventMetadata(); - metadata.Add("Upgrade Step", nameof(this.TryInstallGVFSUpgrade)); - this.tracer.RelatedError(metadata, $"{nameof(this.upgrader.TryRunGVFSInstaller)} failed. {consoleError}"); - return false; - } + if (!this.upgrader.TryRunGVFSInstaller(out installSuccess, out consoleError) || + !installSuccess) + { + EventMetadata metadata = new EventMetadata(); + metadata.Add("Upgrade Step", nameof(this.TryInstallGVFSUpgrade)); + this.tracer.RelatedError(metadata, $"{nameof(this.upgrader.TryRunGVFSInstaller)} failed. {consoleError}"); + return false; + } - this.tracer.RelatedInfo("Successfully installed GVFS version: " + version.ToString()); + activity.RelatedInfo("Successfully installed GVFS version: " + version.ToString()); + } return installSuccess; } diff --git a/GVFS/GVFS/CommandLine/ServiceVerb.cs b/GVFS/GVFS/CommandLine/ServiceVerb.cs index 13b4a5689..e6ef09f66 100644 --- a/GVFS/GVFS/CommandLine/ServiceVerb.cs +++ b/GVFS/GVFS/CommandLine/ServiceVerb.cs @@ -35,6 +35,13 @@ public class ServiceVerb : GVFSVerb HelpText = "Prints a list of all mounted repos")] public bool List { get; set; } + [Option( + "log-mount-failure-in-stderr", + Default = false, + Required = false, + HelpText = "This parameter is reserved for internal use.")] + public bool RedirectMountFailuresToStderr { get; set; } + public override string EnlistmentRootPathParameter { get { throw new InvalidOperationException(); } @@ -103,7 +110,13 @@ public override void Execute() if (failedRepoRoots.Count() > 0) { - this.ReportErrorAndExit("\r\nThe following repos failed to mount:\r\n" + string.Join("\r\n", failedRepoRoots.ToArray())); + string errorString = $"The following repos failed to mount:{Environment.NewLine}{string.Join("\r\n", failedRepoRoots.ToArray())}"; + if (this.RedirectMountFailuresToStderr) + { + Console.Error.WriteLine(errorString); + } + + this.ReportErrorAndExit(Environment.NewLine + errorString); } } else if (this.UnmountAll) @@ -132,7 +145,13 @@ public override void Execute() if (failedRepoRoots.Count() > 0) { - this.ReportErrorAndExit("\r\nThe following repos failed to unmount:\r\n" + string.Join("\r\n", failedRepoRoots.ToArray())); + string errorString = $"The following repos failed to unmount:{Environment.NewLine}{string.Join(Environment.NewLine, failedRepoRoots.ToArray())}"; + if (this.RedirectMountFailuresToStderr) + { + Console.Error.WriteLine(errorString); + } + + this.ReportErrorAndExit(Environment.NewLine + errorString); } } } diff --git a/GVFS/GVFS/CommandLine/UpgradeVerb.cs b/GVFS/GVFS/CommandLine/UpgradeVerb.cs index e914842a8..289d0ac07 100644 --- a/GVFS/GVFS/CommandLine/UpgradeVerb.cs +++ b/GVFS/GVFS/CommandLine/UpgradeVerb.cs @@ -1,6 +1,5 @@ using CommandLine; using GVFS.Common; -using GVFS.Common.Git; using GVFS.Common.Tracing; using GVFS.Upgrader; using System; @@ -34,17 +33,8 @@ public UpgradeVerb( public UpgradeVerb() { - JsonTracer jsonTracer = new JsonTracer(GVFSConstants.GVFSEtwProviderName, "UpgradeVerb"); - string logFilePath = GVFSEnlistment.GetNewGVFSLogFileName( - ProductUpgrader.GetLogDirectoryPath(), - GVFSConstants.LogFileTypes.UpgradeVerb); - jsonTracer.AddLogFileEventListener(logFilePath, EventLevel.Informational, Keywords.Any); - - this.tracer = jsonTracer; - this.prerunChecker = new InstallerPreRunChecker(this.tracer); this.processWrapper = new ProcessLauncher(); this.Output = Console.Out; - this.upgrader = new ProductUpgrader(ProcessHelper.GetCurrentProcessVersion(), this.tracer); } [Option( @@ -64,29 +54,74 @@ protected override string VerbName public override void Execute() { ReturnCode exitCode = ReturnCode.Success; - if (!this.TryRunProductUpgrade()) + if (!this.TryInitializeUpgrader() || !this.TryRunProductUpgrade()) { exitCode = ReturnCode.GenericError; this.ReportErrorAndExit(this.tracer, exitCode, string.Empty); } } - + + private bool TryInitializeUpgrader() + { + OperatingSystem os_info = Environment.OSVersion; + + if (os_info.Platform == PlatformID.Win32NT) + { + if (this.upgrader == null) + { + JsonTracer jsonTracer = new JsonTracer(GVFSConstants.GVFSEtwProviderName, "UpgradeVerb"); + string logFilePath = GVFSEnlistment.GetNewGVFSLogFileName( + ProductUpgrader.GetLogDirectoryPath(), + GVFSConstants.LogFileTypes.UpgradeVerb); + jsonTracer.AddLogFileEventListener(logFilePath, EventLevel.Informational, Keywords.Any); + + this.tracer = jsonTracer; + this.prerunChecker = new InstallerPreRunChecker(this.tracer); + this.upgrader = new ProductUpgrader(ProcessHelper.GetCurrentProcessVersion(), this.tracer); + } + + return true; + } + else + { + this.ReportInfoToConsole($"ERROR: `gvfs upgrade` in only supported on Microsoft Windows Operating System."); + return false; + } + } + private bool TryRunProductUpgrade() { string errorOutputFormat = Environment.NewLine + "ERROR: {0}"; string error = null; Version newestVersion = null; - bool isInstallable = false; + ProductUpgrader.RingType ring = ProductUpgrader.RingType.Invalid; + + bool isInstallable = this.TryCheckUpgradeInstallable(out error); + if (this.Confirmed && !isInstallable) + { + this.ReportInfoToConsole($"Cannot install upgrade on this machine."); + this.Output.WriteLine(errorOutputFormat, error); + this.tracer.RelatedError($"{nameof(this.TryRunProductUpgrade)}: Upgrade is not installable. {error}"); + return false; + } - if (this.upgrader.IsNoneRing()) + if (!this.TryLoadUpgradeRing(out ring, out error)) + { + this.tracer.RelatedError($"{nameof(this.TryRunProductUpgrade)}: Could not load upgrade ring. {error}"); + this.ReportInfoToConsole(GVFSConstants.UpgradeVerbMessages.InvalidRingConsoleAlert); + this.Output.WriteLine(errorOutputFormat, error); + return false; + } + + if (ring == ProductUpgrader.RingType.None) { this.tracer.RelatedInfo($"{nameof(this.TryRunProductUpgrade)}: {GVFSConstants.UpgradeVerbMessages.NoneRingConsoleAlert}"); this.ReportInfoToConsole(GVFSConstants.UpgradeVerbMessages.NoneRingConsoleAlert); this.ReportInfoToConsole(GVFSConstants.UpgradeVerbMessages.SetUpgradeRingCommand); return true; } - - if (!this.TryRunUpgradeChecks(out newestVersion, out isInstallable, out error)) + + if (!this.TryRunUpgradeChecks(out newestVersion, out error)) { this.Output.WriteLine(errorOutputFormat, error); this.tracer.RelatedError($"{nameof(this.TryRunProductUpgrade)}: Upgrade checks failed. {error}"); @@ -98,39 +133,68 @@ private bool TryRunProductUpgrade() this.ReportInfoToConsole($"Great news, you're all caught up on upgrades in the {this.upgrader.Ring} ring!"); return true; } - - this.ReportInfoToConsole("New GVFS version available: {0}", newestVersion.ToString()); - - if (!this.Confirmed && isInstallable) + + string upgradeAvailableMessage = $"New GVFS version available: {newestVersion.ToString()}"; + if (this.Confirmed) { - this.ReportInfoToConsole("Run `gvfs upgrade --confirm` to install it"); - return true; - } + this.ReportInfoToConsole(upgradeAvailableMessage); - if (!isInstallable) - { - this.tracer.RelatedError($"{nameof(this.TryRunProductUpgrade)}: {error}"); - this.Output.WriteLine(errorOutputFormat, error); - return false; + if (!isInstallable) + { + this.tracer.RelatedError($"{nameof(this.TryRunProductUpgrade)}: {error}"); + this.Output.WriteLine(errorOutputFormat, error); + return false; + } + + if (!this.TryRunInstaller(out error)) + { + this.tracer.RelatedError($"{nameof(this.TryRunProductUpgrade)}: Could not launch upgrade tool. {error}"); + this.Output.WriteLine(errorOutputFormat, "Could not launch upgrade tool. " + error); + return false; + } } + else + { + if (isInstallable) + { + string message = string.Join( + Environment.NewLine + Environment.NewLine, + upgradeAvailableMessage, + GVFSConstants.UpgradeVerbMessages.UnmountRepoWarning, + GVFSConstants.UpgradeVerbMessages.UpgradeInstallAdvice); + this.ReportInfoToConsole(message); + } + else + { + this.ReportInfoToConsole($"{Environment.NewLine}{upgradeAvailableMessage}"); + } + } - if (!this.TryRunInstaller(out error)) + return true; + } + + private bool TryLoadUpgradeRing(out ProductUpgrader.RingType ring, out string consoleError) + { + bool loaded = false; + if (!this.upgrader.TryLoadRingConfig(out consoleError)) + { + this.tracer.RelatedError($"{nameof(this.TryLoadUpgradeRing)} failed. {consoleError}"); + } + else { - this.tracer.RelatedError($"{nameof(this.TryRunProductUpgrade)}: Could not launch upgrade tool. {error}"); - this.Output.WriteLine(errorOutputFormat, "Could not launch upgrade tool. " + error); - return false; + consoleError = null; + loaded = true; } - return true; + ring = this.upgrader.Ring; + return loaded; } private bool TryRunUpgradeChecks( out Version latestVersion, - out bool isUpgradeInstallable, out string consoleError) { bool upgradeCheckSuccess = false; - bool upgradeIsInstallable = false; string errorMessage = null; Version version = null; @@ -138,23 +202,12 @@ private bool TryRunUpgradeChecks( () => { upgradeCheckSuccess = this.TryCheckUpgradeAvailable(out version, out errorMessage); - if (upgradeCheckSuccess && version != null) - { - upgradeIsInstallable = true; - - if (!this.TryCheckUpgradeInstallable(out errorMessage)) - { - upgradeIsInstallable = false; - } - } - return upgradeCheckSuccess; }, "Checking for GVFS upgrades", suppressGvfsLogMessage: true); latestVersion = version; - isUpgradeInstallable = upgradeIsInstallable; consoleError = errorMessage; return upgradeCheckSuccess; @@ -185,6 +238,7 @@ private bool TryRunInstaller(out string consoleError) return false; } + this.ReportInfoToConsole($"{Environment.NewLine}Installer launched in a new window. Do not run any git or gvfs commands until the installer has completed."); consoleError = null; return true; } @@ -193,40 +247,42 @@ private bool TryCopyUpgradeTool(out string upgraderExePath, out string consoleEr { upgraderExePath = null; - this.tracer.RelatedInfo("Copying upgrade tool"); - - if (!this.upgrader.TrySetupToolsDirectory(out upgraderExePath, out consoleError)) + using (ITracer activity = this.tracer.StartActivity(nameof(this.TryCopyUpgradeTool), EventLevel.Informational)) { - return false; - } + if (!this.upgrader.TrySetupToolsDirectory(out upgraderExePath, out consoleError)) + { + return false; + } - this.tracer.RelatedInfo($"Successfully Copied upgrade tool to {upgraderExePath}"); + activity.RelatedInfo($"Successfully Copied upgrade tool to {upgraderExePath}"); + } return true; } private bool TryLaunchUpgradeTool(string path, out string consoleError) { - this.tracer.RelatedInfo("Launching upgrade tool"); - - Exception exception; - if (!this.processWrapper.TryStart(path, out exception)) + using (ITracer activity = this.tracer.StartActivity(nameof(this.TryLaunchUpgradeTool), EventLevel.Informational)) { - if (exception != null) + Exception exception; + if (!this.processWrapper.TryStart(path, out exception)) { - consoleError = exception.Message; - this.tracer.RelatedError($"Error launching upgrade tool. {exception.ToString()}"); - } - else - { - consoleError = $"Error launching upgrade tool"; + if (exception != null) + { + consoleError = exception.Message; + this.tracer.RelatedError($"Error launching upgrade tool. {exception.ToString()}"); + } + else + { + consoleError = "Error launching upgrade tool"; + } + + return false; } - - return false; - } - - this.tracer.RelatedInfo("Successfully launched upgrade tool."); + activity.RelatedInfo("Successfully launched upgrade tool."); + } + consoleError = null; return true; } @@ -238,20 +294,21 @@ private bool TryCheckUpgradeAvailable( latestVersion = null; consoleError = null; - this.tracer.RelatedInfo("Checking server for available upgrades."); - - bool checkSucceeded = false; - Version version = null; - - checkSucceeded = this.upgrader.TryGetNewerVersion(out version, out consoleError); - if (!checkSucceeded) + using (ITracer activity = this.tracer.StartActivity(nameof(this.TryCheckUpgradeAvailable), EventLevel.Informational)) { - return false; - } + bool checkSucceeded = false; + Version version = null; - latestVersion = version; + checkSucceeded = this.upgrader.TryGetNewerVersion(out version, out consoleError); + if (!checkSucceeded) + { + return false; + } + + latestVersion = version; - this.tracer.RelatedInfo("Successfully checked server for GVFS upgrades."); + activity.RelatedInfo("Successfully checked server for GVFS upgrades."); + } return true; } @@ -260,18 +317,19 @@ private bool TryCheckUpgradeInstallable(out string consoleError) { consoleError = null; - this.tracer.RelatedInfo("Checking if upgrade is installable on this machine."); - - this.prerunChecker.CommandToRerun = this.Confirmed ? "gvfs upgrade --confirm" : "gvfs upgrade"; - - if (!this.prerunChecker.TryRunPreUpgradeChecks( - out consoleError)) + using (ITracer activity = this.tracer.StartActivity(nameof(this.TryCheckUpgradeInstallable), EventLevel.Informational)) { - return false; - } + this.prerunChecker.CommandToRerun = this.Confirmed ? "gvfs upgrade --confirm" : "gvfs upgrade"; - this.tracer.RelatedInfo("Upgrade is installable."); + if (!this.prerunChecker.TryRunPreUpgradeChecks( + out consoleError)) + { + return false; + } + activity.RelatedInfo("Upgrade is installable."); + } + return true; } diff --git a/start b/start new file mode 100644 index 000000000..c342dc54f --- /dev/null +++ b/start @@ -0,0 +1 @@ +prjflt From 5681da5273567803a9dc191933d5513a7ef1c626 Mon Sep 17 00:00:00 2001 From: Al Ameen Shah Date: Fri, 21 Sep 2018 15:17:21 -0400 Subject: [PATCH 201/272] - renamed upgrade ring config to "gvfs.upgrade-ring" - Cleanup renamed UpgradeDiskLayout constant to MountUpgrade. - Cleanup - replaced usage of the string "gvfs.upgrade-ring" with const. - Added TODO to remove NotSupportedException from GitProcess::TryGitVersion. - Made InstallerTracer.CommandToRerun protected property. Updated UT. - Added issue reference url for InstallerPreRunChecker refactoring TODO. - Added new constants for "gvfs upgrade", "gvfs upgrade --confirm" strings. - Removed pre-check for development version. Upgrade can now be run on Developer machines. - Optimization - InstallerPreRunChecker::IsBlockingProcessRunning now returns HashSet. Removed unnecessary conversion to list. - Better error handling in InstallerPreRunChecker while launching GVFS CLI - Updated messaging in InstallerPreRunChecker. The text "elevated command prompt" was missing in some of the messaging. - Cleanup. Renamed processWrapper to ProcessLauncher. - Added URL info while logging network errors during fetch release. - Removed Randomization of upgrade timer in the service. - Removed GVFSArgs from MockInstallerPreRunChecker UT class. - Removed ShouldExitOnError property in UpgradOrchestrator class. Using Environment.ExitCode instead. - Cleanup. Replaced all occurances of "remount" with "mount". - removed log-mount-failure-in-stderr flag from Service verb. - renamed log files to productupgrade_and mount_repoupgrade --- GVFS/GVFS.Common/DiskLayoutUpgrade.cs | 2 +- GVFS/GVFS.Common/GVFSConstants.cs | 15 ++-- GVFS/GVFS.Common/Git/GitProcess.cs | 2 + GVFS/GVFS.Common/InstallerPreRunChecker.cs | 75 +++++++++---------- GVFS/GVFS.Common/ProductUpgrader.Shared.cs | 23 +----- GVFS/GVFS.Common/ProductUpgrader.cs | 36 +++++++-- GVFS/GVFS.Service/ProductUpgradeTimer.cs | 61 +++++++-------- .../Mock/MockInstallerPreRunChecker.cs | 37 ++++----- .../Windows/Mock/MockProcessLauncher.cs | 5 +- .../Upgrader/UpgradeOrchestratorTests.cs | 5 +- .../Windows/Upgrader/UpgradeVerbTests.cs | 30 ++------ GVFS/GVFS.Upgrader/UpgradeOrchestrator.cs | 50 ++++++------- GVFS/GVFS/CommandLine/ServiceVerb.cs | 19 +---- GVFS/GVFS/CommandLine/UpgradeVerb.cs | 16 ++-- 14 files changed, 161 insertions(+), 215 deletions(-) diff --git a/GVFS/GVFS.Common/DiskLayoutUpgrade.cs b/GVFS/GVFS.Common/DiskLayoutUpgrade.cs index 5ec4115fb..665590595 100644 --- a/GVFS/GVFS.Common/DiskLayoutUpgrade.cs +++ b/GVFS/GVFS.Common/DiskLayoutUpgrade.cs @@ -327,7 +327,7 @@ private static void StartLogFile(string enlistmentRoot, JsonTracer tracer) tracer.AddLogFileEventListener( GVFSEnlistment.GetNewGVFSLogFileName( Path.Combine(enlistmentRoot, GVFSConstants.DotGVFS.LogPath), - GVFSConstants.LogFileTypes.UpgradeDiskLayout), + GVFSConstants.LogFileTypes.MountUpgrade), EventLevel.Informational, Keywords.Any); diff --git a/GVFS/GVFS.Common/GVFSConstants.cs b/GVFS/GVFS.Common/GVFSConstants.cs index b9f6c56d2..39fc28930 100644 --- a/GVFS/GVFS.Common/GVFSConstants.cs +++ b/GVFS/GVFS.Common/GVFSConstants.cs @@ -33,7 +33,7 @@ public static class GitConfig public const string HooksPrefix = GitConfig.GVFSPrefix + "clone.default-"; public const string GVFSTelemetryId = GitConfig.GVFSPrefix + "telemetry-id"; public const string HooksExtension = ".hooks"; - public const string UpgradeRing = GVFSPrefix + "upgradering"; + public const string UpgradeRing = GVFSPrefix + "upgrade-ring"; } public static class GitStatusCache @@ -73,7 +73,7 @@ public static class SpecialGitFiles public static class LogFileTypes { public const string MountPrefix = "mount"; - public const string UpgradePrefix = "upgrade"; + public const string UpgradePrefix = "productupgrade"; public const string Clone = "clone"; public const string Dehydrate = "dehydrate"; @@ -84,7 +84,7 @@ public static class LogFileTypes public const string Service = "service"; public const string UpgradeVerb = UpgradePrefix + "_verb"; public const string UpgradeProcess = UpgradePrefix + "_process"; - public const string UpgradeDiskLayout = MountPrefix + "_upgrade"; + public const string MountUpgrade = MountPrefix + "_repoupgrade"; } public static class DotGVFS @@ -222,12 +222,15 @@ public static class Unmount public static class UpgradeVerbMessages { + public const string GVFSUpgrade = "`gvfs upgrade`"; + public const string GVFSUpgradeConfirm = "`gvfs upgrade --confirm`"; + public const string GVFSUpgradeOptionalConfirm = "`gvfs upgrade [--confirm]`"; public const string NoneRingConsoleAlert = "Upgrade ring set to \"None\". No upgrade check was performed."; public const string InvalidRingConsoleAlert = "Upgrade ring is not set. No upgrade check was performed."; - public const string SetUpgradeRingCommand = "To set or change upgrade ring, run `git config --global gvfs.upgradering [\"Fast\"|\"Slow\"|\"None\"] from a command prompt."; - public const string ReminderNotification = "A new version of GVFS is available. Run `gvfs upgrade` to start the upgrade."; + public const string SetUpgradeRingCommand = "To set or change upgrade ring, run `git config --global " + GitConfig.UpgradeRing + " [\"Fast\"|\"Slow\"|\"None\"] from a command prompt."; + public const string ReminderNotification = "A new version of GVFS is available. Run " + UpgradeVerbMessages.GVFSUpgrade + " to start the upgrade."; public const string UnmountRepoWarning = "The upgrade process will unmount all GVFS repositories for several minutes and remount them when it is complete. Ensure you are at a stopping point prior to upgrading."; - public const string UpgradeInstallAdvice = "When you are ready, run `gvfs upgrade --confirm` from an elevated command prompt."; + public const string UpgradeInstallAdvice = "When you are ready, run " + UpgradeVerbMessages.GVFSUpgradeConfirm + " from an elevated command prompt."; } } } diff --git a/GVFS/GVFS.Common/Git/GitProcess.cs b/GVFS/GVFS.Common/Git/GitProcess.cs index 1ce9236e1..2ed4e1bf4 100644 --- a/GVFS/GVFS.Common/Git/GitProcess.cs +++ b/GVFS/GVFS.Common/Git/GitProcess.cs @@ -94,6 +94,8 @@ public static Result GetFromSystemConfig(string gitBinPath, string settingName) public static bool TryGetVersion(out GitVersion gitVersion, out string error) { + // TODO Implement IGitInstallation in MockPlatform.cs & remove + // NotSupportedException handler. try { string gitPath = GVFSPlatform.Instance.GitInstallation.GetInstalledGitBinPath(); diff --git a/GVFS/GVFS.Common/InstallerPreRunChecker.cs b/GVFS/GVFS.Common/InstallerPreRunChecker.cs index aa5f919e5..ba46681a6 100644 --- a/GVFS/GVFS.Common/InstallerPreRunChecker.cs +++ b/GVFS/GVFS.Common/InstallerPreRunChecker.cs @@ -15,14 +15,14 @@ public class InstallerPreRunChecker private static readonly HashSet BlockingProcessSet = new HashSet { "GVFS", "GVFS.Mount", "git", "ssh-agent", "bash", "wish", "git-bash" }; private ITracer tracer; - - public InstallerPreRunChecker(ITracer tracer) + + public InstallerPreRunChecker(ITracer tracer, string commandToRerun) { this.tracer = tracer; - this.CommandToRerun = string.Empty; + this.CommandToRerun = commandToRerun; } - public string CommandToRerun { get; set; } + protected string CommandToRerun { private get; set; } public bool TryRunPreUpgradeChecks(out string consoleError) { @@ -30,35 +30,29 @@ public bool TryRunPreUpgradeChecks(out string consoleError) { if (this.IsUnattended()) { - consoleError = "`gvfs upgrade` is not supported in unattended mode"; - this.tracer.RelatedError($"{nameof(TryRunPreUpgradeChecks)}: {consoleError}"); - return false; - } - - if (this.IsDevelopmentVersion()) - { - consoleError = "Cannot run upgrade when development version of GVFS is installed."; + consoleError = $"{GVFSConstants.UpgradeVerbMessages.GVFSUpgrade} is not supported in unattended mode"; this.tracer.RelatedError($"{nameof(TryRunPreUpgradeChecks)}: {consoleError}"); return false; } - + if (!this.IsGVFSUpgradeAllowed(out consoleError)) { this.tracer.RelatedError($"{nameof(TryRunPreUpgradeChecks)}: {consoleError}"); return false; } - activity.RelatedInfo("Successfully finished pre upgrade checks. Okay to run GVFS upgrade."); + activity.RelatedInfo($"Successfully finished pre upgrade checks. Okay to run {GVFSConstants.UpgradeVerbMessages.GVFSUpgrade}."); } consoleError = null; return true; } - + // TODO: Move repo mount calls to GVFS.Upgrader project. + // https://github.com/Microsoft/VFSForGit/issues/293 public bool TryMountAllGVFSRepos(out string consoleError) { - return this.TryRunGVFSWithArgs("service --mount-all --log-mount-failure-in-stderr", out consoleError); + return this.TryRunGVFSWithArgs("service --mount-all", out consoleError); } public bool TryUnmountAllGVFSRepos(out string consoleError) @@ -69,7 +63,7 @@ public bool TryUnmountAllGVFSRepos(out string consoleError) using (ITracer activity = this.tracer.StartActivity(nameof(this.TryUnmountAllGVFSRepos), EventLevel.Informational)) { - if (!this.TryRunGVFSWithArgs("service --unmount-all --log-mount-failure-in-stderr", out consoleError)) + if (!this.TryRunGVFSWithArgs("service --unmount-all", out consoleError)) { this.tracer.RelatedError($"{nameof(TryUnmountAllGVFSRepos)}: {consoleError}"); return false; @@ -82,7 +76,7 @@ public bool TryUnmountAllGVFSRepos(out string consoleError) // actually quits. this.tracer.RelatedInfo("Checking if GVFS or dependent processes are running."); int retryCount = 10; - List processList = null; + HashSet processList = null; while (retryCount > 0) { if (!this.IsBlockingProcessRunning(out processList)) @@ -99,7 +93,7 @@ public bool TryUnmountAllGVFSRepos(out string consoleError) consoleError = string.Join( Environment.NewLine, "Blocking processes are running.", - $"Run `{this.CommandToRerun}` again after quitting these processes - " + string.Join(", ", processList.ToArray())); + $"Run {this.CommandToRerun} again after quitting these processes - " + string.Join(", ", processList.ToArray())); this.tracer.RelatedError($"{nameof(TryUnmountAllGVFSRepos)}: {consoleError}"); return false; } @@ -131,13 +125,8 @@ protected virtual bool IsUnattended() { return GVFSEnlistment.IsUnattended(this.tracer); } - - protected virtual bool IsDevelopmentVersion() - { - return ProcessHelper.IsDevelopmentVersion(); - } - - protected virtual bool IsBlockingProcessRunning(out List processes) + + protected virtual bool IsBlockingProcessRunning(out HashSet processes) { int currentProcessId = Process.GetCurrentProcess().Id; Process[] allProcesses = Process.GetProcesses(); @@ -153,26 +142,32 @@ protected virtual bool IsBlockingProcessRunning(out List processes) matchingNames.Add(process.ProcessName); } - processes = matchingNames.ToList(); + processes = matchingNames; return processes.Count > 0; } protected virtual bool TryRunGVFSWithArgs(string args, out string consoleError) { - consoleError = null; - - string gvfsPath = Path.Combine( - ProcessHelper.WhereDirectory(GVFSPlatform.Instance.Constants.GVFSExecutableName), - GVFSPlatform.Instance.Constants.GVFSExecutableName); - - ProcessResult processResult = ProcessHelper.Run(gvfsPath, args); - if (processResult.ExitCode == 0) + string gvfsDirectory = ProcessHelper.WhereDirectory(GVFSPlatform.Instance.Constants.GVFSExecutableName); + if (!string.IsNullOrEmpty(gvfsDirectory)) { - return true; + string gvfsPath = Path.Combine(gvfsDirectory, GVFSPlatform.Instance.Constants.GVFSExecutableName); + + ProcessResult processResult = ProcessHelper.Run(gvfsPath, args); + if (processResult.ExitCode == 0) + { + consoleError = null; + return true; + } + else + { + consoleError = string.IsNullOrEmpty(processResult.Errors) ? $"GVFS {args} failed." : processResult.Errors; + return false; + } } else { - consoleError = string.IsNullOrEmpty(processResult.Errors) ? $"GVFS {args} failed." : processResult.Errors; + consoleError = $"Could not locate {GVFSPlatform.Instance.Constants.GVFSExecutableName}"; return false; } } @@ -186,7 +181,7 @@ private bool IsGVFSUpgradeAllowed(out string consoleError) consoleError = string.Join( Environment.NewLine, "The installer needs to be run from an elevated command prompt.", - $"Run `{this.CommandToRerun}` again from an elevated command prompt."); + $"Run {this.CommandToRerun} again from an elevated command prompt."); return false; } @@ -194,7 +189,7 @@ private bool IsGVFSUpgradeAllowed(out string consoleError) { consoleError = string.Join( Environment.NewLine, - "ProjFS configuration does not support `gvfs upgrade`.", + $"ProjFS configuration does not support {GVFSConstants.UpgradeVerbMessages.GVFSUpgrade}.", "Check your team's documentation for how to upgrade."); return false; } @@ -204,7 +199,7 @@ private bool IsGVFSUpgradeAllowed(out string consoleError) consoleError = string.Join( Environment.NewLine, "GVFS Service is not running.", - $"Run `sc start GVFS.Service` and run `{this.CommandToRerun}` again."); + $"Run `sc start GVFS.Service` and run {this.CommandToRerun} again from an elevated command prompt."); return false; } diff --git a/GVFS/GVFS.Common/ProductUpgrader.Shared.cs b/GVFS/GVFS.Common/ProductUpgrader.Shared.cs index e8a995efc..30d8aed52 100644 --- a/GVFS/GVFS.Common/ProductUpgrader.Shared.cs +++ b/GVFS/GVFS.Common/ProductUpgrader.Shared.cs @@ -19,7 +19,7 @@ public static bool IsLocalUpgradeAvailable() if (Directory.Exists(downloadDirectory)) { string[] installers = Directory.GetFiles( - GetAssetDownloadsPath(), + downloadDirectory, $"{GVFSInstallerFileNamePrefix}*.*", SearchOption.TopDirectoryOnly); return installers.Length > 0; @@ -38,27 +38,6 @@ public static string GetLogDirectoryPath() return Path.Combine(Paths.GetServiceDataRoot(RootDirectory), LogDirectory); } - private static bool TryCreateDirectory(string path, out Exception exception) - { - try - { - Directory.CreateDirectory(path); - } - catch (IOException e) - { - exception = e; - return false; - } - catch (UnauthorizedAccessException e) - { - exception = e; - return false; - } - - exception = null; - return true; - } - private static string GetAssetDownloadsPath() { return Path.Combine( diff --git a/GVFS/GVFS.Common/ProductUpgrader.cs b/GVFS/GVFS.Common/ProductUpgrader.cs index b8b411a0c..af0a7e813 100644 --- a/GVFS/GVFS.Common/ProductUpgrader.cs +++ b/GVFS/GVFS.Common/ProductUpgrader.cs @@ -66,8 +66,8 @@ public enum RingType // upgrade logic. Invalid = 0, None = 10, - Slow = 20, - Fast = 30, + Slow = None + 1, + Fast = Slow + 1, } public RingType Ring { get; protected set; } @@ -193,7 +193,7 @@ public virtual bool TrySetupToolsDirectory(out string upgraderToolPath, out stri error = string.Join( Environment.NewLine, "File copy error - " + e.Message, - $"Make sure you have write permissions to directory {rootDirectoryPath} and run `gvfs upgrade --confirm` again."); + $"Make sure you have write permissions to directory {rootDirectoryPath} and run {GVFSConstants.UpgradeVerbMessages.GVFSUpgradeConfirm} again."); this.TraceException(e, nameof(this.TrySetupToolsDirectory), $"Error copying {toolPath} to {destinationPath}."); break; } @@ -244,7 +244,6 @@ public bool TryCleanup(out string error) public virtual bool TryLoadRingConfig(out string error) { - string errorAdvisory = "Run `git config --global gvfs.upgradering [\"Fast\"|\"Slow\"|\"None\"]` and run `gvfs upgrade [--confirm]` again."; string gitPath = GVFSPlatform.Instance.GitInstallation.GetInstalledGitBinPath(); GitProcess.Result result = GitProcess.GetFromGlobalConfig(gitPath, GVFSConstants.GitConfig.UpgradeRing); if (!result.HasErrors && !string.IsNullOrEmpty(result.Output)) @@ -270,7 +269,7 @@ public virtual bool TryLoadRingConfig(out string error) error = string.IsNullOrEmpty(result.Errors) ? "Unable to determine upgrade ring." : result.Errors; } - error += Environment.NewLine + errorAdvisory; + error += Environment.NewLine + GVFSConstants.UpgradeVerbMessages.SetUpgradeRingCommand; this.Ring = RingType.Invalid; return false; } @@ -332,12 +331,12 @@ protected virtual bool TryFetchReleases(out List releases, out string e } catch (HttpRequestException exception) { - errorMessage = string.Format("Network error: could not connect to GitHub. {0}", exception.Message); + errorMessage = string.Format("Network error: could not connect to GitHub({0}). {1}", GitHubReleaseURL, exception.Message); this.TraceException(exception, nameof(this.TryFetchReleases), $"Error fetching release info."); } catch (SerializationException exception) { - errorMessage = string.Format("Parse error: could not parse releases info from GitHub. {0}", exception.Message); + errorMessage = string.Format("Parse error: could not parse releases info from GitHub({0}). {1}", GitHubReleaseURL, exception.Message); this.TraceException(exception, nameof(this.TryFetchReleases), $"Error parsing release info."); } @@ -352,6 +351,27 @@ protected virtual void RunInstaller(string path, string args, out int exitCode, error = processResult.Errors; } + private static bool TryCreateDirectory(string path, out Exception exception) + { + try + { + Directory.CreateDirectory(path); + } + catch (IOException e) + { + exception = e; + return false; + } + catch (UnauthorizedAccessException e) + { + exception = e; + return false; + } + + exception = null; + return true; + } + private bool TryRunInstallerForAsset(string name, out int installerExitCode, out string error) { error = null; @@ -415,7 +435,7 @@ private bool TryGetLocalInstallerPath(string name, out string path, out string a args = null; return false; } - + [DataContract(Name = "asset")] protected class Asset { diff --git a/GVFS/GVFS.Service/ProductUpgradeTimer.cs b/GVFS/GVFS.Service/ProductUpgradeTimer.cs index 48c37d415..b276b28eb 100644 --- a/GVFS/GVFS.Service/ProductUpgradeTimer.cs +++ b/GVFS/GVFS.Service/ProductUpgradeTimer.cs @@ -21,14 +21,14 @@ public ProductUpgradeTimer(JsonTracer tracer) public void Start() { Random random = new Random(); - TimeSpan startTime = TimeSpan.FromMinutes(random.Next(0, 60)); + TimeSpan startTime = TimeSpan.Zero; - this.tracer.RelatedInfo($"Starting auto upgrade checks. Start time - {startTime.ToString()}"); + this.tracer.RelatedInfo("Starting auto upgrade checks."); this.timer = new Timer( this.TimerCallback, - null, - startTime, - TimeInterval); + state: null, + dueTime: startTime, + period: TimeInterval); } public void Stop() @@ -41,7 +41,7 @@ private void TimerCallback(object unusedState) { string errorMessage = null; - InstallerPreRunChecker prerunChecker = new InstallerPreRunChecker(this.tracer); + InstallerPreRunChecker prerunChecker = new InstallerPreRunChecker(this.tracer, string.Empty); if (prerunChecker.TryRunPreUpgradeChecks(out string _) && !this.TryDownloadUpgrade(out errorMessage)) { this.tracer.RelatedError(errorMessage); @@ -50,33 +50,34 @@ private void TimerCallback(object unusedState) private bool TryDownloadUpgrade(out string errorMessage) { - this.tracer.RelatedInfo("Checking for product upgrades."); - - ProductUpgrader productUpgrader = new ProductUpgrader(ProcessHelper.GetCurrentProcessVersion(), this.tracer); - Version newerVersion = null; - string detailedError = null; - if (!productUpgrader.TryGetNewerVersion(out newerVersion, out detailedError)) + using (ITracer activity = this.tracer.StartActivity("Checking for product upgrades.", EventLevel.Informational)) { - errorMessage = "Could not fetch new version info. " + detailedError; - return false; - } + ProductUpgrader productUpgrader = new ProductUpgrader(ProcessHelper.GetCurrentProcessVersion(), this.tracer); + Version newerVersion = null; + string detailedError = null; + if (!productUpgrader.TryGetNewerVersion(out newerVersion, out detailedError)) + { + errorMessage = "Could not fetch new version info. " + detailedError; + return false; + } - if (newerVersion == null) - { - // Already up-to-date - errorMessage = null; - return true; - } + if (newerVersion == null) + { + // Already up-to-date + errorMessage = null; + return true; + } - if (productUpgrader.TryDownloadNewestVersion(out detailedError)) - { - errorMessage = null; - return true; - } - else - { - errorMessage = "Could not download product upgrade. " + detailedError; - return false; + if (productUpgrader.TryDownloadNewestVersion(out detailedError)) + { + errorMessage = null; + return true; + } + else + { + errorMessage = "Could not download product upgrade. " + detailedError; + return false; + } } } } diff --git a/GVFS/GVFS.UnitTests.Windows/Windows/Mock/MockInstallerPreRunChecker.cs b/GVFS/GVFS.UnitTests.Windows/Windows/Mock/MockInstallerPreRunChecker.cs index 6dc432e4a..55255631e 100644 --- a/GVFS/GVFS.UnitTests.Windows/Windows/Mock/MockInstallerPreRunChecker.cs +++ b/GVFS/GVFS.UnitTests.Windows/Windows/Mock/MockInstallerPreRunChecker.cs @@ -15,7 +15,7 @@ public class MockInstallerPrerunChecker : InstallerPreRunChecker private FailOnCheckType failOnCheck; - public MockInstallerPrerunChecker(ITracer tracer) : base(tracer) + public MockInstallerPrerunChecker(ITracer tracer) : base(tracer, string.Empty) { } @@ -27,15 +27,11 @@ public enum FailOnCheckType IsElevated = 0x2, BlockingProcessesRunning = 0x4, UnattendedMode = 0x8, - IsDevelopmentVersion = 0x10, - IsGitUpgradeAllowed = 0x20, - UnMountRepos = 0x40, - RemountRepos = 0x80, - IsServiceInstalledAndNotRunning = 0x100 + UnMountRepos = 0x10, + RemountRepos = 0x20, + IsServiceInstalledAndNotRunning = 0x40, } - public List GVFSArgs { get; private set; } = new List(); - public void SetReturnFalseOnCheck(FailOnCheckType prerunCheck) { this.failOnCheck |= prerunCheck; @@ -50,12 +46,14 @@ public void Reset() { this.failOnCheck = FailOnCheckType.Invalid; - this.SetReturnFalseOnCheck(MockInstallerPrerunChecker.FailOnCheckType.IsDevelopmentVersion); this.SetReturnFalseOnCheck(MockInstallerPrerunChecker.FailOnCheckType.UnattendedMode); this.SetReturnFalseOnCheck(MockInstallerPrerunChecker.FailOnCheckType.BlockingProcessesRunning); this.SetReturnFalseOnCheck(MockInstallerPrerunChecker.FailOnCheckType.IsServiceInstalledAndNotRunning); + } - this.GVFSArgs.Clear(); + public void SetCommandToRerun(string command) + { + this.CommandToRerun = command; } protected override bool IsServiceInstalledAndNotRunning() @@ -77,15 +75,10 @@ protected override bool IsUnattended() { return this.FakedResultOfCheck(FailOnCheckType.UnattendedMode); } - - protected override bool IsDevelopmentVersion() - { - return this.FakedResultOfCheck(FailOnCheckType.IsDevelopmentVersion); - } - - protected override bool IsBlockingProcessRunning(out List processes) + + protected override bool IsBlockingProcessRunning(out HashSet processes) { - processes = new List(); + processes = new HashSet(); bool isRunning = this.FakedResultOfCheck(FailOnCheckType.BlockingProcessesRunning); if (isRunning) @@ -98,17 +91,15 @@ protected override bool IsBlockingProcessRunning(out List processes) } protected override bool TryRunGVFSWithArgs(string args, out string error) - { - this.GVFSArgs.Add(args); - - if (string.CompareOrdinal(args, "service --unmount-all --log-mount-failure-in-stderr") == 0) + { + if (string.CompareOrdinal(args, "service --unmount-all") == 0) { bool result = this.FakedResultOfCheck(FailOnCheckType.UnMountRepos); error = result == false ? "Unmount of some of the repositories failed." : null; return result; } - if (string.CompareOrdinal(args, "service --mount-all --log-mount-failure-in-stderr") == 0) + if (string.CompareOrdinal(args, "service --mount-all") == 0) { bool result = this.FakedResultOfCheck(FailOnCheckType.RemountRepos); error = result == false ? "Auto remount failed." : null; diff --git a/GVFS/GVFS.UnitTests.Windows/Windows/Mock/MockProcessLauncher.cs b/GVFS/GVFS.UnitTests.Windows/Windows/Mock/MockProcessLauncher.cs index dc3ddde55..3eb47e4bf 100644 --- a/GVFS/GVFS.UnitTests.Windows/Windows/Mock/MockProcessLauncher.cs +++ b/GVFS/GVFS.UnitTests.Windows/Windows/Mock/MockProcessLauncher.cs @@ -1,9 +1,8 @@ using System; -using static GVFS.CommandLine.UpgradeVerb; namespace GVFS.UnitTests.Windows.Upgrader { - public class MockProcessLauncher : ProcessLauncher + public class MockProcessLauncher : GVFS.CommandLine.UpgradeVerb.ProcessLauncher { private int exitCode; private bool hasExited; @@ -12,7 +11,7 @@ public class MockProcessLauncher : ProcessLauncher public MockProcessLauncher( int exitCode, bool hasExited, - bool startResult) : base() + bool startResult) { this.exitCode = exitCode; this.hasExited = hasExited; diff --git a/GVFS/GVFS.UnitTests.Windows/Windows/Upgrader/UpgradeOrchestratorTests.cs b/GVFS/GVFS.UnitTests.Windows/Windows/Upgrader/UpgradeOrchestratorTests.cs index abced85c4..356d80628 100644 --- a/GVFS/GVFS.UnitTests.Windows/Windows/Upgrader/UpgradeOrchestratorTests.cs +++ b/GVFS/GVFS.UnitTests.Windows/Windows/Upgrader/UpgradeOrchestratorTests.cs @@ -22,9 +22,8 @@ public override void Setup() this.Tracer, this.PrerunChecker, input: null, - output: this.Output, - shouldExitOnError: false); - this.PrerunChecker.CommandToRerun = "gvfs upgrade --confirm"; + output: this.Output); + this.PrerunChecker.SetCommandToRerun("`gvfs upgrade --confirm`"); } [TestCase] diff --git a/GVFS/GVFS.UnitTests.Windows/Windows/Upgrader/UpgradeVerbTests.cs b/GVFS/GVFS.UnitTests.Windows/Windows/Upgrader/UpgradeVerbTests.cs index d76ef303c..cc4b43065 100644 --- a/GVFS/GVFS.UnitTests.Windows/Windows/Upgrader/UpgradeVerbTests.cs +++ b/GVFS/GVFS.UnitTests.Windows/Windows/Upgrader/UpgradeVerbTests.cs @@ -27,7 +27,7 @@ public override void Setup() this.ProcessWrapper, this.Output); this.UpgradeVerb.Confirmed = false; - this.PrerunChecker.CommandToRerun = "gvfs upgrade"; + this.PrerunChecker.SetCommandToRerun("`gvfs upgrade`"); } [TestCase] @@ -75,7 +75,7 @@ public void LaunchInstaller() configure: () => { this.UpgradeVerb.Confirmed = true; - this.PrerunChecker.CommandToRerun = "gvfs upgrade --confirm"; + this.PrerunChecker.SetCommandToRerun("`gvfs upgrade --confirm`"); }, expectedReturn: ReturnCode.Success, expectedOutput: new List @@ -111,7 +111,7 @@ public void CopyTools() { this.Upgrader.SetFailOnAction(MockProductUpgrader.ActionType.CopyTools); this.UpgradeVerb.Confirmed = true; - this.PrerunChecker.CommandToRerun = "gvfs upgrade --confirm"; + this.PrerunChecker.SetCommandToRerun("`gvfs upgrade --confirm`"); }, expectedReturn: ReturnCode.GenericError, expectedOutput: new List @@ -148,6 +148,7 @@ public void ProjFSPreCheck() [TestCase] public void IsGVFSServiceRunningPreCheck() { + this.PrerunChecker.SetCommandToRerun("`gvfs upgrade --confirm`"); this.ConfigureRunAndVerify( configure: () => { @@ -158,7 +159,7 @@ public void IsGVFSServiceRunningPreCheck() expectedOutput: new List { "GVFS Service is not running.", - "Run `sc start GVFS.Service` and run `gvfs upgrade --confirm` again." + "Run `sc start GVFS.Service` and run `gvfs upgrade --confirm` again from an elevated command prompt." }, expectedErrors: new List { @@ -169,6 +170,7 @@ public void IsGVFSServiceRunningPreCheck() [TestCase] public void ElevatedRunPreCheck() { + this.PrerunChecker.SetCommandToRerun("`gvfs upgrade --confirm`"); this.ConfigureRunAndVerify( configure: () => { @@ -207,26 +209,6 @@ public void UnAttendedModePreCheck() }); } - [TestCase] - public void DeveloperMachinePreCheck() - { - this.ConfigureRunAndVerify( - configure: () => - { - this.UpgradeVerb.Confirmed = true; - this.PrerunChecker.SetReturnTrueOnCheck(MockInstallerPrerunChecker.FailOnCheckType.IsDevelopmentVersion); - }, - expectedReturn: ReturnCode.GenericError, - expectedOutput: new List - { - "Cannot run upgrade when development version of GVFS is installed." - }, - expectedErrors: new List - { - "Cannot run upgrade when development version of GVFS is installed." - }); - } - protected override void RunUpgrade() { try diff --git a/GVFS/GVFS.Upgrader/UpgradeOrchestrator.cs b/GVFS/GVFS.Upgrader/UpgradeOrchestrator.cs index a5e51dae0..155adf9f3 100644 --- a/GVFS/GVFS.Upgrader/UpgradeOrchestrator.cs +++ b/GVFS/GVFS.Upgrader/UpgradeOrchestrator.cs @@ -15,24 +15,21 @@ public class UpgradeOrchestrator private InstallerPreRunChecker preRunChecker; private TextWriter output; private TextReader input; - private bool remount; - private bool shouldExitOnError; + private bool mount; public UpgradeOrchestrator( ProductUpgrader upgrader, ITracer tracer, InstallerPreRunChecker preRunChecker, TextReader input, - TextWriter output, - bool shouldExitOnError) + TextWriter output) { this.upgrader = upgrader; this.tracer = tracer; this.preRunChecker = preRunChecker; this.output = output; this.input = input; - this.remount = false; - this.shouldExitOnError = shouldExitOnError; + this.mount = false; this.ExitCode = ReturnCode.Success; } @@ -48,12 +45,11 @@ public UpgradeOrchestrator() Keywords.Any); this.tracer = jsonTracer; - this.preRunChecker = new InstallerPreRunChecker(this.tracer); + this.preRunChecker = new InstallerPreRunChecker(this.tracer, GVFSConstants.UpgradeVerbMessages.GVFSUpgradeConfirm); this.upgrader = new ProductUpgrader(ProcessHelper.GetCurrentProcessVersion(), this.tracer); this.output = Console.Out; this.input = Console.In; - this.remount = false; - this.shouldExitOnError = false; + this.mount = false; this.ExitCode = ReturnCode.Success; } @@ -82,18 +78,18 @@ public void Execute() try { Version newVersion = null; - if (!this.TryRunUpgradeInstall(out newVersion, out error)) + if (!this.TryRunUpgrade(out newVersion, out error)) { this.ExitCode = ReturnCode.GenericError; } } finally { - string remountError = null; - if (!this.TryRemountRepositories(out remountError)) + string mountError = null; + if (!this.TryMountRepositories(out mountError)) { - remountError = Environment.NewLine + "WARNING: " + remountError; - this.output.WriteLine(remountError); + mountError = Environment.NewLine + "WARNING: " + mountError; + this.output.WriteLine(mountError); this.ExitCode = ReturnCode.Success; } @@ -116,11 +112,8 @@ public void Execute() this.output.WriteLine("Press Enter to exit."); this.input.ReadLine(); } - - if (this.shouldExitOnError) - { - Environment.Exit((int)this.ExitCode); - } + + Environment.ExitCode = (int)this.ExitCode; } private bool LaunchInsideSpinner(Func method, string message) @@ -154,7 +147,7 @@ private bool TryLoadUpgradeRing(out ProductUpgrader.RingType ring, out string co return loaded; } - private bool TryRunUpgradeInstall(out Version newVersion, out string consoleError) + private bool TryRunUpgrade(out Version newVersion, out string consoleError) { newVersion = null; @@ -173,7 +166,6 @@ private bool TryRunUpgradeInstall(out Version newVersion, out string consoleErro this.LogInstalledVersionInfo(); this.LogVersionInfo(newGVFSVersion, newGitVersion, "Available Version"); - this.preRunChecker.CommandToRerun = "gvfs upgrade --confirm"; if (!this.preRunChecker.TryRunPreUpgradeChecks(out errorMessage)) { return false; @@ -200,7 +192,7 @@ private bool TryRunUpgradeInstall(out Version newVersion, out string consoleErro return false; } - this.remount = true; + this.mount = true; return true; }, @@ -249,20 +241,20 @@ private bool TryRunUpgradeInstall(out Version newVersion, out string consoleErro return true; } - private bool TryRemountRepositories(out string consoleError) + private bool TryMountRepositories(out string consoleError) { string errorMessage = string.Empty; - if (this.remount && !this.LaunchInsideSpinner( + if (this.mount && !this.LaunchInsideSpinner( () => { - string remountError; - if (!this.preRunChecker.TryMountAllGVFSRepos(out remountError)) + string mountError; + if (!this.preRunChecker.TryMountAllGVFSRepos(out mountError)) { EventMetadata metadata = new EventMetadata(); - metadata.Add("Upgrade Step", nameof(this.TryRemountRepositories)); - metadata.Add("Remount Error", remountError); + metadata.Add("Upgrade Step", nameof(this.TryMountRepositories)); + metadata.Add("Mount Error", mountError); this.tracer.RelatedError(metadata, $"{nameof(this.preRunChecker.TryMountAllGVFSRepos)} failed."); - errorMessage += remountError; + errorMessage += mountError; return false; } diff --git a/GVFS/GVFS/CommandLine/ServiceVerb.cs b/GVFS/GVFS/CommandLine/ServiceVerb.cs index e6ef09f66..640c99a08 100644 --- a/GVFS/GVFS/CommandLine/ServiceVerb.cs +++ b/GVFS/GVFS/CommandLine/ServiceVerb.cs @@ -35,13 +35,6 @@ public class ServiceVerb : GVFSVerb HelpText = "Prints a list of all mounted repos")] public bool List { get; set; } - [Option( - "log-mount-failure-in-stderr", - Default = false, - Required = false, - HelpText = "This parameter is reserved for internal use.")] - public bool RedirectMountFailuresToStderr { get; set; } - public override string EnlistmentRootPathParameter { get { throw new InvalidOperationException(); } @@ -111,11 +104,7 @@ public override void Execute() if (failedRepoRoots.Count() > 0) { string errorString = $"The following repos failed to mount:{Environment.NewLine}{string.Join("\r\n", failedRepoRoots.ToArray())}"; - if (this.RedirectMountFailuresToStderr) - { - Console.Error.WriteLine(errorString); - } - + Console.Error.WriteLine(errorString); this.ReportErrorAndExit(Environment.NewLine + errorString); } } @@ -146,11 +135,7 @@ public override void Execute() if (failedRepoRoots.Count() > 0) { string errorString = $"The following repos failed to unmount:{Environment.NewLine}{string.Join(Environment.NewLine, failedRepoRoots.ToArray())}"; - if (this.RedirectMountFailuresToStderr) - { - Console.Error.WriteLine(errorString); - } - + Console.Error.WriteLine(errorString); this.ReportErrorAndExit(Environment.NewLine + errorString); } } diff --git a/GVFS/GVFS/CommandLine/UpgradeVerb.cs b/GVFS/GVFS/CommandLine/UpgradeVerb.cs index 289d0ac07..020a16fdc 100644 --- a/GVFS/GVFS/CommandLine/UpgradeVerb.cs +++ b/GVFS/GVFS/CommandLine/UpgradeVerb.cs @@ -8,14 +8,14 @@ namespace GVFS.CommandLine { - [Verb(UpgradeVerbName, HelpText = "Checks if a new GVFS release is available.")] + [Verb(UpgradeVerbName, HelpText = "Checks for new GVFS release, downloads and installs it when available.")] public class UpgradeVerb : GVFSVerb { private const string UpgradeVerbName = "upgrade"; private ITracer tracer; private ProductUpgrader upgrader; private InstallerPreRunChecker prerunChecker; - private ProcessLauncher processWrapper; + private ProcessLauncher processLauncher; public UpgradeVerb( ProductUpgrader upgrader, @@ -27,13 +27,13 @@ public UpgradeVerb( this.upgrader = upgrader; this.tracer = tracer; this.prerunChecker = prerunChecker; - this.processWrapper = processWrapper; + this.processLauncher = processWrapper; this.Output = output; } public UpgradeVerb() { - this.processWrapper = new ProcessLauncher(); + this.processLauncher = new ProcessLauncher(); this.Output = Console.Out; } @@ -76,7 +76,7 @@ private bool TryInitializeUpgrader() jsonTracer.AddLogFileEventListener(logFilePath, EventLevel.Informational, Keywords.Any); this.tracer = jsonTracer; - this.prerunChecker = new InstallerPreRunChecker(this.tracer); + this.prerunChecker = new InstallerPreRunChecker(this.tracer, this.Confirmed ? GVFSConstants.UpgradeVerbMessages.GVFSUpgradeConfirm : GVFSConstants.UpgradeVerbMessages.GVFSUpgrade); this.upgrader = new ProductUpgrader(ProcessHelper.GetCurrentProcessVersion(), this.tracer); } @@ -84,7 +84,7 @@ private bool TryInitializeUpgrader() } else { - this.ReportInfoToConsole($"ERROR: `gvfs upgrade` in only supported on Microsoft Windows Operating System."); + this.ReportInfoToConsole($"ERROR: {GVFSConstants.UpgradeVerbMessages.GVFSUpgrade} in only supported on Microsoft Windows Operating System."); return false; } } @@ -265,7 +265,7 @@ private bool TryLaunchUpgradeTool(string path, out string consoleError) using (ITracer activity = this.tracer.StartActivity(nameof(this.TryLaunchUpgradeTool), EventLevel.Informational)) { Exception exception; - if (!this.processWrapper.TryStart(path, out exception)) + if (!this.processLauncher.TryStart(path, out exception)) { if (exception != null) { @@ -319,8 +319,6 @@ private bool TryCheckUpgradeInstallable(out string consoleError) using (ITracer activity = this.tracer.StartActivity(nameof(this.TryCheckUpgradeInstallable), EventLevel.Informational)) { - this.prerunChecker.CommandToRerun = this.Confirmed ? "gvfs upgrade --confirm" : "gvfs upgrade"; - if (!this.prerunChecker.TryRunPreUpgradeChecks( out consoleError)) { From 4206f2670ce27134696b82dda053100805f90176 Mon Sep 17 00:00:00 2001 From: Al Ameen Shah Date: Mon, 24 Sep 2018 17:15:12 -0400 Subject: [PATCH 202/272] - resolving merge issues --- GVFS/GVFS.Platform.Windows/ProjFSFilter.cs | 5 ----- 1 file changed, 5 deletions(-) diff --git a/GVFS/GVFS.Platform.Windows/ProjFSFilter.cs b/GVFS/GVFS.Platform.Windows/ProjFSFilter.cs index a5a0c3208..d82693c42 100644 --- a/GVFS/GVFS.Platform.Windows/ProjFSFilter.cs +++ b/GVFS/GVFS.Platform.Windows/ProjFSFilter.cs @@ -584,11 +584,6 @@ private static ProcessResult GetProjFSOptionalFeatureStatus() (int)ProjFSInboxStatus.NotInbox + "}else{if($var.State -eq 'Enabled'){exit " + (int)ProjFSInboxStatus.Enabled + "}else{exit " + (int)ProjFSInboxStatus.Disabled + "}}"); } - private static ProcessResult CallPowershellCommand(string command) - { - return ProcessHelper.Run("powershell.exe", "-NonInteractive -NoProfile -Command \"& { " + command + " }\""); - } - private static EventMetadata CreateEventMetadata(Exception e = null) { EventMetadata metadata = new EventMetadata(); From 9e225a8c6ae01b91988fcda42f4167a54514c278 Mon Sep 17 00:00:00 2001 From: Al Ameen Shah Date: Thu, 27 Sep 2018 11:30:55 -0400 Subject: [PATCH 203/272] - Removed GitProcess.Version() - Using GitProcess.TryGetVersion() in GVFSVerb and Diagnose Verb. - Save directory listing of GVFS.Upgrade\Downloads directory during diagnose. - Created MockGitInstallation.cs - Removed NotSupportedException handler from GitProcess.cs --- GVFS/GVFS.Common/Git/GitProcess.cs | 15 ++++--------- GVFS/GVFS.Common/ProductUpgrader.Shared.cs | 2 +- .../Mock/Common/MockPlatform.cs | 3 ++- .../Mock/Git/MockGitInstallation.cs | 22 +++++++++++++++++++ GVFS/GVFS/CommandLine/DiagnoseVerb.cs | 18 ++++++++++++++- GVFS/GVFS/CommandLine/GVFSVerb.cs | 16 +++----------- 6 files changed, 49 insertions(+), 27 deletions(-) create mode 100644 GVFS/GVFS.UnitTests/Mock/Git/MockGitInstallation.cs diff --git a/GVFS/GVFS.Common/Git/GitProcess.cs b/GVFS/GVFS.Common/Git/GitProcess.cs index 2ed4e1bf4..2d3a2206e 100644 --- a/GVFS/GVFS.Common/Git/GitProcess.cs +++ b/GVFS/GVFS.Common/Git/GitProcess.cs @@ -77,11 +77,6 @@ public static Result Init(Enlistment enlistment) return new GitProcess(enlistment).InvokeGitOutsideEnlistment("init \"" + enlistment.WorkingDirectoryRoot + "\""); } - public static Result Version(Enlistment enlistment) - { - return new GitProcess(enlistment).InvokeGitOutsideEnlistment("--version"); - } - public static Result GetFromGlobalConfig(string gitBinPath, string settingName) { return new GitProcess(gitBinPath, workingDirectoryRoot: null, gvfsHooksRoot: null).InvokeGitOutsideEnlistment("config --global " + settingName); @@ -94,11 +89,9 @@ public static Result GetFromSystemConfig(string gitBinPath, string settingName) public static bool TryGetVersion(out GitVersion gitVersion, out string error) { - // TODO Implement IGitInstallation in MockPlatform.cs & remove - // NotSupportedException handler. - try + string gitPath = GVFSPlatform.Instance.GitInstallation.GetInstalledGitBinPath(); + if (gitPath != null) { - string gitPath = GVFSPlatform.Instance.GitInstallation.GetInstalledGitBinPath(); GitProcess gitProcess = new GitProcess(gitPath, null, null); Result result = gitProcess.InvokeGitOutsideEnlistment("--version"); string version = result.Output; @@ -112,10 +105,10 @@ public static bool TryGetVersion(out GitVersion gitVersion, out string error) error = null; return true; } - catch (NotSupportedException exception) + else { - error = exception.ToString(); gitVersion = null; + error = "Unable to determine installed git path."; return false; } } diff --git a/GVFS/GVFS.Common/ProductUpgrader.Shared.cs b/GVFS/GVFS.Common/ProductUpgrader.Shared.cs index 30d8aed52..87d5438b1 100644 --- a/GVFS/GVFS.Common/ProductUpgrader.Shared.cs +++ b/GVFS/GVFS.Common/ProductUpgrader.Shared.cs @@ -8,9 +8,9 @@ public partial class ProductUpgrader { public const string UpgradeDirectoryName = "GVFS.Upgrade"; public const string LogDirectory = "Logs"; + public const string DownloadDirectory = "Downloads"; private const string RootDirectory = UpgradeDirectoryName; - private const string DownloadDirectory = "Downloads"; private const string GVFSInstallerFileNamePrefix = "SetupGVFS"; public static bool IsLocalUpgradeAvailable() diff --git a/GVFS/GVFS.UnitTests/Mock/Common/MockPlatform.cs b/GVFS/GVFS.UnitTests/Mock/Common/MockPlatform.cs index 4fb290f50..f4ba15d35 100644 --- a/GVFS/GVFS.UnitTests/Mock/Common/MockPlatform.cs +++ b/GVFS/GVFS.UnitTests/Mock/Common/MockPlatform.cs @@ -4,6 +4,7 @@ using GVFS.Common.Tracing; using GVFS.UnitTests.Mock.Common.Tracing; using GVFS.UnitTests.Mock.FileSystem; +using GVFS.UnitTests.Mock.Git; using System; using System.Collections.Generic; using System.IO.Pipes; @@ -19,7 +20,7 @@ public MockPlatform() public override IKernelDriver KernelDriver => throw new NotSupportedException(); - public override IGitInstallation GitInstallation => throw new NotSupportedException(); + public override IGitInstallation GitInstallation { get; } = new MockGitInstallation(); public override IDiskLayoutUpgradeData DiskLayoutUpgrade => throw new NotSupportedException(); diff --git a/GVFS/GVFS.UnitTests/Mock/Git/MockGitInstallation.cs b/GVFS/GVFS.UnitTests/Mock/Git/MockGitInstallation.cs new file mode 100644 index 000000000..792b0291f --- /dev/null +++ b/GVFS/GVFS.UnitTests/Mock/Git/MockGitInstallation.cs @@ -0,0 +1,22 @@ +using GVFS.Common.Git; +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.Threading.Tasks; + +namespace GVFS.UnitTests.Mock.Git +{ + public class MockGitInstallation : IGitInstallation + { + public bool GitExists(string gitBinPath) + { + return false; + } + + public string GetInstalledGitBinPath() + { + return null; + } + } +} diff --git a/GVFS/GVFS/CommandLine/DiagnoseVerb.cs b/GVFS/GVFS/CommandLine/DiagnoseVerb.cs index b2e932f96..6f59d5901 100644 --- a/GVFS/GVFS/CommandLine/DiagnoseVerb.cs +++ b/GVFS/GVFS/CommandLine/DiagnoseVerb.cs @@ -49,7 +49,18 @@ protected override void Execute(GVFSEnlistment enlistment) this.WriteMessage(string.Empty); this.WriteMessage("gvfs version " + ProcessHelper.GetCurrentProcessVersion()); - this.WriteMessage(GitProcess.Version(enlistment).Output); + + GitVersion gitVersion; + string error; + if (GitProcess.TryGetVersion(out gitVersion, out error)) + { + this.WriteMessage("git version " + gitVersion.ToString()); + } + else + { + this.WriteMessage("Could not determine git version. " + error); + } + this.WriteMessage(GVFSPlatform.Instance.GitInstallation.GetInstalledGitBinPath()); this.WriteMessage(string.Empty); this.WriteMessage("Enlistment root: " + enlistment.EnlistmentRoot); @@ -120,6 +131,11 @@ protected override void Execute(GVFSEnlistment enlistment) ProductUpgrader.LogDirectory, copySubFolders: true, targetFolderName: ProductUpgrader.UpgradeDirectoryName); + this.LogDirectoryEnumeration( + ProductUpgrader.GetUpgradesDirectoryPath(), + Path.Combine(archiveFolderPath, ProductUpgrader.UpgradeDirectoryName), + ProductUpgrader.DownloadDirectory, + "downloaded-assets.txt"); return true; }, diff --git a/GVFS/GVFS/CommandLine/GVFSVerb.cs b/GVFS/GVFS/CommandLine/GVFSVerb.cs index eb4370621..8b9f1a8b6 100644 --- a/GVFS/GVFS/CommandLine/GVFSVerb.cs +++ b/GVFS/GVFS/CommandLine/GVFSVerb.cs @@ -619,23 +619,13 @@ private void CheckFileSystemSupportsRequiredFeatures(ITracer tracer, Enlistment private void CheckGitVersion(ITracer tracer, GVFSEnlistment enlistment, out string version) { - GitProcess.Result versionResult = GitProcess.Version(enlistment); - if (versionResult.HasErrors) - { - this.ReportErrorAndExit(tracer, "Error: Unable to retrieve the git version"); - } - GitVersion gitVersion; - version = versionResult.Output; - if (version.StartsWith("git version ")) + if (!GitProcess.TryGetVersion(out gitVersion, out string _)) { - version = version.Substring(12); + this.ReportErrorAndExit(tracer, "Error: Unable to retrieve the git version"); } - if (!GitVersion.TryParseVersion(version, out gitVersion)) - { - this.ReportErrorAndExit(tracer, "Error: Unable to parse the git version. {0}", version); - } + version = gitVersion.ToString(); if (gitVersion.Platform != GVFSConstants.SupportedGitVersion.Platform) { From f4536802cc224216eb327cfaa1ba124122f611cf Mon Sep 17 00:00:00 2001 From: Al Ameen Shah Date: Wed, 26 Sep 2018 10:40:23 -0400 Subject: [PATCH 204/272] - Renamed GVFSConfig.cs to ServerGVFSConfig.cs. - Renamed all variables with name gvfsConfig to serverGVFSConfig --- GVFS/GVFS.Common/Http/CacheServerResolver.cs | 10 +- GVFS/GVFS.Common/Http/ConfigHttpRequestor.cs | 20 ++-- GVFS/GVFS.Common/LocalCacheResolver.cs | 14 +-- GVFS/GVFS.Common/LocalGVFSConfig.cs | 99 +++++++++++++++++++ .../{GVFSConfig.cs => ServerGVFSConfig.cs} | 4 +- .../Common/CacheServerResolverTests.cs | 8 +- GVFS/GVFS/CommandLine/CacheServerVerb.cs | 8 +- GVFS/GVFS/CommandLine/CloneVerb.cs | 22 ++--- GVFS/GVFS/CommandLine/DehydrateVerb.cs | 2 +- GVFS/GVFS/CommandLine/GVFSVerb.cs | 38 +++---- GVFS/GVFS/CommandLine/MountVerb.cs | 14 +-- GVFS/GVFS/CommandLine/PrefetchVerb.cs | 14 +-- 12 files changed, 176 insertions(+), 77 deletions(-) create mode 100644 GVFS/GVFS.Common/LocalGVFSConfig.cs rename GVFS/GVFS.Common/{GVFSConfig.cs => ServerGVFSConfig.cs} (89%) diff --git a/GVFS/GVFS.Common/Http/CacheServerResolver.cs b/GVFS/GVFS.Common/Http/CacheServerResolver.cs index ea39119d0..86a693ea6 100644 --- a/GVFS/GVFS.Common/Http/CacheServerResolver.cs +++ b/GVFS/GVFS.Common/Http/CacheServerResolver.cs @@ -39,7 +39,7 @@ public static string GetUrlFromConfig(Enlistment enlistment) public bool TryResolveUrlFromRemote( string cacheServerName, - GVFSConfig gvfsConfig, + ServerGVFSConfig serverGVFSConfig, out CacheServerInfo cacheServer, out string error) { @@ -54,12 +54,12 @@ public bool TryResolveUrlFromRemote( if (cacheServerName.Equals(CacheServerInfo.ReservedNames.Default, StringComparison.OrdinalIgnoreCase)) { cacheServer = - gvfsConfig.CacheServers.FirstOrDefault(cache => cache.GlobalDefault) + serverGVFSConfig.CacheServers.FirstOrDefault(cache => cache.GlobalDefault) ?? this.CreateNone(); } else { - cacheServer = gvfsConfig.CacheServers.FirstOrDefault(cache => + cacheServer = serverGVFSConfig.CacheServers.FirstOrDefault(cache => cache.Name.Equals(cacheServerName, StringComparison.OrdinalIgnoreCase)); if (cacheServer == null) @@ -74,7 +74,7 @@ public bool TryResolveUrlFromRemote( public CacheServerInfo ResolveNameFromRemote( string cacheServerUrl, - GVFSConfig gvfsConfig) + ServerGVFSConfig serverGVFSConfig) { if (string.IsNullOrWhiteSpace(cacheServerUrl)) { @@ -87,7 +87,7 @@ public CacheServerInfo ResolveNameFromRemote( } return - gvfsConfig.CacheServers.FirstOrDefault(cache => cache.Url.Equals(cacheServerUrl, StringComparison.OrdinalIgnoreCase)) + serverGVFSConfig.CacheServers.FirstOrDefault(cache => cache.Url.Equals(cacheServerUrl, StringComparison.OrdinalIgnoreCase)) ?? new CacheServerInfo(cacheServerUrl, CacheServerInfo.ReservedNames.UserDefined); } diff --git a/GVFS/GVFS.Common/Http/ConfigHttpRequestor.cs b/GVFS/GVFS.Common/Http/ConfigHttpRequestor.cs index a44f079fd..8c6e450c1 100644 --- a/GVFS/GVFS.Common/Http/ConfigHttpRequestor.cs +++ b/GVFS/GVFS.Common/Http/ConfigHttpRequestor.cs @@ -16,9 +16,9 @@ public ConfigHttpRequestor(ITracer tracer, Enlistment enlistment, RetryConfig re this.repoUrl = enlistment.RepoUrl; } - public bool TryQueryGVFSConfig(out GVFSConfig gvfsConfig) + public bool TryQueryGVFSConfig(out ServerGVFSConfig serverGVFSConfig) { - gvfsConfig = null; + serverGVFSConfig = null; Uri gvfsConfigEndpoint; string gvfsConfigEndpointString = this.repoUrl + GVFSConstants.Endpoints.GVFSConfig; @@ -38,10 +38,10 @@ public bool TryQueryGVFSConfig(out GVFSConfig gvfsConfig) } long requestId = HttpRequestor.GetNewRequestId(); - RetryWrapper retrier = new RetryWrapper(this.RetryConfig.MaxAttempts, CancellationToken.None); - retrier.OnFailure += RetryWrapper.StandardErrorHandler(this.Tracer, requestId, "QueryGvfsConfig"); + RetryWrapper retrier = new RetryWrapper(this.RetryConfig.MaxAttempts, CancellationToken.None); + retrier.OnFailure += RetryWrapper.StandardErrorHandler(this.Tracer, requestId, "QueryGvfsConfig"); - RetryWrapper.InvocationResult output = retrier.Invoke( + RetryWrapper.InvocationResult output = retrier.Invoke( tryCount => { using (GitEndPointResponseData response = this.SendRequest( @@ -53,25 +53,25 @@ public bool TryQueryGVFSConfig(out GVFSConfig gvfsConfig) { if (response.HasErrors) { - return new RetryWrapper.CallbackResult(response.Error, response.ShouldRetry); + return new RetryWrapper.CallbackResult(response.Error, response.ShouldRetry); } try { string configString = response.RetryableReadToEnd(); - GVFSConfig config = JsonConvert.DeserializeObject(configString); - return new RetryWrapper.CallbackResult(config); + ServerGVFSConfig config = JsonConvert.DeserializeObject(configString); + return new RetryWrapper.CallbackResult(config); } catch (JsonReaderException e) { - return new RetryWrapper.CallbackResult(e, shouldRetry: false); + return new RetryWrapper.CallbackResult(e, shouldRetry: false); } } }); if (output.Succeeded) { - gvfsConfig = output.Result; + serverGVFSConfig = output.Result; return true; } diff --git a/GVFS/GVFS.Common/LocalCacheResolver.cs b/GVFS/GVFS.Common/LocalCacheResolver.cs index 12a5a7c23..9e4cba525 100644 --- a/GVFS/GVFS.Common/LocalCacheResolver.cs +++ b/GVFS/GVFS.Common/LocalCacheResolver.cs @@ -44,15 +44,15 @@ public static string GetDefaultLocalCacheRoot(GVFSEnlistment enlistment) public bool TryGetLocalCacheKeyFromLocalConfigOrRemoteCacheServers( ITracer tracer, - GVFSConfig gvfsConfig, + ServerGVFSConfig serverGVFSConfig, CacheServerInfo currentCacheServer, string localCacheRoot, out string localCacheKey, out string errorMessage) { - if (gvfsConfig == null) + if (serverGVFSConfig == null) { - throw new ArgumentNullException(nameof(gvfsConfig)); + throw new ArgumentNullException(nameof(serverGVFSConfig)); } localCacheKey = null; @@ -115,7 +115,7 @@ public bool TryGetLocalCacheKeyFromLocalConfigOrRemoteCacheServers( metadata.Add("currentCacheServer", currentCacheServer.ToString()); string getLocalCacheKeyError; - if (this.TryGetLocalCacheKeyFromRemoteCacheServers(tracer, gvfsConfig, currentCacheServer, mappingFile, out localCacheKey, out getLocalCacheKeyError)) + if (this.TryGetLocalCacheKeyFromRemoteCacheServers(tracer, serverGVFSConfig, currentCacheServer, mappingFile, out localCacheKey, out getLocalCacheKeyError)) { metadata.Add("localCacheKey", localCacheKey); metadata.Add(TracingConstants.MessageKey.InfoMessage, nameof(this.TryGetLocalCacheKeyFromLocalConfigOrRemoteCacheServers) + ": Generated new local cache key"); @@ -192,7 +192,7 @@ private bool TryOpenMappingFile(ITracer tracer, string localCacheRoot, out FileB private bool TryGetLocalCacheKeyFromRemoteCacheServers( ITracer tracer, - GVFSConfig gvfsConfig, + ServerGVFSConfig serverGVFSConfig, CacheServerInfo currentCacheServer, FileBasedDictionary mappingFile, out string localCacheKey, @@ -203,7 +203,7 @@ private bool TryGetLocalCacheKeyFromRemoteCacheServers( try { - if (this.TryFindExistingLocalCacheKey(mappingFile, gvfsConfig.CacheServers, out localCacheKey)) + if (this.TryFindExistingLocalCacheKey(mappingFile, serverGVFSConfig.CacheServers, out localCacheKey)) { EventMetadata metadata = CreateEventMetadata(); metadata.Add("currentCacheServer", currentCacheServer.ToString()); @@ -233,7 +233,7 @@ private bool TryGetLocalCacheKeyFromRemoteCacheServers( mappingFileUpdates.Add(new KeyValuePair(this.ToMappingKey(currentCacheServer.Url), localCacheKey)); } - foreach (CacheServerInfo cacheServer in gvfsConfig.CacheServers) + foreach (CacheServerInfo cacheServer in serverGVFSConfig.CacheServers) { string persistedLocalCacheKey; if (mappingFile.TryGetValue(this.ToMappingKey(cacheServer.Url), out persistedLocalCacheKey)) diff --git a/GVFS/GVFS.Common/LocalGVFSConfig.cs b/GVFS/GVFS.Common/LocalGVFSConfig.cs new file mode 100644 index 000000000..62d4a4064 --- /dev/null +++ b/GVFS/GVFS.Common/LocalGVFSConfig.cs @@ -0,0 +1,99 @@ +using GVFS.Common.FileSystem; +using GVFS.Common.Git; +using System; +using System.IO; + +namespace GVFS.Common +{ + public class LocalGVFSConfig + { + private const string FileName = "local_config.dat"; + private string configFile; + private string gitPath; + private PhysicalFileSystem fileSystem; + + public LocalGVFSConfig(string gitPath) + { + string servicePath = Paths.GetServiceDataRoot(GVFSConstants.Service.ServiceName); + string gvfsDirectory = Path.GetDirectoryName(servicePath); + + this.configFile = Path.Combine(gvfsDirectory, FileName); + this.gitPath = gitPath; + this.fileSystem = new PhysicalFileSystem(); + } + + public bool TryGetValueForKey(string key, out string value, out string error) + { + return this.TryGetConfig(this.KeyWithGVFSSectionPrefix(key), out value, out error); + } + + public bool TrySetValueForKey(string key, string value, out string error) + { + return this.TrySetConfig(this.KeyWithGVFSSectionPrefix(key), value, out error); + } + + private bool TryGetConfig(string key, out string value, out string error) + { + if (!this.fileSystem.FileExists(this.configFile)) + { + error = $"Error reading {key}. Config file({this.configFile}) does not exist."; + value = null; + return false; + } + + GitProcess.Result result = GitProcess.GetFromFileConfig(this.gitPath, this.configFile, key); + if (!result.HasErrors && !string.IsNullOrEmpty(result.Output)) + { + error = null; + value = result.Output.TrimEnd('\r', '\n'); + return true; + } + else + { + error = string.IsNullOrEmpty(result.Errors) ? $"Error reading \"{key}\" from file {this.configFile}." : result.Errors; + value = null; + return false; + } + } + + private bool TrySetConfig(string key, string value, out string error) + { + if (!this.fileSystem.FileExists(this.configFile) && !this.TryCreateConfigFile(out error)) + { + error = $"Error setting config value {key}: {value}. {error}"; + return false; + } + + GitProcess git = new GitProcess(this.gitPath, workingDirectoryRoot: null, gvfsHooksRoot: null); + GitProcess.Result result = git.SetInFileConfig(this.configFile, key, value, replaceAll: true); + if (result.HasErrors) + { + error = string.IsNullOrEmpty(result.Errors) ? $"Error setting config value {key}: {value}. Config file {this.configFile}." : result.Errors; + return false; + } + + error = null; + return true; + } + + private bool TryCreateConfigFile(out string error) + { + Exception exception = null; + if (!this.fileSystem.TryWriteTempFileAndRename(this.configFile, string.Empty, out exception)) + { + error = $"Could not create config file {this.configFile}. {exception.Message}"; + return false; + } + + error = null; + return true; + } + + private string KeyWithGVFSSectionPrefix(string key) + { + const string GVFSSectionName = "gvfs"; + + return string.Join(".", GVFSSectionName, key); + } + } +} diff --git a/GVFS/GVFS.Common/GVFSConfig.cs b/GVFS/GVFS.Common/ServerGVFSConfig.cs similarity index 89% rename from GVFS/GVFS.Common/GVFSConfig.cs rename to GVFS/GVFS.Common/ServerGVFSConfig.cs index 17d805c39..1bc9bef36 100644 --- a/GVFS/GVFS.Common/GVFSConfig.cs +++ b/GVFS/GVFS.Common/ServerGVFSConfig.cs @@ -5,7 +5,7 @@ namespace GVFS.Common { - public class GVFSConfig + public class ServerGVFSConfig { public IEnumerable AllowedGVFSClientVersions { get; set; } @@ -17,4 +17,4 @@ public class VersionRange public Version Max { get; set; } } } -} +} \ No newline at end of file diff --git a/GVFS/GVFS.UnitTests/Common/CacheServerResolverTests.cs b/GVFS/GVFS.UnitTests/Common/CacheServerResolverTests.cs index c46d4b966..3a5026a09 100644 --- a/GVFS/GVFS.UnitTests/Common/CacheServerResolverTests.cs +++ b/GVFS/GVFS.UnitTests/Common/CacheServerResolverTests.cs @@ -204,9 +204,9 @@ private MockGVFSEnlistment CreateEnlistment(string newConfigValue = null, string return new MockGVFSEnlistment(gitProcess); } - private GVFSConfig CreateGVFSConfig() + private ServerGVFSConfig CreateGVFSConfig() { - return new GVFSConfig + return new ServerGVFSConfig { CacheServers = new[] { @@ -215,9 +215,9 @@ private GVFSConfig CreateGVFSConfig() }; } - private GVFSConfig CreateDefaultDeserializedGVFSConfig() + private ServerGVFSConfig CreateDefaultDeserializedGVFSConfig() { - return JsonConvert.DeserializeObject("{}"); + return JsonConvert.DeserializeObject("{}"); } private CacheServerResolver CreateResolver(MockGVFSEnlistment enlistment = null) diff --git a/GVFS/GVFS/CommandLine/CacheServerVerb.cs b/GVFS/GVFS/CommandLine/CacheServerVerb.cs index d922129f9..0209bf86e 100644 --- a/GVFS/GVFS/CommandLine/CacheServerVerb.cs +++ b/GVFS/GVFS/CommandLine/CacheServerVerb.cs @@ -42,7 +42,7 @@ protected override void Execute(GVFSEnlistment enlistment) using (ITracer tracer = new JsonTracer(GVFSConstants.GVFSEtwProviderName, "CacheVerb")) { - GVFSConfig gvfsConfig = this.QueryGVFSConfig(tracer, enlistment, retryConfig); + ServerGVFSConfig serverGVFSConfig = this.QueryGVFSConfig(tracer, enlistment, retryConfig); CacheServerResolver cacheServerResolver = new CacheServerResolver(tracer, enlistment); string error = null; @@ -50,7 +50,7 @@ protected override void Execute(GVFSEnlistment enlistment) if (this.CacheToSet != null) { CacheServerInfo cacheServer = cacheServerResolver.ParseUrlOrFriendlyName(this.CacheToSet); - cacheServer = this.ResolveCacheServer(tracer, cacheServer, cacheServerResolver, gvfsConfig); + cacheServer = this.ResolveCacheServer(tracer, cacheServer, cacheServerResolver, serverGVFSConfig); if (!cacheServerResolver.TrySaveUrlToLocalConfig(cacheServer, out error)) { @@ -61,7 +61,7 @@ protected override void Execute(GVFSEnlistment enlistment) } else if (this.ListCacheServers) { - List cacheServers = gvfsConfig.CacheServers.ToList(); + List cacheServers = serverGVFSConfig.CacheServers.ToList(); if (cacheServers != null && cacheServers.Any()) { @@ -80,7 +80,7 @@ protected override void Execute(GVFSEnlistment enlistment) else { string cacheServerUrl = CacheServerResolver.GetUrlFromConfig(enlistment); - CacheServerInfo cacheServer = cacheServerResolver.ResolveNameFromRemote(cacheServerUrl, gvfsConfig); + CacheServerInfo cacheServer = cacheServerResolver.ResolveNameFromRemote(cacheServerUrl, serverGVFSConfig); this.Output.WriteLine("Using cache server: " + cacheServer); } diff --git a/GVFS/GVFS/CommandLine/CloneVerb.cs b/GVFS/GVFS/CommandLine/CloneVerb.cs index ef59eea86..d3861f929 100644 --- a/GVFS/GVFS/CommandLine/CloneVerb.cs +++ b/GVFS/GVFS/CommandLine/CloneVerb.cs @@ -119,7 +119,7 @@ public override void Execute() Result cloneResult = new Result(false); CacheServerInfo cacheServer = null; - GVFSConfig gvfsConfig = null; + ServerGVFSConfig serverGVFSConfig = null; using (JsonTracer tracer = new JsonTracer(GVFSConstants.GVFSEtwProviderName, "GVFSClone")) { @@ -178,19 +178,19 @@ public override void Execute() } RetryConfig retryConfig = this.GetRetryConfig(tracer, enlistment, TimeSpan.FromMinutes(RetryConfig.FetchAndCloneTimeoutMinutes)); - gvfsConfig = this.QueryGVFSConfig(tracer, enlistment, retryConfig); + serverGVFSConfig = this.QueryGVFSConfig(tracer, enlistment, retryConfig); - cacheServer = this.ResolveCacheServer(tracer, cacheServer, cacheServerResolver, gvfsConfig); + cacheServer = this.ResolveCacheServer(tracer, cacheServer, cacheServerResolver, serverGVFSConfig); if (!GVFSPlatform.Instance.IsUnderConstruction) { - this.ValidateClientVersions(tracer, enlistment, gvfsConfig, showWarnings: true); + this.ValidateClientVersions(tracer, enlistment, serverGVFSConfig, showWarnings: true); } this.ShowStatusWhileRunning( () => { - cloneResult = this.TryClone(tracer, enlistment, cacheServer, retryConfig, gvfsConfig, resolvedLocalCacheRoot); + cloneResult = this.TryClone(tracer, enlistment, cacheServer, retryConfig, serverGVFSConfig, resolvedLocalCacheRoot); return cloneResult.Success; }, "Cloning", @@ -214,7 +214,7 @@ public override void Execute() verb.Commits = true; verb.SkipVersionCheck = true; verb.ResolvedCacheServer = cacheServer; - verb.GVFSConfig = gvfsConfig; + verb.ServerGVFSConfig = serverGVFSConfig; }); if (result != ReturnCode.Success) @@ -238,7 +238,7 @@ public override void Execute() verb.SkipMountedCheck = true; verb.SkipVersionCheck = true; verb.ResolvedCacheServer = cacheServer; - verb.DownloadedGVFSConfig = gvfsConfig; + verb.DownloadedGVFSConfig = serverGVFSConfig; }); } } @@ -323,7 +323,7 @@ private Result TryClone( GVFSEnlistment enlistment, CacheServerInfo cacheServer, RetryConfig retryConfig, - GVFSConfig gvfsConfig, + ServerGVFSConfig serverGVFSConfig, string resolvedLocalCacheRoot) { Result pipeResult; @@ -372,7 +372,7 @@ private Result TryClone( } string localCacheError; - if (!this.TryDetermineLocalCacheAndInitializePaths(tracer, enlistment, gvfsConfig, cacheServer, resolvedLocalCacheRoot, out localCacheError)) + if (!this.TryDetermineLocalCacheAndInitializePaths(tracer, enlistment, serverGVFSConfig, cacheServer, resolvedLocalCacheRoot, out localCacheError)) { tracer.RelatedError(localCacheError); return new Result(localCacheError); @@ -460,7 +460,7 @@ private void CheckNotInsideExistingRepo(string normalizedEnlistmentRootPath) private bool TryDetermineLocalCacheAndInitializePaths( ITracer tracer, GVFSEnlistment enlistment, - GVFSConfig gvfsConfig, + ServerGVFSConfig serverGVFSConfig, CacheServerInfo currentCacheServer, string localCacheRoot, out string errorMessage) @@ -472,7 +472,7 @@ private bool TryDetermineLocalCacheAndInitializePaths( string localCacheKey; if (!localCacheResolver.TryGetLocalCacheKeyFromLocalConfigOrRemoteCacheServers( tracer, - gvfsConfig, + serverGVFSConfig, currentCacheServer, localCacheRoot, localCacheKey: out localCacheKey, diff --git a/GVFS/GVFS/CommandLine/DehydrateVerb.cs b/GVFS/GVFS/CommandLine/DehydrateVerb.cs index 906745f3c..742e8c271 100644 --- a/GVFS/GVFS/CommandLine/DehydrateVerb.cs +++ b/GVFS/GVFS/CommandLine/DehydrateVerb.cs @@ -104,7 +104,7 @@ of your enlistment's src folder. } // Local cache and objects paths are required for TryDownloadGitObjects - this.InitializeLocalCacheAndObjectsPaths(tracer, enlistment, retryConfig, gvfsConfig: null, cacheServer: null); + this.InitializeLocalCacheAndObjectsPaths(tracer, enlistment, retryConfig, serverGVFSConfig: null, cacheServer: null); if (this.TryBackupFiles(tracer, enlistment, backupRoot)) { diff --git a/GVFS/GVFS/CommandLine/GVFSVerb.cs b/GVFS/GVFS/CommandLine/GVFSVerb.cs index 8b9f1a8b6..a01fe866d 100644 --- a/GVFS/GVFS/CommandLine/GVFSVerb.cs +++ b/GVFS/GVFS/CommandLine/GVFSVerb.cs @@ -244,15 +244,15 @@ protected RetryConfig GetRetryConfig(ITracer tracer, GVFSEnlistment enlistment, return retryConfig; } - protected GVFSConfig QueryGVFSConfig(ITracer tracer, GVFSEnlistment enlistment, RetryConfig retryConfig) + protected ServerGVFSConfig QueryGVFSConfig(ITracer tracer, GVFSEnlistment enlistment, RetryConfig retryConfig) { - GVFSConfig gvfsConfig = null; + ServerGVFSConfig serverGVFSConfig = null; if (!this.ShowStatusWhileRunning( () => { using (ConfigHttpRequestor configRequestor = new ConfigHttpRequestor(tracer, enlistment, retryConfig)) { - return configRequestor.TryQueryGVFSConfig(out gvfsConfig); + return configRequestor.TryQueryGVFSConfig(out serverGVFSConfig); } }, "Querying remote for config", @@ -260,11 +260,11 @@ protected GVFSConfig QueryGVFSConfig(ITracer tracer, GVFSEnlistment enlistment, { this.ReportErrorAndExit(tracer, "Unable to query /gvfs/config"); } + + return serverGVFSConfig; + } - return gvfsConfig; - } - - protected void ValidateClientVersions(ITracer tracer, GVFSEnlistment enlistment, GVFSConfig gvfsConfig, bool showWarnings) + protected void ValidateClientVersions(ITracer tracer, GVFSEnlistment enlistment, ServerGVFSConfig gvfsConfig, bool showWarnings) { if (!GVFSPlatform.Instance.IsUnderConstruction) { @@ -354,7 +354,7 @@ protected CacheServerInfo ResolveCacheServer( ITracer tracer, CacheServerInfo cacheServer, CacheServerResolver cacheServerResolver, - GVFSConfig gvfsConfig) + ServerGVFSConfig serverGVFSConfig) { CacheServerInfo resolvedCacheServer = cacheServer; @@ -365,7 +365,7 @@ protected CacheServerInfo ResolveCacheServer( if (!cacheServerResolver.TryResolveUrlFromRemote( cacheServerName, - gvfsConfig, + serverGVFSConfig, out resolvedCacheServer, out error)) { @@ -374,7 +374,7 @@ protected CacheServerInfo ResolveCacheServer( } else if (cacheServer.Name.Equals(CacheServerInfo.ReservedNames.UserDefined)) { - resolvedCacheServer = cacheServerResolver.ResolveNameFromRemote(cacheServer.Url, gvfsConfig); + resolvedCacheServer = cacheServerResolver.ResolveNameFromRemote(cacheServer.Url, serverGVFSConfig); } this.Output.WriteLine("Using cache server: " + resolvedCacheServer); @@ -660,7 +660,7 @@ private void CheckGitVersion(ITracer tracer, GVFSEnlistment enlistment, out stri } } - private bool TryValidateGVFSVersion(GVFSEnlistment enlistment, ITracer tracer, GVFSConfig config, out string errorMessage, out bool errorIsFatal) + private bool TryValidateGVFSVersion(GVFSEnlistment enlistment, ITracer tracer, ServerGVFSConfig config, out string errorMessage, out bool errorIsFatal) { errorMessage = null; errorIsFatal = false; @@ -669,7 +669,7 @@ private bool TryValidateGVFSVersion(GVFSEnlistment enlistment, ITracer tracer, G { Version currentVersion = new Version(ProcessHelper.GetCurrentProcessVersion()); - IEnumerable allowedGvfsClientVersions = + IEnumerable allowedGvfsClientVersions = config != null ? config.AllowedGVFSClientVersions : null; @@ -692,7 +692,7 @@ private bool TryValidateGVFSVersion(GVFSEnlistment enlistment, ITracer tracer, G return false; } - foreach (GVFSConfig.VersionRange versionRange in config.AllowedGVFSClientVersions) + foreach (ServerGVFSConfig.VersionRange versionRange in config.AllowedGVFSClientVersions) { if (currentVersion >= versionRange.Min && (versionRange.Max == null || currentVersion <= versionRange.Max)) @@ -751,7 +751,7 @@ protected void InitializeLocalCacheAndObjectsPaths( ITracer tracer, GVFSEnlistment enlistment, RetryConfig retryConfig, - GVFSConfig gvfsConfig, + ServerGVFSConfig serverGVFSConfig, CacheServerInfo cacheServer) { string error; @@ -765,7 +765,7 @@ protected void InitializeLocalCacheAndObjectsPaths( // Note: Repos cloned with a version of GVFS that predates the local cache will not have a local cache configured if (!string.IsNullOrWhiteSpace(enlistment.LocalCacheRoot)) { - this.EnsureLocalCacheIsHealthy(tracer, enlistment, retryConfig, gvfsConfig, cacheServer); + this.EnsureLocalCacheIsHealthy(tracer, enlistment, retryConfig, serverGVFSConfig, cacheServer); } RepoMetadata.Shutdown(); @@ -813,7 +813,7 @@ private void EnsureLocalCacheIsHealthy( ITracer tracer, GVFSEnlistment enlistment, RetryConfig retryConfig, - GVFSConfig gvfsConfig, + ServerGVFSConfig serverGVFSConfig, CacheServerInfo cacheServer) { if (!Directory.Exists(enlistment.LocalCacheRoot)) @@ -901,7 +901,7 @@ private void EnsureLocalCacheIsHealthy( } string error; - if (gvfsConfig == null) + if (serverGVFSConfig == null) { if (retryConfig == null) { @@ -911,14 +911,14 @@ private void EnsureLocalCacheIsHealthy( } } - gvfsConfig = this.QueryGVFSConfig(tracer, enlistment, retryConfig); + serverGVFSConfig = this.QueryGVFSConfig(tracer, enlistment, retryConfig); } string localCacheKey; LocalCacheResolver localCacheResolver = new LocalCacheResolver(enlistment); if (!localCacheResolver.TryGetLocalCacheKeyFromLocalConfigOrRemoteCacheServers( tracer, - gvfsConfig, + serverGVFSConfig, cacheServer, enlistment.LocalCacheRoot, localCacheKey: out localCacheKey, diff --git a/GVFS/GVFS/CommandLine/MountVerb.cs b/GVFS/GVFS/CommandLine/MountVerb.cs index 9487b2f1e..ad8e0a2f4 100644 --- a/GVFS/GVFS/CommandLine/MountVerb.cs +++ b/GVFS/GVFS/CommandLine/MountVerb.cs @@ -37,7 +37,7 @@ public class MountVerb : GVFSVerb.ForExistingEnlistment public bool SkipMountedCheck { get; set; } public bool SkipVersionCheck { get; set; } public CacheServerInfo ResolvedCacheServer { get; set; } - public GVFSConfig DownloadedGVFSConfig { get; set; } + public ServerGVFSConfig DownloadedGVFSConfig { get; set; } protected override string VerbName { @@ -128,7 +128,7 @@ protected override void Execute(GVFSEnlistment enlistment) } RetryConfig retryConfig = null; - GVFSConfig gvfsConfig = this.DownloadedGVFSConfig; + ServerGVFSConfig serverGVFSConfig = this.DownloadedGVFSConfig; if (!this.SkipVersionCheck) { string authErrorMessage = null; @@ -140,24 +140,24 @@ protected override void Execute(GVFSEnlistment enlistment) this.Output.WriteLine(" Mount will proceed, but new files cannot be accessed until GVFS can authenticate."); } - if (gvfsConfig == null) + if (serverGVFSConfig == null) { if (retryConfig == null) { retryConfig = this.GetRetryConfig(tracer, enlistment); } - gvfsConfig = this.QueryGVFSConfig(tracer, enlistment, retryConfig); + serverGVFSConfig = this.QueryGVFSConfig(tracer, enlistment, retryConfig); } - this.ValidateClientVersions(tracer, enlistment, gvfsConfig, showWarnings: true); + this.ValidateClientVersions(tracer, enlistment, serverGVFSConfig, showWarnings: true); CacheServerResolver cacheServerResolver = new CacheServerResolver(tracer, enlistment); - cacheServer = cacheServerResolver.ResolveNameFromRemote(cacheServer.Url, gvfsConfig); + cacheServer = cacheServerResolver.ResolveNameFromRemote(cacheServer.Url, serverGVFSConfig); this.Output.WriteLine("Configured cache server: " + cacheServer); } - this.InitializeLocalCacheAndObjectsPaths(tracer, enlistment, retryConfig, gvfsConfig, cacheServer); + this.InitializeLocalCacheAndObjectsPaths(tracer, enlistment, retryConfig, serverGVFSConfig, cacheServer); if (!this.ShowStatusWhileRunning( () => { return this.PerformPreMountValidation(tracer, enlistment, out mountExecutableLocation, out errorMessage); }, diff --git a/GVFS/GVFS/CommandLine/PrefetchVerb.cs b/GVFS/GVFS/CommandLine/PrefetchVerb.cs index aa9d1c1be..acea80b6b 100644 --- a/GVFS/GVFS/CommandLine/PrefetchVerb.cs +++ b/GVFS/GVFS/CommandLine/PrefetchVerb.cs @@ -70,7 +70,7 @@ public class PrefetchVerb : GVFSVerb.ForExistingEnlistment public bool SkipVersionCheck { get; set; } public CacheServerInfo ResolvedCacheServer { get; set; } - public GVFSConfig GVFSConfig { get; set; } + public ServerGVFSConfig ServerGVFSConfig { get; set; } protected override string VerbName { @@ -100,7 +100,7 @@ protected override void Execute(GVFSEnlistment enlistment) RetryConfig retryConfig = this.GetRetryConfig(tracer, enlistment, TimeSpan.FromMinutes(RetryConfig.FetchAndCloneTimeoutMinutes)); CacheServerInfo cacheServer = this.ResolvedCacheServer; - GVFSConfig gvfsConfig = this.GVFSConfig; + ServerGVFSConfig serverGVFSConfig = this.ServerGVFSConfig; if (!this.SkipVersionCheck) { string authErrorMessage; @@ -111,23 +111,23 @@ protected override void Execute(GVFSEnlistment enlistment) this.ReportErrorAndExit(tracer, "Unable to prefetch because authentication failed"); } - if (gvfsConfig == null) + if (serverGVFSConfig == null) { - gvfsConfig = this.QueryGVFSConfig(tracer, enlistment, retryConfig); + serverGVFSConfig = this.QueryGVFSConfig(tracer, enlistment, retryConfig); } if (cacheServer == null) { CacheServerResolver cacheServerResolver = new CacheServerResolver(tracer, enlistment); - cacheServer = cacheServerResolver.ResolveNameFromRemote(cacheServerUrl, gvfsConfig); + cacheServer = cacheServerResolver.ResolveNameFromRemote(cacheServerUrl, serverGVFSConfig); } - this.ValidateClientVersions(tracer, enlistment, gvfsConfig, showWarnings: false); + this.ValidateClientVersions(tracer, enlistment, serverGVFSConfig, showWarnings: false); this.Output.WriteLine("Configured cache server: " + cacheServer); } - this.InitializeLocalCacheAndObjectsPaths(tracer, enlistment, retryConfig, gvfsConfig, cacheServer); + this.InitializeLocalCacheAndObjectsPaths(tracer, enlistment, retryConfig, serverGVFSConfig, cacheServer); try { From 127435051345688dc7e1893124b387f6dd6b1ef8 Mon Sep 17 00:00:00 2001 From: Al Ameen Shah Date: Wed, 26 Sep 2018 10:41:19 -0400 Subject: [PATCH 205/272] - New ConfigVerb. - Usage gvfs config - New LocalGVFSConfig.cs file. It writes config key value pairs into a git config file. - Updated ProductUpgrader.cs to use the new LocalGVFSConfig class to read ring config. --- GVFS/GVFS.Common/GVFSConstants.cs | 4 +- GVFS/GVFS.Common/Git/GitProcess.cs | 15 +++++++ GVFS/GVFS.Common/ProductUpgrader.cs | 17 ++++---- GVFS/GVFS/CommandLine/ConfigVerb.cs | 61 +++++++++++++++++++++++++++++ GVFS/GVFS/GVFS.Windows.csproj | 3 +- GVFS/GVFS/Program.cs | 9 ++++- 6 files changed, 95 insertions(+), 14 deletions(-) create mode 100644 GVFS/GVFS/CommandLine/ConfigVerb.cs diff --git a/GVFS/GVFS.Common/GVFSConstants.cs b/GVFS/GVFS.Common/GVFSConstants.cs index 39fc28930..27d7d4da8 100644 --- a/GVFS/GVFS.Common/GVFSConstants.cs +++ b/GVFS/GVFS.Common/GVFSConstants.cs @@ -33,7 +33,7 @@ public static class GitConfig public const string HooksPrefix = GitConfig.GVFSPrefix + "clone.default-"; public const string GVFSTelemetryId = GitConfig.GVFSPrefix + "telemetry-id"; public const string HooksExtension = ".hooks"; - public const string UpgradeRing = GVFSPrefix + "upgrade-ring"; + public const string UpgradeRing = "upgrade-ring"; } public static class GitStatusCache @@ -227,7 +227,7 @@ public static class UpgradeVerbMessages public const string GVFSUpgradeOptionalConfirm = "`gvfs upgrade [--confirm]`"; public const string NoneRingConsoleAlert = "Upgrade ring set to \"None\". No upgrade check was performed."; public const string InvalidRingConsoleAlert = "Upgrade ring is not set. No upgrade check was performed."; - public const string SetUpgradeRingCommand = "To set or change upgrade ring, run `git config --global " + GitConfig.UpgradeRing + " [\"Fast\"|\"Slow\"|\"None\"] from a command prompt."; + public const string SetUpgradeRingCommand = "To set or change upgrade ring, run `gvfs config " + GitConfig.UpgradeRing + " [\"Fast\"|\"Slow\"|\"None\"] from an elevated command prompt."; public const string ReminderNotification = "A new version of GVFS is available. Run " + UpgradeVerbMessages.GVFSUpgrade + " to start the upgrade."; public const string UnmountRepoWarning = "The upgrade process will unmount all GVFS repositories for several minutes and remount them when it is complete. Ensure you are at a stopping point prior to upgrading."; public const string UpgradeInstallAdvice = "When you are ready, run " + UpgradeVerbMessages.GVFSUpgradeConfirm + " from an elevated command prompt."; diff --git a/GVFS/GVFS.Common/Git/GitProcess.cs b/GVFS/GVFS.Common/Git/GitProcess.cs index 2d3a2206e..8701605ab 100644 --- a/GVFS/GVFS.Common/Git/GitProcess.cs +++ b/GVFS/GVFS.Common/Git/GitProcess.cs @@ -87,6 +87,11 @@ public static Result GetFromSystemConfig(string gitBinPath, string settingName) return new GitProcess(gitBinPath, workingDirectoryRoot: null, gvfsHooksRoot: null).InvokeGitOutsideEnlistment("config --system " + settingName); } + public static Result GetFromFileConfig(string gitBinPath, string configFile, string settingName) + { + return new GitProcess(gitBinPath, workingDirectoryRoot: null, gvfsHooksRoot: null).InvokeGitOutsideEnlistment("config --file " + configFile + " " + settingName); + } + public static bool TryGetVersion(out GitVersion gitVersion, out string error) { string gitPath = GVFSPlatform.Instance.GitInstallation.GetInstalledGitBinPath(); @@ -203,6 +208,16 @@ public Result AddInLocalConfig(string settingName, string value) value)); } + public Result SetInFileConfig(string configFile, string settingName, string value, bool replaceAll = false) + { + return this.InvokeGitOutsideEnlistment(string.Format( + "config --file {0} {1} \"{2}\" \"{3}\"", + configFile, + replaceAll ? "--replace-all " : string.Empty, + settingName, + value)); + } + public bool TryGetAllConfig(bool localOnly, out Dictionary configSettings) { string localParameter = localOnly ? "--local" : string.Empty; diff --git a/GVFS/GVFS.Common/ProductUpgrader.cs b/GVFS/GVFS.Common/ProductUpgrader.cs index af0a7e813..d65421064 100644 --- a/GVFS/GVFS.Common/ProductUpgrader.cs +++ b/GVFS/GVFS.Common/ProductUpgrader.cs @@ -245,13 +245,14 @@ public bool TryCleanup(out string error) public virtual bool TryLoadRingConfig(out string error) { string gitPath = GVFSPlatform.Instance.GitInstallation.GetInstalledGitBinPath(); - GitProcess.Result result = GitProcess.GetFromGlobalConfig(gitPath, GVFSConstants.GitConfig.UpgradeRing); - if (!result.HasErrors && !string.IsNullOrEmpty(result.Output)) + LocalGVFSConfig localConfig = new LocalGVFSConfig(gitPath); + + string ringConfig = null; + if (localConfig.TryGetValueForKey(GVFSConstants.GitConfig.UpgradeRing, out ringConfig, out error)) { - string ringConfig = result.Output.TrimEnd('\r', '\n'); RingType ringType; - if (Enum.TryParse(ringConfig, ignoreCase: true, result: out ringType) && + if (Enum.TryParse(ringConfig, ignoreCase: true, result: out ringType) && Enum.IsDefined(typeof(RingType), ringType) && ringType != RingType.Invalid) { @@ -261,14 +262,10 @@ public virtual bool TryLoadRingConfig(out string error) } else { - error = "Invalid upgrade ring `" + ringConfig + "` specified in Git config."; + error = "Invalid upgrade ring `" + ringConfig + "` specified in Git config."; } } - else - { - error = string.IsNullOrEmpty(result.Errors) ? "Unable to determine upgrade ring." : result.Errors; - } - + error += Environment.NewLine + GVFSConstants.UpgradeVerbMessages.SetUpgradeRingCommand; this.Ring = RingType.Invalid; return false; diff --git a/GVFS/GVFS/CommandLine/ConfigVerb.cs b/GVFS/GVFS/CommandLine/ConfigVerb.cs new file mode 100644 index 000000000..4641f16c1 --- /dev/null +++ b/GVFS/GVFS/CommandLine/ConfigVerb.cs @@ -0,0 +1,61 @@ +using CommandLine; +using GVFS.Common; +using System; + +namespace GVFS.CommandLine +{ + [Verb(ConfigVerbName, HelpText = "Set and Get GVFS config settings.")] + public class ConfigVerb : GVFSVerb + { + private const string ConfigVerbName = "config"; + private LocalGVFSConfig localConfig; + + public override string EnlistmentRootPathParameter { get; set; } + + protected override string VerbName + { + get { return ConfigVerbName; } + } + + public override void Execute() + { + string[] args = Environment.GetCommandLineArgs(); + if (args.Length < 3 || args.Length > 4) + { + string usageString = string.Join( + Environment.NewLine, + "Error: wrong number of arguments.", + "Usage: gvfs config "); + this.ReportErrorAndExit(usageString); + } + + this.localConfig = new LocalGVFSConfig(GVFSPlatform.Instance.GitInstallation.GetInstalledGitBinPath()); + + string key = args[2]; + string value = null; + string error = null; + bool isRead = args.Length == 3; + + key = args[2]; + if (isRead) + { + if (this.localConfig.TryGetValueForKey(key, out value, out error)) + { + Console.WriteLine(value); + } + else + { + this.ReportErrorAndExit(error); + } + } + else + { + value = args[3]; + if (!this.localConfig.TrySetValueForKey(key, value, out error)) + { + this.ReportErrorAndExit(error); + } + } + } + } +} diff --git a/GVFS/GVFS/GVFS.Windows.csproj b/GVFS/GVFS/GVFS.Windows.csproj index d8de69db9..392c6cbb4 100644 --- a/GVFS/GVFS/GVFS.Windows.csproj +++ b/GVFS/GVFS/GVFS.Windows.csproj @@ -1,4 +1,4 @@ - + @@ -74,6 +74,7 @@ + diff --git a/GVFS/GVFS/Program.cs b/GVFS/GVFS/Program.cs index 43061a95a..e97ad3334 100644 --- a/GVFS/GVFS/Program.cs +++ b/GVFS/GVFS/Program.cs @@ -27,7 +27,8 @@ public static void Main(string[] args) typeof(ServiceVerb), typeof(StatusVerb), typeof(UnmountVerb), - typeof(UpgradeVerb) + typeof(UpgradeVerb), + typeof(ConfigVerb) }; int consoleWidth = 80; @@ -89,6 +90,12 @@ public static void Main(string[] args) upgrade.Execute(); Environment.Exit((int)ReturnCode.Success); }) + .WithParsed( + config => + { + config.Execute(); + Environment.Exit((int)ReturnCode.Success); + }) .WithParsed( verb => { From 1827893fa5cb0c8e1fa6286d99e2e46473dc37c8 Mon Sep 17 00:00:00 2001 From: Al Ameen Shah Date: Wed, 26 Sep 2018 13:23:04 -0400 Subject: [PATCH 206/272] - Replaced git config format and writer with FileBasedDictionary - New base class for Verbs that don't need an enlistment. - Cleanups --- GVFS/GVFS.Common/GVFSConstants.cs | 10 +- GVFS/GVFS.Common/LocalGVFSConfig.cs | 105 ++++++++++-------- GVFS/GVFS.Common/ProductUpgrader.cs | 6 +- .../Windows/Mock/MockProductUpgrader.cs | 2 +- .../Windows/Upgrader/UpgradeTests.cs | 2 +- GVFS/GVFS/CommandLine/ConfigVerb.cs | 43 +++---- GVFS/GVFS/CommandLine/GVFSVerb.cs | 13 +++ GVFS/GVFS/CommandLine/ServiceVerb.cs | 8 +- GVFS/GVFS/CommandLine/UpgradeVerb.cs | 4 +- GVFS/GVFS/Program.cs | 24 +--- 10 files changed, 112 insertions(+), 105 deletions(-) diff --git a/GVFS/GVFS.Common/GVFSConstants.cs b/GVFS/GVFS.Common/GVFSConstants.cs index 27d7d4da8..813663a0c 100644 --- a/GVFS/GVFS.Common/GVFSConstants.cs +++ b/GVFS/GVFS.Common/GVFSConstants.cs @@ -32,8 +32,12 @@ public static class GitConfig public const string DeprecatedCacheEndpointSuffix = ".cache-server-url"; public const string HooksPrefix = GitConfig.GVFSPrefix + "clone.default-"; public const string GVFSTelemetryId = GitConfig.GVFSPrefix + "telemetry-id"; - public const string HooksExtension = ".hooks"; - public const string UpgradeRing = "upgrade-ring"; + public const string HooksExtension = ".hooks"; + } + + public static class LocalGVFSConfig + { + public const string UpgradeRing = "upgrade.ring"; } public static class GitStatusCache @@ -227,7 +231,7 @@ public static class UpgradeVerbMessages public const string GVFSUpgradeOptionalConfirm = "`gvfs upgrade [--confirm]`"; public const string NoneRingConsoleAlert = "Upgrade ring set to \"None\". No upgrade check was performed."; public const string InvalidRingConsoleAlert = "Upgrade ring is not set. No upgrade check was performed."; - public const string SetUpgradeRingCommand = "To set or change upgrade ring, run `gvfs config " + GitConfig.UpgradeRing + " [\"Fast\"|\"Slow\"|\"None\"] from an elevated command prompt."; + public const string SetUpgradeRingCommand = "To set or change upgrade ring, run `gvfs config " + LocalGVFSConfig.UpgradeRing + " [\"Fast\"|\"Slow\"|\"None\"] from a command prompt."; public const string ReminderNotification = "A new version of GVFS is available. Run " + UpgradeVerbMessages.GVFSUpgrade + " to start the upgrade."; public const string UnmountRepoWarning = "The upgrade process will unmount all GVFS repositories for several minutes and remount them when it is complete. Ensure you are at a stopping point prior to upgrading."; public const string UpgradeInstallAdvice = "When you are ready, run " + UpgradeVerbMessages.GVFSUpgradeConfirm + " from an elevated command prompt."; diff --git a/GVFS/GVFS.Common/LocalGVFSConfig.cs b/GVFS/GVFS.Common/LocalGVFSConfig.cs index 62d4a4064..a7c225873 100644 --- a/GVFS/GVFS.Common/LocalGVFSConfig.cs +++ b/GVFS/GVFS.Common/LocalGVFSConfig.cs @@ -1,99 +1,112 @@ using GVFS.Common.FileSystem; -using GVFS.Common.Git; -using System; +using GVFS.Common.Tracing; using System.IO; namespace GVFS.Common { public class LocalGVFSConfig { - private const string FileName = "local_config.dat"; - private string configFile; - private string gitPath; - private PhysicalFileSystem fileSystem; + private const string FileName = "gvfs.config"; + private readonly string configFile; + private readonly PhysicalFileSystem fileSystem; - public LocalGVFSConfig(string gitPath) + public LocalGVFSConfig() { string servicePath = Paths.GetServiceDataRoot(GVFSConstants.Service.ServiceName); string gvfsDirectory = Path.GetDirectoryName(servicePath); this.configFile = Path.Combine(gvfsDirectory, FileName); - this.gitPath = gitPath; this.fileSystem = new PhysicalFileSystem(); } - public bool TryGetValueForKey(string key, out string value, out string error) - { - return this.TryGetConfig(this.KeyWithGVFSSectionPrefix(key), out value, out error); - } - - public bool TrySetValueForKey(string key, string value, out string error) - { - return this.TrySetConfig(this.KeyWithGVFSSectionPrefix(key), value, out error); - } + private FileBasedDictionary AllSettings { get; set; } - private bool TryGetConfig(string key, out string value, out string error) + public bool TryGetConfig( + string key, + out string value, + out string error, + ITracer tracer) { - if (!this.fileSystem.FileExists(this.configFile)) + if (!this.TryLoadSettings(tracer, out error)) { - error = $"Error reading {key}. Config file({this.configFile}) does not exist."; + error = $"Error getting config value {key}. {error}"; value = null; return false; } - - GitProcess.Result result = GitProcess.GetFromFileConfig(this.gitPath, this.configFile, key); - if (!result.HasErrors && !string.IsNullOrEmpty(result.Output)) + + try { + this.AllSettings.TryGetValue(key, out value); error = null; - value = result.Output.TrimEnd('\r', '\n'); return true; } - else + catch (FileBasedCollectionException exception) { - error = string.IsNullOrEmpty(result.Errors) ? $"Error reading \"{key}\" from file {this.configFile}." : result.Errors; + const string ErrorFormat = "Error getting config value for {0}. Config file {1}. {2}"; + if (tracer != null) + { + tracer.RelatedError(ErrorFormat, key, this.configFile, exception.ToString()); + } + + error = string.Format(ErrorFormat, key, this.configFile, exception.Message); value = null; return false; } } - private bool TrySetConfig(string key, string value, out string error) + public bool TrySetConfig( + string key, + string value, + out string error, + ITracer tracer) { - if (!this.fileSystem.FileExists(this.configFile) && !this.TryCreateConfigFile(out error)) + if (!this.TryLoadSettings(tracer, out error)) { error = $"Error setting config value {key}: {value}. {error}"; return false; } - GitProcess git = new GitProcess(this.gitPath, workingDirectoryRoot: null, gvfsHooksRoot: null); - GitProcess.Result result = git.SetInFileConfig(this.configFile, key, value, replaceAll: true); - if (result.HasErrors) + try { - error = string.IsNullOrEmpty(result.Errors) ? $"Error setting config value {key}: {value}. Config file {this.configFile}." : result.Errors; - return false; + this.AllSettings.SetValueAndFlush(key, value); + error = null; + return true; } + catch (FileBasedCollectionException exception) + { + const string ErrorFormat = "Error setting config value {0}: {1}. Config file {2}. {3}"; + if (tracer != null) + { + tracer.RelatedError(ErrorFormat, key, value, this.configFile, exception.ToString()); + } - error = null; - return true; + error = string.Format(ErrorFormat, key, value, this.configFile, exception.Message); + value = null; + return false; + } } - private bool TryCreateConfigFile(out string error) + private bool TryLoadSettings(ITracer tracer, out string error) { - Exception exception = null; - if (!this.fileSystem.TryWriteTempFileAndRename(this.configFile, string.Empty, out exception)) + if (this.AllSettings == null) { - error = $"Could not create config file {this.configFile}. {exception.Message}"; + FileBasedDictionary config = null; + if (FileBasedDictionary.TryCreate( + tracer: tracer, + dictionaryPath: this.configFile, + fileSystem: this.fileSystem, + output: out config, + error: out error)) + { + this.AllSettings = config; + return true; + } + return false; } error = null; return true; } - - private string KeyWithGVFSSectionPrefix(string key) - { - const string GVFSSectionName = "gvfs"; - - return string.Join(".", GVFSSectionName, key); - } } } diff --git a/GVFS/GVFS.Common/ProductUpgrader.cs b/GVFS/GVFS.Common/ProductUpgrader.cs index d65421064..eb023e1d8 100644 --- a/GVFS/GVFS.Common/ProductUpgrader.cs +++ b/GVFS/GVFS.Common/ProductUpgrader.cs @@ -245,10 +245,10 @@ public bool TryCleanup(out string error) public virtual bool TryLoadRingConfig(out string error) { string gitPath = GVFSPlatform.Instance.GitInstallation.GetInstalledGitBinPath(); - LocalGVFSConfig localConfig = new LocalGVFSConfig(gitPath); + LocalGVFSConfig localConfig = new LocalGVFSConfig(); string ringConfig = null; - if (localConfig.TryGetValueForKey(GVFSConstants.GitConfig.UpgradeRing, out ringConfig, out error)) + if (localConfig.TryGetConfig(GVFSConstants.LocalGVFSConfig.UpgradeRing, out ringConfig, out error, this.tracer)) { RingType ringType; @@ -262,7 +262,7 @@ public virtual bool TryLoadRingConfig(out string error) } else { - error = "Invalid upgrade ring `" + ringConfig + "` specified in Git config."; + error = "Invalid upgrade ring `" + ringConfig + "` specified in gvfs config."; } } diff --git a/GVFS/GVFS.UnitTests.Windows/Windows/Mock/MockProductUpgrader.cs b/GVFS/GVFS.UnitTests.Windows/Windows/Mock/MockProductUpgrader.cs index 3b8c30633..9227d2cd8 100644 --- a/GVFS/GVFS.UnitTests.Windows/Windows/Mock/MockProductUpgrader.cs +++ b/GVFS/GVFS.UnitTests.Windows/Windows/Mock/MockProductUpgrader.cs @@ -107,7 +107,7 @@ public override bool TryLoadRingConfig(out string error) if (this.LocalRingConfig == RingType.Invalid) { - error = "Invalid upgrade ring `Invalid` specified in Git config."; + error = "Invalid upgrade ring `Invalid` specified in gvfs config."; return false; } diff --git a/GVFS/GVFS.UnitTests.Windows/Windows/Upgrader/UpgradeTests.cs b/GVFS/GVFS.UnitTests.Windows/Windows/Upgrader/UpgradeTests.cs index 4ac28563b..da82211c3 100644 --- a/GVFS/GVFS.UnitTests.Windows/Windows/Upgrader/UpgradeTests.cs +++ b/GVFS/GVFS.UnitTests.Windows/Windows/Upgrader/UpgradeTests.cs @@ -56,7 +56,7 @@ public virtual void NoneLocalRing() [TestCase] public virtual void InvalidUpgradeRing() { - string errorString = "Invalid upgrade ring `Invalid` specified in Git config."; + string errorString = "Invalid upgrade ring `Invalid` specified in gvfs config."; this.ConfigureRunAndVerify( configure: () => { diff --git a/GVFS/GVFS/CommandLine/ConfigVerb.cs b/GVFS/GVFS/CommandLine/ConfigVerb.cs index 4641f16c1..24d072550 100644 --- a/GVFS/GVFS/CommandLine/ConfigVerb.cs +++ b/GVFS/GVFS/CommandLine/ConfigVerb.cs @@ -4,13 +4,27 @@ namespace GVFS.CommandLine { - [Verb(ConfigVerbName, HelpText = "Set and Get GVFS config settings.")] - public class ConfigVerb : GVFSVerb + [Verb(ConfigVerbName, HelpText = "Get and set GVFS options.")] + public class ConfigVerb : GVFSVerb.NonRepoVerb { private const string ConfigVerbName = "config"; private LocalGVFSConfig localConfig; - public override string EnlistmentRootPathParameter { get; set; } + [Value( + 0, + Required = true, + Default = "", + MetaName = "GVFS config setting name", + HelpText = "Name of GVFS config setting that is to be set or read")] + public string Key { get; set; } + + [Value( + 1, + Required = false, + Default = "", + MetaName = "GVFS config setting value", + HelpText = "Value of GVFS config setting to be set")] + public string Value { get; set; } protected override string VerbName { @@ -19,27 +33,15 @@ protected override string VerbName public override void Execute() { - string[] args = Environment.GetCommandLineArgs(); - if (args.Length < 3 || args.Length > 4) - { - string usageString = string.Join( - Environment.NewLine, - "Error: wrong number of arguments.", - "Usage: gvfs config "); - this.ReportErrorAndExit(usageString); - } - - this.localConfig = new LocalGVFSConfig(GVFSPlatform.Instance.GitInstallation.GetInstalledGitBinPath()); + this.localConfig = new LocalGVFSConfig(); - string key = args[2]; - string value = null; string error = null; - bool isRead = args.Length == 3; + bool isRead = string.IsNullOrEmpty(this.Value); - key = args[2]; if (isRead) { - if (this.localConfig.TryGetValueForKey(key, out value, out error)) + string value = null; + if (this.localConfig.TryGetConfig(this.Key, out value, out error, tracer: null)) { Console.WriteLine(value); } @@ -50,8 +52,7 @@ public override void Execute() } else { - value = args[3]; - if (!this.localConfig.TrySetValueForKey(key, value, out error)) + if (!this.localConfig.TrySetConfig(this.Key, this.Value, out error, tracer: null)) { this.ReportErrorAndExit(error); } diff --git a/GVFS/GVFS/CommandLine/GVFSVerb.cs b/GVFS/GVFS/CommandLine/GVFSVerb.cs index a01fe866d..bf5de3379 100644 --- a/GVFS/GVFS/CommandLine/GVFSVerb.cs +++ b/GVFS/GVFS/CommandLine/GVFSVerb.cs @@ -1041,6 +1041,19 @@ private GVFSEnlistment CreateEnlistment(string enlistmentRootPath) } } + public abstract class NonRepoVerb : GVFSVerb + { + public NonRepoVerb(bool validateOrigin = true) : base(validateOrigin) + { + } + + public override string EnlistmentRootPathParameter + { + get { throw new InvalidOperationException(); } + set { throw new InvalidOperationException(); } + } + } + public class VerbAbortedException : Exception { public VerbAbortedException(GVFSVerb verb) diff --git a/GVFS/GVFS/CommandLine/ServiceVerb.cs b/GVFS/GVFS/CommandLine/ServiceVerb.cs index 640c99a08..b547b3bcc 100644 --- a/GVFS/GVFS/CommandLine/ServiceVerb.cs +++ b/GVFS/GVFS/CommandLine/ServiceVerb.cs @@ -10,7 +10,7 @@ namespace GVFS.CommandLine { [Verb(ServiceVerbName, HelpText = "Runs commands for the GVFS service.")] - public class ServiceVerb : GVFSVerb + public class ServiceVerb : GVFSVerb.NonRepoVerb { private const string ServiceVerbName = "service"; @@ -35,12 +35,6 @@ public class ServiceVerb : GVFSVerb HelpText = "Prints a list of all mounted repos")] public bool List { get; set; } - public override string EnlistmentRootPathParameter - { - get { throw new InvalidOperationException(); } - set { throw new InvalidOperationException(); } - } - protected override string VerbName { get { return ServiceVerbName; } diff --git a/GVFS/GVFS/CommandLine/UpgradeVerb.cs b/GVFS/GVFS/CommandLine/UpgradeVerb.cs index 020a16fdc..e34f62402 100644 --- a/GVFS/GVFS/CommandLine/UpgradeVerb.cs +++ b/GVFS/GVFS/CommandLine/UpgradeVerb.cs @@ -9,7 +9,7 @@ namespace GVFS.CommandLine { [Verb(UpgradeVerbName, HelpText = "Checks for new GVFS release, downloads and installs it when available.")] - public class UpgradeVerb : GVFSVerb + public class UpgradeVerb : GVFSVerb.NonRepoVerb { private const string UpgradeVerbName = "upgrade"; private ITracer tracer; @@ -44,8 +44,6 @@ public UpgradeVerb() HelpText = "Pass in this flag to actually install the newest release")] public bool Confirmed { get; set; } - public override string EnlistmentRootPathParameter { get; set; } - protected override string VerbName { get { return UpgradeVerbName; } diff --git a/GVFS/GVFS/Program.cs b/GVFS/GVFS/Program.cs index e97ad3334..b72b8355d 100644 --- a/GVFS/GVFS/Program.cs +++ b/GVFS/GVFS/Program.cs @@ -28,7 +28,7 @@ public static void Main(string[] args) typeof(StatusVerb), typeof(UnmountVerb), typeof(UpgradeVerb), - typeof(ConfigVerb) + typeof(ConfigVerb), }; int consoleWidth = 80; @@ -74,26 +74,10 @@ public static void Main(string[] args) clone.Execute(); Environment.Exit((int)ReturnCode.Success); }) - .WithParsed( - service => - { - // The service verb doesn't operate on a repo, so it doesn't use the enlistment - // path at all. - service.Execute(); - Environment.Exit((int)ReturnCode.Success); - }) - .WithParsed( - upgrade => - { - // The upgrade verb doesn't operate on a repo, so it doesn't use the enlistment - // path at all. - upgrade.Execute(); - Environment.Exit((int)ReturnCode.Success); - }) - .WithParsed( - config => + .WithParsed( + verb => { - config.Execute(); + verb.Execute(); Environment.Exit((int)ReturnCode.Success); }) .WithParsed( From bab403f406b54dbf9afa37ffe2f42a48e782ada5 Mon Sep 17 00:00:00 2001 From: Al Ameen Shah Date: Wed, 26 Sep 2018 16:56:37 -0400 Subject: [PATCH 207/272] - Allow gvfs config to succeed from a Non-elevated command prompt. GVFS Config is written to a temporary file and then renamed to its final destination. For this rename operation to succeed, user needs to have delete permission on the destination file, in case it is pre-existing. If the pre-existing file was created by a different user, then the delete will fail. Reference: https://stackoverflow.com/questions/22107812/privileges-owner-issue-when-writing-in-c-programdata. This work around allows safe write to succeed in C:\ProgramData directory. --- GVFS/GVFS.Service/GvfsService.cs | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/GVFS/GVFS.Service/GvfsService.cs b/GVFS/GVFS.Service/GvfsService.cs index 4a51e988d..9f47a6f7e 100644 --- a/GVFS/GVFS.Service/GvfsService.cs +++ b/GVFS/GVFS.Service/GvfsService.cs @@ -8,6 +8,8 @@ using System.Linq; using System.Reflection; using System.Runtime.Serialization; +using System.Security.AccessControl; +using System.Security.Principal; using System.ServiceProcess; using System.Threading; @@ -150,6 +152,7 @@ protected override void OnStart(string[] args) this.serviceDataLocation = Paths.GetServiceDataRoot(this.serviceName); Directory.CreateDirectory(this.serviceDataLocation); + this.EnableAccessToAuthenticatedUsers(Path.GetDirectoryName(this.serviceDataLocation)); this.tracer.AddLogFileEventListener( GVFSEnlistment.GetNewGVFSLogFileName(Paths.GetServiceLogsPath(this.serviceName), GVFSConstants.LogFileTypes.Service), @@ -344,5 +347,20 @@ private void LogExceptionAndExit(Exception e, string method) this.tracer.RelatedError(metadata, "Unhandled exception in " + method); Environment.Exit((int)ReturnCode.GenericError); } + + private void EnableAccessToAuthenticatedUsers(string rootDirectory) + { + // GVFS Config is written to a temporary file and then renamed to its final destination. + // For this rename operation to succeed, user needs to have delete permission on the + // destination file, in case it is pre-existing. If the pre-existing file was created + // by a different user, then the delete will fail. + // Reference: https://stackoverflow.com/questions/22107812/privileges-owner-issue-when-writing-in-c-programdata + // This work around allows safe write to succeed in C:\ProgramData directory. + + DirectorySecurity security = Directory.GetAccessControl(Path.GetDirectoryName(this.serviceDataLocation)); + SecurityIdentifier authenticatedUsers = new SecurityIdentifier(WellKnownSidType.AuthenticatedUserSid, null); + security.AddAccessRule(new FileSystemAccessRule(authenticatedUsers, FileSystemRights.FullControl, AccessControlType.Allow)); + Directory.SetAccessControl(Path.GetDirectoryName(this.serviceDataLocation), security); + } } } From 380fd992352313434221011e8f026fe7337b005c Mon Sep 17 00:00:00 2001 From: Al Ameen Shah Date: Wed, 26 Sep 2018 17:14:33 -0400 Subject: [PATCH 208/272] - Renamed GVFSVerb.NonRepoVerb to GVFSVerb.ForNoEnlistment. --- GVFS/GVFS/CommandLine/ConfigVerb.cs | 2 +- GVFS/GVFS/CommandLine/GVFSVerb.cs | 4 ++-- GVFS/GVFS/CommandLine/ServiceVerb.cs | 2 +- GVFS/GVFS/CommandLine/UpgradeVerb.cs | 2 +- GVFS/GVFS/Program.cs | 2 +- 5 files changed, 6 insertions(+), 6 deletions(-) diff --git a/GVFS/GVFS/CommandLine/ConfigVerb.cs b/GVFS/GVFS/CommandLine/ConfigVerb.cs index 24d072550..c978888da 100644 --- a/GVFS/GVFS/CommandLine/ConfigVerb.cs +++ b/GVFS/GVFS/CommandLine/ConfigVerb.cs @@ -5,7 +5,7 @@ namespace GVFS.CommandLine { [Verb(ConfigVerbName, HelpText = "Get and set GVFS options.")] - public class ConfigVerb : GVFSVerb.NonRepoVerb + public class ConfigVerb : GVFSVerb.ForNoEnlistment { private const string ConfigVerbName = "config"; private LocalGVFSConfig localConfig; diff --git a/GVFS/GVFS/CommandLine/GVFSVerb.cs b/GVFS/GVFS/CommandLine/GVFSVerb.cs index bf5de3379..d38a28ed4 100644 --- a/GVFS/GVFS/CommandLine/GVFSVerb.cs +++ b/GVFS/GVFS/CommandLine/GVFSVerb.cs @@ -1041,9 +1041,9 @@ private GVFSEnlistment CreateEnlistment(string enlistmentRootPath) } } - public abstract class NonRepoVerb : GVFSVerb + public abstract class ForNoEnlistment : GVFSVerb { - public NonRepoVerb(bool validateOrigin = true) : base(validateOrigin) + public ForNoEnlistment(bool validateOrigin = true) : base(validateOrigin) { } diff --git a/GVFS/GVFS/CommandLine/ServiceVerb.cs b/GVFS/GVFS/CommandLine/ServiceVerb.cs index b547b3bcc..e701a74c8 100644 --- a/GVFS/GVFS/CommandLine/ServiceVerb.cs +++ b/GVFS/GVFS/CommandLine/ServiceVerb.cs @@ -10,7 +10,7 @@ namespace GVFS.CommandLine { [Verb(ServiceVerbName, HelpText = "Runs commands for the GVFS service.")] - public class ServiceVerb : GVFSVerb.NonRepoVerb + public class ServiceVerb : GVFSVerb.ForNoEnlistment { private const string ServiceVerbName = "service"; diff --git a/GVFS/GVFS/CommandLine/UpgradeVerb.cs b/GVFS/GVFS/CommandLine/UpgradeVerb.cs index e34f62402..282dacea2 100644 --- a/GVFS/GVFS/CommandLine/UpgradeVerb.cs +++ b/GVFS/GVFS/CommandLine/UpgradeVerb.cs @@ -9,7 +9,7 @@ namespace GVFS.CommandLine { [Verb(UpgradeVerbName, HelpText = "Checks for new GVFS release, downloads and installs it when available.")] - public class UpgradeVerb : GVFSVerb.NonRepoVerb + public class UpgradeVerb : GVFSVerb.ForNoEnlistment { private const string UpgradeVerbName = "upgrade"; private ITracer tracer; diff --git a/GVFS/GVFS/Program.cs b/GVFS/GVFS/Program.cs index b72b8355d..4e52ee1d9 100644 --- a/GVFS/GVFS/Program.cs +++ b/GVFS/GVFS/Program.cs @@ -74,7 +74,7 @@ public static void Main(string[] args) clone.Execute(); Environment.Exit((int)ReturnCode.Success); }) - .WithParsed( + .WithParsed( verb => { verb.Execute(); From d263eb165904c2dad2fad3a8f430eac1bc76d8d1 Mon Sep 17 00:00:00 2001 From: Al Ameen Shah Date: Fri, 28 Sep 2018 17:36:55 -0400 Subject: [PATCH 209/272] - Support for New Asset installer name format. - New GVFSPlatform constant for installer file name extension. - Renamed installer file name from "SetupGVFS.version.exe" to "VFSGit.version.exe" - Updated UT & FT. --- GVFS/GVFS.Common/GVFSPlatform.cs | 9 ++++++--- GVFS/GVFS.Common/ProductUpgrader.Shared.cs | 2 +- GVFS/GVFS.Common/ProductUpgrader.cs | 6 +++--- .../EnlistmentPerFixture/GVFSUpgradeReminderTests.cs | 2 +- GVFS/GVFS.Platform.Mac/MacPlatform.cs | 2 +- GVFS/GVFS.Platform.Windows/WindowsPlatform.cs | 2 +- .../Windows/Mock/MockProductUpgrader.cs | 4 ++-- GVFS/GVFS.UnitTests/Mock/Common/MockPlatform.cs | 2 +- GVFS/GVFS.Upgrader/UpgradeOrchestrator.cs | 1 - 9 files changed, 16 insertions(+), 14 deletions(-) diff --git a/GVFS/GVFS.Common/GVFSPlatform.cs b/GVFS/GVFS.Common/GVFSPlatform.cs index db124104b..4341b589f 100644 --- a/GVFS/GVFS.Common/GVFSPlatform.cs +++ b/GVFS/GVFS.Common/GVFSPlatform.cs @@ -10,9 +10,9 @@ namespace GVFS.Common { public abstract class GVFSPlatform { - public GVFSPlatform(string executableExtension) + public GVFSPlatform(string executableExtension, string installerExtension) { - this.Constants = new GVFSPlatformConstants(executableExtension); + this.Constants = new GVFSPlatformConstants(executableExtension, installerExtension); } public static GVFSPlatform Instance { get; private set; } @@ -81,12 +81,15 @@ public bool TryGetNormalizedPathRoot(string path, out string pathRoot, out strin public class GVFSPlatformConstants { - public GVFSPlatformConstants(string executableExtension) + public GVFSPlatformConstants(string executableExtension, string installerExtension) { this.ExecutableExtension = executableExtension; + this.InstallerExtension = installerExtension; } public string ExecutableExtension { get; } + public string InstallerExtension { get; } + public string GVFSExecutableName { get { return "GVFS" + this.ExecutableExtension; } diff --git a/GVFS/GVFS.Common/ProductUpgrader.Shared.cs b/GVFS/GVFS.Common/ProductUpgrader.Shared.cs index 87d5438b1..edd4db874 100644 --- a/GVFS/GVFS.Common/ProductUpgrader.Shared.cs +++ b/GVFS/GVFS.Common/ProductUpgrader.Shared.cs @@ -11,7 +11,7 @@ public partial class ProductUpgrader public const string DownloadDirectory = "Downloads"; private const string RootDirectory = UpgradeDirectoryName; - private const string GVFSInstallerFileNamePrefix = "SetupGVFS"; + private const string GVFSInstallerFileNamePrefix = "VFSGit"; public static bool IsLocalUpgradeAvailable() { diff --git a/GVFS/GVFS.Common/ProductUpgrader.cs b/GVFS/GVFS.Common/ProductUpgrader.cs index eb023e1d8..d95c30624 100644 --- a/GVFS/GVFS.Common/ProductUpgrader.cs +++ b/GVFS/GVFS.Common/ProductUpgrader.cs @@ -131,7 +131,8 @@ public bool TryDownloadNewestVersion(out string errorMessage) foreach (Asset asset in this.newestRelease.Assets) { - if (!this.TryDownloadAsset(asset, out errorMessage)) + if (string.Equals(Path.GetExtension(asset.Name), GVFSPlatform.Instance.Constants.InstallerExtension, StringComparison.OrdinalIgnoreCase) && + !this.TryDownloadAsset(asset, out errorMessage)) { return false; } @@ -410,8 +411,7 @@ private bool TryGetLocalInstallerPath(string name, out string path, out string a { foreach (Asset asset in this.newestRelease.Assets) { - string extension = Path.GetExtension(asset.Name); - if (string.Equals(extension, ".exe", StringComparison.OrdinalIgnoreCase)) + if (string.Equals(Path.GetExtension(asset.Name), GVFSPlatform.Instance.Constants.InstallerExtension, StringComparison.OrdinalIgnoreCase)) { path = asset.LocalPath; if (name == GitAssetNamePrefix && asset.Name.StartsWith(GitInstallerFileNamePrefix, StringComparison.OrdinalIgnoreCase)) diff --git a/GVFS/GVFS.FunctionalTests/Tests/EnlistmentPerFixture/GVFSUpgradeReminderTests.cs b/GVFS/GVFS.FunctionalTests/Tests/EnlistmentPerFixture/GVFSUpgradeReminderTests.cs index 376e35ea5..54e4c8eff 100644 --- a/GVFS/GVFS.FunctionalTests/Tests/EnlistmentPerFixture/GVFSUpgradeReminderTests.cs +++ b/GVFS/GVFS.FunctionalTests/Tests/EnlistmentPerFixture/GVFSUpgradeReminderTests.cs @@ -14,7 +14,7 @@ namespace GVFS.FunctionalTests.Tests.EnlistmentPerFixture [Category(Categories.WindowsOnly)] public class UpgradeReminderTests : TestsWithEnlistmentPerFixture { - private const string GVFSInstallerName = "SetupGVFS.1.0.18234.1.exe"; + private const string GVFSInstallerName = "VFSGit.1.0.18234.1.exe"; private const string GitInstallerName = "Git-2.17.1.gvfs.2.5.g2962052-64-bit.exe"; private string upgradeDirectory; diff --git a/GVFS/GVFS.Platform.Mac/MacPlatform.cs b/GVFS/GVFS.Platform.Mac/MacPlatform.cs index 6dade62c7..06dcc527a 100644 --- a/GVFS/GVFS.Platform.Mac/MacPlatform.cs +++ b/GVFS/GVFS.Platform.Mac/MacPlatform.cs @@ -12,7 +12,7 @@ namespace GVFS.Platform.Mac public partial class MacPlatform : GVFSPlatform { public MacPlatform() - : base(executableExtension: string.Empty) + : base(executableExtension: string.Empty, installerExtension: string.Empty) { } diff --git a/GVFS/GVFS.Platform.Windows/WindowsPlatform.cs b/GVFS/GVFS.Platform.Windows/WindowsPlatform.cs index 0880ce9a4..763f2d61b 100644 --- a/GVFS/GVFS.Platform.Windows/WindowsPlatform.cs +++ b/GVFS/GVFS.Platform.Windows/WindowsPlatform.cs @@ -24,7 +24,7 @@ public partial class WindowsPlatform : GVFSPlatform private const string BuildLabExRegistryValue = "BuildLabEx"; public WindowsPlatform() - : base(executableExtension: ".exe") + : base(executableExtension: ".exe", installerExtension: ".exe") { } diff --git a/GVFS/GVFS.UnitTests.Windows/Windows/Mock/MockProductUpgrader.cs b/GVFS/GVFS.UnitTests.Windows/Windows/Mock/MockProductUpgrader.cs index 9227d2cd8..7312d0d49 100644 --- a/GVFS/GVFS.UnitTests.Windows/Windows/Mock/MockProductUpgrader.cs +++ b/GVFS/GVFS.UnitTests.Windows/Windows/Mock/MockProductUpgrader.cs @@ -69,11 +69,11 @@ public void PretendNewReleaseAvailableAtRemote(string upgradeVersion, RingType r Random random = new Random(); Asset gvfsAsset = new Asset(); - gvfsAsset.Name = "SetupGVFS." + upgradeVersion + ".exe"; + gvfsAsset.Name = "VFSGit." + upgradeVersion + ".exe"; // This is not cross-checked anywhere, random value is good. gvfsAsset.Size = random.Next(int.MaxValue / 10, int.MaxValue / 2); - gvfsAsset.DownloadURL = new Uri(assetDownloadURLPrefix + "/SetupGVFS." + upgradeVersion + ".exe"); + gvfsAsset.DownloadURL = new Uri(assetDownloadURLPrefix + "/VFSGit." + upgradeVersion + ".exe"); release.Assets.Add(gvfsAsset); Asset gitAsset = new Asset(); diff --git a/GVFS/GVFS.UnitTests/Mock/Common/MockPlatform.cs b/GVFS/GVFS.UnitTests/Mock/Common/MockPlatform.cs index f4ba15d35..38381800c 100644 --- a/GVFS/GVFS.UnitTests/Mock/Common/MockPlatform.cs +++ b/GVFS/GVFS.UnitTests/Mock/Common/MockPlatform.cs @@ -14,7 +14,7 @@ namespace GVFS.UnitTests.Mock.Common public class MockPlatform : GVFSPlatform { public MockPlatform() - : base(executableExtension: ".mockexe") + : base(executableExtension: ".mockexe", installerExtension: ".exe") { } diff --git a/GVFS/GVFS.Upgrader/UpgradeOrchestrator.cs b/GVFS/GVFS.Upgrader/UpgradeOrchestrator.cs index 155adf9f3..deb0cec57 100644 --- a/GVFS/GVFS.Upgrader/UpgradeOrchestrator.cs +++ b/GVFS/GVFS.Upgrader/UpgradeOrchestrator.cs @@ -90,7 +90,6 @@ public void Execute() { mountError = Environment.NewLine + "WARNING: " + mountError; this.output.WriteLine(mountError); - this.ExitCode = ReturnCode.Success; } this.DeletedDownloadedAssets(); From 28f1633b51071a4b4d1623ff27994cbcb2f8f88c Mon Sep 17 00:00:00 2001 From: Al Ameen Shah Date: Mon, 1 Oct 2018 12:15:15 -0400 Subject: [PATCH 210/272] - Updated InstallerExtension of Mock & Mac platforms. New extenstions are .mockexe and .dmg - Update UT - Changed GVFS installer name to New Format "VFSGit.Version.Extension" --- GVFS/GVFS.Common/Git/GitVersion.cs | 4 ++-- GVFS/GVFS.Common/ProductUpgrader.cs | 2 +- GVFS/GVFS.Installer/Setup.iss | 2 +- GVFS/GVFS.Platform.Mac/MacPlatform.cs | 2 +- .../Windows/Mock/MockProductUpgrader.cs | 9 ++++----- GVFS/GVFS.UnitTests/Common/GitVersionTests.cs | 11 ++++++----- GVFS/GVFS.UnitTests/Mock/Common/MockPlatform.cs | 2 +- Readme.md | 2 +- 8 files changed, 17 insertions(+), 17 deletions(-) diff --git a/GVFS/GVFS.Common/Git/GitVersion.cs b/GVFS/GVFS.Common/Git/GitVersion.cs index 97a1bdebc..4ffcdd477 100644 --- a/GVFS/GVFS.Common/Git/GitVersion.cs +++ b/GVFS/GVFS.Common/Git/GitVersion.cs @@ -36,7 +36,7 @@ public static bool TryParseGitVersionCommandResult(string input, out GitVersion return TryParseVersion(input, out version); } - public static bool TryParseInstallerName(string input, out GitVersion version) + public static bool TryParseInstallerName(string input, string installerExtension, out GitVersion version) { // Installer name is of the form // Git-2.14.1.gvfs.1.1.gb16030b-64-bit.exe @@ -48,7 +48,7 @@ public static bool TryParseInstallerName(string input, out GitVersion version) return false; } - if (!input.EndsWith("-64-bit.exe", StringComparison.InvariantCultureIgnoreCase)) + if (!input.EndsWith("-64-bit" + installerExtension, StringComparison.InvariantCultureIgnoreCase)) { return false; } diff --git a/GVFS/GVFS.Common/ProductUpgrader.cs b/GVFS/GVFS.Common/ProductUpgrader.cs index d95c30624..05f3127c1 100644 --- a/GVFS/GVFS.Common/ProductUpgrader.cs +++ b/GVFS/GVFS.Common/ProductUpgrader.cs @@ -114,7 +114,7 @@ public bool TryGetGitVersion(out GitVersion gitVersion, out string error) foreach (Asset asset in this.newestRelease.Assets) { if (asset.Name.StartsWith(GitInstallerFileNamePrefix) && - GitVersion.TryParseInstallerName(asset.Name, out gitVersion)) + GitVersion.TryParseInstallerName(asset.Name, GVFSPlatform.Instance.Constants.InstallerExtension, out gitVersion)) { return true; } diff --git a/GVFS/GVFS.Installer/Setup.iss b/GVFS/GVFS.Installer/Setup.iss index e01597ee2..c6648f7ac 100644 --- a/GVFS/GVFS.Installer/Setup.iss +++ b/GVFS/GVFS.Installer/Setup.iss @@ -39,7 +39,7 @@ AppCopyright=Copyright � Microsoft 2018 BackColor=clWhite BackSolid=yes DefaultDirName={pf}\{#MyAppName} -OutputBaseFilename=SetupGVFS.{#GVFSVersion} +OutputBaseFilename=VFSGit.{#GVFSVersion} OutputDir=Setup Compression=lzma2 InternalCompressLevel=ultra64 diff --git a/GVFS/GVFS.Platform.Mac/MacPlatform.cs b/GVFS/GVFS.Platform.Mac/MacPlatform.cs index 06dcc527a..e31d892bb 100644 --- a/GVFS/GVFS.Platform.Mac/MacPlatform.cs +++ b/GVFS/GVFS.Platform.Mac/MacPlatform.cs @@ -12,7 +12,7 @@ namespace GVFS.Platform.Mac public partial class MacPlatform : GVFSPlatform { public MacPlatform() - : base(executableExtension: string.Empty, installerExtension: string.Empty) + : base(executableExtension: string.Empty, installerExtension: ".dmg") { } diff --git a/GVFS/GVFS.UnitTests.Windows/Windows/Mock/MockProductUpgrader.cs b/GVFS/GVFS.UnitTests.Windows/Windows/Mock/MockProductUpgrader.cs index 7312d0d49..f18ae2998 100644 --- a/GVFS/GVFS.UnitTests.Windows/Windows/Mock/MockProductUpgrader.cs +++ b/GVFS/GVFS.UnitTests.Windows/Windows/Mock/MockProductUpgrader.cs @@ -59,7 +59,6 @@ public void ResetFailedAction() public void PretendNewReleaseAvailableAtRemote(string upgradeVersion, RingType remoteRing) { string assetDownloadURLPrefix = "https://github.com/Microsoft/VFSForGit/releases/download/v" + upgradeVersion; - Release release = new Release(); release.Name = "GVFS " + upgradeVersion; @@ -69,17 +68,17 @@ public void PretendNewReleaseAvailableAtRemote(string upgradeVersion, RingType r Random random = new Random(); Asset gvfsAsset = new Asset(); - gvfsAsset.Name = "VFSGit." + upgradeVersion + ".exe"; + gvfsAsset.Name = "VFSGit." + upgradeVersion + GVFSPlatform.Instance.Constants.InstallerExtension; // This is not cross-checked anywhere, random value is good. gvfsAsset.Size = random.Next(int.MaxValue / 10, int.MaxValue / 2); - gvfsAsset.DownloadURL = new Uri(assetDownloadURLPrefix + "/VFSGit." + upgradeVersion + ".exe"); + gvfsAsset.DownloadURL = new Uri(assetDownloadURLPrefix + "/VFSGit." + upgradeVersion + GVFSPlatform.Instance.Constants.InstallerExtension); release.Assets.Add(gvfsAsset); Asset gitAsset = new Asset(); - gitAsset.Name = "Git-2.17.1.gvfs.2.1.4.g4385455-64-bit.exe"; + gitAsset.Name = "Git-2.17.1.gvfs.2.1.4.g4385455-64-bit" + GVFSPlatform.Instance.Constants.InstallerExtension; gitAsset.Size = random.Next(int.MaxValue / 10, int.MaxValue / 2); - gitAsset.DownloadURL = new Uri(assetDownloadURLPrefix + "/Git-2.17.1.gvfs.2.1.4.g4385455-64-bit.exe"); + gitAsset.DownloadURL = new Uri(assetDownloadURLPrefix + "/Git-2.17.1.gvfs.2.1.4.g4385455-64-bit" + GVFSPlatform.Instance.Constants.InstallerExtension); release.Assets.Add(gitAsset); this.expectedGVFSAssetName = gvfsAsset.Name; diff --git a/GVFS/GVFS.UnitTests/Common/GitVersionTests.cs b/GVFS/GVFS.UnitTests/Common/GitVersionTests.cs index e33443ab9..1f196b3e0 100644 --- a/GVFS/GVFS.UnitTests/Common/GitVersionTests.cs +++ b/GVFS/GVFS.UnitTests/Common/GitVersionTests.cs @@ -1,4 +1,5 @@ -using GVFS.Common.Git; +using GVFS.Common; +using GVFS.Common.Git; using GVFS.Tests.Should; using NUnit.Framework; @@ -10,9 +11,9 @@ public class GitVersionTests [TestCase] public void TryParseInstallerName() { - this.ParseAndValidateInstallerVersion("Git-1.2.3.gvfs.4.5.gb16030b-64-bit.exe"); - this.ParseAndValidateInstallerVersion("git-1.2.3.gvfs.4.5.gb16030b-64-bit.exe"); - this.ParseAndValidateInstallerVersion("Git-1.2.3.gvfs.4.5.gb16030b-64-bit.EXE"); + this.ParseAndValidateInstallerVersion("Git-1.2.3.gvfs.4.5.gb16030b-64-bit" + GVFSPlatform.Instance.Constants.InstallerExtension); + this.ParseAndValidateInstallerVersion("git-1.2.3.gvfs.4.5.gb16030b-64-bit" + GVFSPlatform.Instance.Constants.InstallerExtension); + this.ParseAndValidateInstallerVersion("Git-1.2.3.gvfs.4.5.gb16030b-64-bit" + GVFSPlatform.Instance.Constants.InstallerExtension); } [TestCase] @@ -206,7 +207,7 @@ public void Allow_Invalid_Minor_Revision() private void ParseAndValidateInstallerVersion(string installerName) { GitVersion version; - bool success = GitVersion.TryParseInstallerName(installerName, out version); + bool success = GitVersion.TryParseInstallerName(installerName, GVFSPlatform.Instance.Constants.InstallerExtension, out version); success.ShouldBeTrue(); version.Major.ShouldEqual(1); diff --git a/GVFS/GVFS.UnitTests/Mock/Common/MockPlatform.cs b/GVFS/GVFS.UnitTests/Mock/Common/MockPlatform.cs index 38381800c..996cf2ff4 100644 --- a/GVFS/GVFS.UnitTests/Mock/Common/MockPlatform.cs +++ b/GVFS/GVFS.UnitTests/Mock/Common/MockPlatform.cs @@ -14,7 +14,7 @@ namespace GVFS.UnitTests.Mock.Common public class MockPlatform : GVFSPlatform { public MockPlatform() - : base(executableExtension: ".mockexe", installerExtension: ".exe") + : base(executableExtension: ".mockexe", installerExtension: ".mockexe") { } diff --git a/Readme.md b/Readme.md index 133b93754..5af238112 100644 --- a/Readme.md +++ b/Readme.md @@ -50,7 +50,7 @@ If you'd like to build your own VFS for Git Windows installer: build will fail, and the second and subsequent builds will succeed. This is because the build requires a prebuild code generation step. For details, see the build script in the previous step. -The installer can now be found at `C:\Repos\VFSForGit\BuildOutput\GVFS.Installer\bin\x64\[Debug|Release]\SetupGVFS..exe` +The installer can now be found at `C:\Repos\VFSForGit\BuildOutput\GVFS.Installer\bin\x64\[Debug|Release]\VFSGit..exe` ## Trying out VFS for Git From 49802305692458783776e2f491afd5e5f8e71bf3 Mon Sep 17 00:00:00 2001 From: Al Ameen Shah Date: Mon, 1 Oct 2018 12:43:09 -0400 Subject: [PATCH 211/272] - Fixing FunctionalTest failure. --- GVFS/GVFS.Build/GVFS.PreBuild.csproj | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/GVFS/GVFS.Build/GVFS.PreBuild.csproj b/GVFS/GVFS.Build/GVFS.PreBuild.csproj index 00ff6598c..1a64a9536 100644 --- a/GVFS/GVFS.Build/GVFS.PreBuild.csproj +++ b/GVFS/GVFS.Build/GVFS.PreBuild.csproj @@ -140,7 +140,7 @@ - $(BuildOutputDir)\GVFS.Installer\bin\x64\$(Configuration)\SetupGVFS.$(GVFSVersion).exe + $(BuildOutputDir)\GVFS.Installer\bin\x64\$(Configuration)\VFSGit.$(GVFSVersion).exe $(BuildOutputDir)\GVFS.Build\ $(OutDir)GVFSConstants.GitVersion.cs $(OutDir)InstallG4W.bat From f215afc94eb00659d80fd8ac90bb5aea0b1a0b68 Mon Sep 17 00:00:00 2001 From: Al Ameen Shah Date: Mon, 1 Oct 2018 14:58:28 -0400 Subject: [PATCH 212/272] Fixing a typo --- GVFS/GVFS.Common/GVFSConstants.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/GVFS/GVFS.Common/GVFSConstants.cs b/GVFS/GVFS.Common/GVFSConstants.cs index 813663a0c..9f5496159 100644 --- a/GVFS/GVFS.Common/GVFSConstants.cs +++ b/GVFS/GVFS.Common/GVFSConstants.cs @@ -231,7 +231,7 @@ public static class UpgradeVerbMessages public const string GVFSUpgradeOptionalConfirm = "`gvfs upgrade [--confirm]`"; public const string NoneRingConsoleAlert = "Upgrade ring set to \"None\". No upgrade check was performed."; public const string InvalidRingConsoleAlert = "Upgrade ring is not set. No upgrade check was performed."; - public const string SetUpgradeRingCommand = "To set or change upgrade ring, run `gvfs config " + LocalGVFSConfig.UpgradeRing + " [\"Fast\"|\"Slow\"|\"None\"] from a command prompt."; + public const string SetUpgradeRingCommand = "To set or change upgrade ring, run `gvfs config " + LocalGVFSConfig.UpgradeRing + " [\"Fast\"|\"Slow\"|\"None\"]` from a command prompt."; public const string ReminderNotification = "A new version of GVFS is available. Run " + UpgradeVerbMessages.GVFSUpgrade + " to start the upgrade."; public const string UnmountRepoWarning = "The upgrade process will unmount all GVFS repositories for several minutes and remount them when it is complete. Ensure you are at a stopping point prior to upgrading."; public const string UpgradeInstallAdvice = "When you are ready, run " + UpgradeVerbMessages.GVFSUpgradeConfirm + " from an elevated command prompt."; From 0322ab5bc507ee39e739349c599573d7facf117e Mon Sep 17 00:00:00 2001 From: Al Ameen Shah Date: Wed, 3 Oct 2018 11:04:40 -0400 Subject: [PATCH 213/272] - Revert back to old installer name "SetupGVFS..exe" - Upgrade supports both old and new gvfs installer name formats now. - Early exit upgrade when either Git or GVFS asset installers are missing in the Release. - New success message when repository mount fails. - updated messaging when gvfs mount fails - Don't re-mount when GVFS installation fails. - Shorter pre-upgrade warning text. - rephrased ProjFS error message - Rephrased Cannot install upgrade messaging. - Added advice on what to do next when an upgrade is available but not installable. - Include Ring information in upgrade-available message. Updated UT. - cleanup made LocalGVFSConfig.allSettings a private property --- GVFS/GVFS.Build/GVFS.PreBuild.csproj | 2 +- GVFS/GVFS.Common/GVFSConstants.cs | 4 +- GVFS/GVFS.Common/InstallerPreRunChecker.cs | 15 ++-- GVFS/GVFS.Common/LocalGVFSConfig.cs | 11 ++- GVFS/GVFS.Common/ProductUpgrader.Shared.cs | 8 +- GVFS/GVFS.Common/ProductUpgrader.cs | 73 +++++++++++++++---- .../GVFSUpgradeReminderTests.cs | 2 +- GVFS/GVFS.Installer/Setup.iss | 2 +- .../Windows/Mock/MockProductUpgrader.cs | 4 +- .../Windows/Upgrader/UpgradeVerbTests.cs | 10 +-- GVFS/GVFS.Upgrader/UpgradeOrchestrator.cs | 6 +- GVFS/GVFS/CommandLine/UpgradeVerb.cs | 21 ++++-- Readme.md | 2 +- 13 files changed, 107 insertions(+), 53 deletions(-) diff --git a/GVFS/GVFS.Build/GVFS.PreBuild.csproj b/GVFS/GVFS.Build/GVFS.PreBuild.csproj index 1a64a9536..00ff6598c 100644 --- a/GVFS/GVFS.Build/GVFS.PreBuild.csproj +++ b/GVFS/GVFS.Build/GVFS.PreBuild.csproj @@ -140,7 +140,7 @@ - $(BuildOutputDir)\GVFS.Installer\bin\x64\$(Configuration)\VFSGit.$(GVFSVersion).exe + $(BuildOutputDir)\GVFS.Installer\bin\x64\$(Configuration)\SetupGVFS.$(GVFSVersion).exe $(BuildOutputDir)\GVFS.Build\ $(OutDir)GVFSConstants.GitVersion.cs $(OutDir)InstallG4W.bat diff --git a/GVFS/GVFS.Common/GVFSConstants.cs b/GVFS/GVFS.Common/GVFSConstants.cs index 9f5496159..393004b5c 100644 --- a/GVFS/GVFS.Common/GVFSConstants.cs +++ b/GVFS/GVFS.Common/GVFSConstants.cs @@ -233,8 +233,8 @@ public static class UpgradeVerbMessages public const string InvalidRingConsoleAlert = "Upgrade ring is not set. No upgrade check was performed."; public const string SetUpgradeRingCommand = "To set or change upgrade ring, run `gvfs config " + LocalGVFSConfig.UpgradeRing + " [\"Fast\"|\"Slow\"|\"None\"]` from a command prompt."; public const string ReminderNotification = "A new version of GVFS is available. Run " + UpgradeVerbMessages.GVFSUpgrade + " to start the upgrade."; - public const string UnmountRepoWarning = "The upgrade process will unmount all GVFS repositories for several minutes and remount them when it is complete. Ensure you are at a stopping point prior to upgrading."; - public const string UpgradeInstallAdvice = "When you are ready, run " + UpgradeVerbMessages.GVFSUpgradeConfirm + " from an elevated command prompt."; + public const string UnmountRepoWarning = "Upgrade will unmount and remount gvfs repos, ensure you are at a stopping point."; + public const string UpgradeInstallAdvice = "When ready, run " + UpgradeVerbMessages.GVFSUpgradeConfirm + " from an elevated command prompt."; } } } diff --git a/GVFS/GVFS.Common/InstallerPreRunChecker.cs b/GVFS/GVFS.Common/InstallerPreRunChecker.cs index ba46681a6..1970d0864 100644 --- a/GVFS/GVFS.Common/InstallerPreRunChecker.cs +++ b/GVFS/GVFS.Common/InstallerPreRunChecker.cs @@ -161,7 +161,7 @@ protected virtual bool TryRunGVFSWithArgs(string args, out string consoleError) } else { - consoleError = string.IsNullOrEmpty(processResult.Errors) ? $"GVFS {args} failed." : processResult.Errors; + consoleError = string.IsNullOrEmpty(processResult.Errors) ? $"`gvfs {args}` failed." : processResult.Errors; return false; } } @@ -174,14 +174,15 @@ protected virtual bool TryRunGVFSWithArgs(string args, out string consoleError) private bool IsGVFSUpgradeAllowed(out string consoleError) { - consoleError = null; - + bool isConfirmed = string.Equals(this.CommandToRerun, GVFSConstants.UpgradeVerbMessages.GVFSUpgradeConfirm, StringComparison.OrdinalIgnoreCase); + string adviceText = null; if (!this.IsElevated()) { + adviceText = isConfirmed ? $"Run {this.CommandToRerun} again from an elevated command prompt." : $"To install, run {GVFSConstants.UpgradeVerbMessages.GVFSUpgradeConfirm} from an elevated command prompt."; consoleError = string.Join( Environment.NewLine, "The installer needs to be run from an elevated command prompt.", - $"Run {this.CommandToRerun} again from an elevated command prompt."); + adviceText); return false; } @@ -189,20 +190,22 @@ private bool IsGVFSUpgradeAllowed(out string consoleError) { consoleError = string.Join( Environment.NewLine, - $"ProjFS configuration does not support {GVFSConstants.UpgradeVerbMessages.GVFSUpgrade}.", + $"{GVFSConstants.UpgradeVerbMessages.GVFSUpgrade} is not supported because you have previously installed an out of band ProjFS driver.", "Check your team's documentation for how to upgrade."); return false; } if (this.IsServiceInstalledAndNotRunning()) { + adviceText = isConfirmed ? $"Run `sc start GVFS.Service` and run {this.CommandToRerun} again from an elevated command prompt." : $"To install, run `sc start GVFS.Service` and run {GVFSConstants.UpgradeVerbMessages.GVFSUpgradeConfirm} from an elevated command prompt."; consoleError = string.Join( Environment.NewLine, "GVFS Service is not running.", - $"Run `sc start GVFS.Service` and run {this.CommandToRerun} again from an elevated command prompt."); + adviceText); return false; } + consoleError = null; return true; } } diff --git a/GVFS/GVFS.Common/LocalGVFSConfig.cs b/GVFS/GVFS.Common/LocalGVFSConfig.cs index a7c225873..722b28081 100644 --- a/GVFS/GVFS.Common/LocalGVFSConfig.cs +++ b/GVFS/GVFS.Common/LocalGVFSConfig.cs @@ -9,6 +9,7 @@ public class LocalGVFSConfig private const string FileName = "gvfs.config"; private readonly string configFile; private readonly PhysicalFileSystem fileSystem; + private FileBasedDictionary allSettings; public LocalGVFSConfig() { @@ -19,8 +20,6 @@ public LocalGVFSConfig() this.fileSystem = new PhysicalFileSystem(); } - private FileBasedDictionary AllSettings { get; set; } - public bool TryGetConfig( string key, out string value, @@ -36,7 +35,7 @@ public bool TryGetConfig( try { - this.AllSettings.TryGetValue(key, out value); + this.allSettings.TryGetValue(key, out value); error = null; return true; } @@ -68,7 +67,7 @@ public bool TrySetConfig( try { - this.AllSettings.SetValueAndFlush(key, value); + this.allSettings.SetValueAndFlush(key, value); error = null; return true; } @@ -88,7 +87,7 @@ public bool TrySetConfig( private bool TryLoadSettings(ITracer tracer, out string error) { - if (this.AllSettings == null) + if (this.allSettings == null) { FileBasedDictionary config = null; if (FileBasedDictionary.TryCreate( @@ -98,7 +97,7 @@ private bool TryLoadSettings(ITracer tracer, out string error) output: out config, error: out error)) { - this.AllSettings = config; + this.allSettings = config; return true; } diff --git a/GVFS/GVFS.Common/ProductUpgrader.Shared.cs b/GVFS/GVFS.Common/ProductUpgrader.Shared.cs index edd4db874..178f58038 100644 --- a/GVFS/GVFS.Common/ProductUpgrader.Shared.cs +++ b/GVFS/GVFS.Common/ProductUpgrader.Shared.cs @@ -11,16 +11,18 @@ public partial class ProductUpgrader public const string DownloadDirectory = "Downloads"; private const string RootDirectory = UpgradeDirectoryName; - private const string GVFSInstallerFileNamePrefix = "VFSGit"; + private const string GVFSInstallerFileNamePrefix = "SetupGVFS"; + private const string VFSForGitInstallerFileNamePrefix = "VFSForGit"; public static bool IsLocalUpgradeAvailable() { string downloadDirectory = GetAssetDownloadsPath(); if (Directory.Exists(downloadDirectory)) { + const string PotentialInstallerName = "*VFS*.*"; string[] installers = Directory.GetFiles( - downloadDirectory, - $"{GVFSInstallerFileNamePrefix}*.*", + downloadDirectory, + PotentialInstallerName, SearchOption.TopDirectoryOnly); return installers.Length > 0; } diff --git a/GVFS/GVFS.Common/ProductUpgrader.cs b/GVFS/GVFS.Common/ProductUpgrader.cs index 05f3127c1..33a584fe8 100644 --- a/GVFS/GVFS.Common/ProductUpgrader.cs +++ b/GVFS/GVFS.Common/ProductUpgrader.cs @@ -20,8 +20,8 @@ public partial class ProductUpgrader private const string CommonInstallerArgs = "/VERYSILENT /CLOSEAPPLICATIONS /SUPPRESSMSGBOXES /NORESTART"; private const string GVFSInstallerArgs = CommonInstallerArgs + " /MOUNTREPOS=false"; private const string GitInstallerArgs = CommonInstallerArgs + " /ALLOWDOWNGRADE=1"; - private const string GitAssetNamePrefix = "Git"; - private const string GVFSAssetNamePrefix = "GVFS"; + private const string GitAssetId = "Git"; + private const string GVFSAssetId = "GVFS"; private const string GitInstallerFileNamePrefix = "Git-"; private const int RepoMountFailureExitCode = 17; private const string ToolsDirectory = "Tools"; @@ -127,17 +127,37 @@ public bool TryGetGitVersion(out GitVersion gitVersion, out string error) public bool TryDownloadNewestVersion(out string errorMessage) { - errorMessage = null; - + bool downloadedGit = false; + bool downloadedGVFS = false; foreach (Asset asset in this.newestRelease.Assets) { - if (string.Equals(Path.GetExtension(asset.Name), GVFSPlatform.Instance.Constants.InstallerExtension, StringComparison.OrdinalIgnoreCase) && - !this.TryDownloadAsset(asset, out errorMessage)) + bool targetOSMatch = string.Equals(Path.GetExtension(asset.Name), GVFSPlatform.Instance.Constants.InstallerExtension, StringComparison.OrdinalIgnoreCase); + bool isGitAsset = this.IsGitAsset(asset); + bool isGVFSAsset = isGitAsset ? false : this.IsGVFSAsset(asset); + if (!targetOSMatch || (!isGVFSAsset && !isGitAsset)) + { + continue; + } + + if (!this.TryDownloadAsset(asset, out errorMessage)) { + errorMessage = $"Could not download {(isGVFSAsset ? GVFSAssetId : GitAssetId)} installer. {errorMessage}"; return false; } + else + { + downloadedGit = isGitAsset ? true : downloadedGit; + downloadedGVFS = isGVFSAsset ? true : downloadedGVFS; + } } + if (!downloadedGit || !downloadedGVFS) + { + errorMessage = $"Could not find {(!downloadedGit ? GitAssetId : GVFSAssetId)} installer in the latest release."; + return false; + } + + errorMessage = null; return true; } @@ -147,7 +167,7 @@ public bool TryRunGitInstaller(out bool installationSucceeded, out string error) installationSucceeded = false; int exitCode = 0; - bool launched = this.TryRunInstallerForAsset(GitAssetNamePrefix, out exitCode, out error); + bool launched = this.TryRunInstallerForAsset(GitAssetId, out exitCode, out error); installationSucceeded = exitCode == 0; return launched; @@ -159,7 +179,7 @@ public bool TryRunGVFSInstaller(out bool installationSucceeded, out string error installationSucceeded = false; int exitCode = 0; - bool launched = this.TryRunInstallerForAsset(GVFSAssetNamePrefix, out exitCode, out error); + bool launched = this.TryRunInstallerForAsset(GVFSAssetId, out exitCode, out error); installationSucceeded = exitCode == 0 || exitCode == RepoMountFailureExitCode; return launched; @@ -370,7 +390,7 @@ private static bool TryCreateDirectory(string path, out Exception exception) return true; } - private bool TryRunInstallerForAsset(string name, out int installerExitCode, out string error) + private bool TryRunInstallerForAsset(string assetId, out int installerExitCode, out string error) { error = null; installerExitCode = 0; @@ -378,7 +398,7 @@ private bool TryRunInstallerForAsset(string name, out int installerExitCode, out bool installerIsRun = false; string path; string installerArgs; - if (this.TryGetLocalInstallerPath(name, out path, out installerArgs)) + if (this.TryGetLocalInstallerPath(assetId, out path, out installerArgs)) { string logFilePath = GVFSEnlistment.GetNewLogFileName(GetLogDirectoryPath(), Path.GetFileNameWithoutExtension(path)); string args = installerArgs + " /Log=" + logFilePath; @@ -386,14 +406,14 @@ private bool TryRunInstallerForAsset(string name, out int installerExitCode, out if (installerExitCode != 0 && string.IsNullOrEmpty(error)) { - error = name + " installer failed. Error log: " + logFilePath; + error = assetId + " installer failed. Error log: " + logFilePath; } installerIsRun = true; } else { - error = "Could not find downloaded installer for " + name; + error = "Could not find downloaded installer for " + assetId; } return installerIsRun; @@ -407,20 +427,20 @@ private void TraceException(Exception exception, string method, string message) this.tracer.RelatedError(metadata, message, Keywords.Telemetry); } - private bool TryGetLocalInstallerPath(string name, out string path, out string args) + private bool TryGetLocalInstallerPath(string assetId, out string path, out string args) { foreach (Asset asset in this.newestRelease.Assets) { if (string.Equals(Path.GetExtension(asset.Name), GVFSPlatform.Instance.Constants.InstallerExtension, StringComparison.OrdinalIgnoreCase)) { path = asset.LocalPath; - if (name == GitAssetNamePrefix && asset.Name.StartsWith(GitInstallerFileNamePrefix, StringComparison.OrdinalIgnoreCase)) + if (assetId == GitAssetId && this.IsGitAsset(asset)) { args = GitInstallerArgs; return true; } - if (name == GVFSAssetNamePrefix && asset.Name.StartsWith(GVFSInstallerFileNamePrefix, StringComparison.OrdinalIgnoreCase)) + if (assetId == GVFSAssetId && this.IsGVFSAsset(asset)) { args = GVFSInstallerArgs; return true; @@ -433,6 +453,29 @@ private bool TryGetLocalInstallerPath(string name, out string path, out string a return false; } + private bool IsGVFSAsset(Asset asset) + { + return this.AssetInstallerNameCompare(asset, GVFSInstallerFileNamePrefix, VFSForGitInstallerFileNamePrefix); + } + + private bool IsGitAsset(Asset asset) + { + return this.AssetInstallerNameCompare(asset, GitInstallerFileNamePrefix); + } + + private bool AssetInstallerNameCompare(Asset asset, params string[] expectedFileNamePrefixes) + { + foreach (string fileNamePrefix in expectedFileNamePrefixes) + { + if (asset.Name.StartsWith(fileNamePrefix, StringComparison.OrdinalIgnoreCase)) + { + return true; + } + } + + return false; + } + [DataContract(Name = "asset")] protected class Asset { diff --git a/GVFS/GVFS.FunctionalTests/Tests/EnlistmentPerFixture/GVFSUpgradeReminderTests.cs b/GVFS/GVFS.FunctionalTests/Tests/EnlistmentPerFixture/GVFSUpgradeReminderTests.cs index 54e4c8eff..905f9e939 100644 --- a/GVFS/GVFS.FunctionalTests/Tests/EnlistmentPerFixture/GVFSUpgradeReminderTests.cs +++ b/GVFS/GVFS.FunctionalTests/Tests/EnlistmentPerFixture/GVFSUpgradeReminderTests.cs @@ -14,7 +14,7 @@ namespace GVFS.FunctionalTests.Tests.EnlistmentPerFixture [Category(Categories.WindowsOnly)] public class UpgradeReminderTests : TestsWithEnlistmentPerFixture { - private const string GVFSInstallerName = "VFSGit.1.0.18234.1.exe"; + private const string GVFSInstallerName = "VFSForGit.1.0.18234.1.exe"; private const string GitInstallerName = "Git-2.17.1.gvfs.2.5.g2962052-64-bit.exe"; private string upgradeDirectory; diff --git a/GVFS/GVFS.Installer/Setup.iss b/GVFS/GVFS.Installer/Setup.iss index c6648f7ac..e01597ee2 100644 --- a/GVFS/GVFS.Installer/Setup.iss +++ b/GVFS/GVFS.Installer/Setup.iss @@ -39,7 +39,7 @@ AppCopyright=Copyright � Microsoft 2018 BackColor=clWhite BackSolid=yes DefaultDirName={pf}\{#MyAppName} -OutputBaseFilename=VFSGit.{#GVFSVersion} +OutputBaseFilename=SetupGVFS.{#GVFSVersion} OutputDir=Setup Compression=lzma2 InternalCompressLevel=ultra64 diff --git a/GVFS/GVFS.UnitTests.Windows/Windows/Mock/MockProductUpgrader.cs b/GVFS/GVFS.UnitTests.Windows/Windows/Mock/MockProductUpgrader.cs index f18ae2998..68281dfa3 100644 --- a/GVFS/GVFS.UnitTests.Windows/Windows/Mock/MockProductUpgrader.cs +++ b/GVFS/GVFS.UnitTests.Windows/Windows/Mock/MockProductUpgrader.cs @@ -68,11 +68,11 @@ public void PretendNewReleaseAvailableAtRemote(string upgradeVersion, RingType r Random random = new Random(); Asset gvfsAsset = new Asset(); - gvfsAsset.Name = "VFSGit." + upgradeVersion + GVFSPlatform.Instance.Constants.InstallerExtension; + gvfsAsset.Name = "VFSForGit." + upgradeVersion + GVFSPlatform.Instance.Constants.InstallerExtension; // This is not cross-checked anywhere, random value is good. gvfsAsset.Size = random.Next(int.MaxValue / 10, int.MaxValue / 2); - gvfsAsset.DownloadURL = new Uri(assetDownloadURLPrefix + "/VFSGit." + upgradeVersion + GVFSPlatform.Instance.Constants.InstallerExtension); + gvfsAsset.DownloadURL = new Uri(assetDownloadURLPrefix + "/VFSForGit." + upgradeVersion + GVFSPlatform.Instance.Constants.InstallerExtension); release.Assets.Add(gvfsAsset); Asset gitAsset = new Asset(); diff --git a/GVFS/GVFS.UnitTests.Windows/Windows/Upgrader/UpgradeVerbTests.cs b/GVFS/GVFS.UnitTests.Windows/Windows/Upgrader/UpgradeVerbTests.cs index cc4b43065..f5e560465 100644 --- a/GVFS/GVFS.UnitTests.Windows/Windows/Upgrader/UpgradeVerbTests.cs +++ b/GVFS/GVFS.UnitTests.Windows/Windows/Upgrader/UpgradeVerbTests.cs @@ -43,8 +43,8 @@ public void UpgradeAvailabilityReporting() expectedReturn: ReturnCode.Success, expectedOutput: new List { - "New GVFS version available: " + NewerThanLocalVersion, - "When you are ready, run `gvfs upgrade --confirm` from an elevated command prompt." + "New GVFS version " + NewerThanLocalVersion + " available in ring Slow", + "When ready, run `gvfs upgrade --confirm` from an elevated command prompt." }, expectedErrors: null); } @@ -80,7 +80,7 @@ public void LaunchInstaller() expectedReturn: ReturnCode.Success, expectedOutput: new List { - "New GVFS version available: " + NewerThanLocalVersion, + "New GVFS version " + NewerThanLocalVersion + " available in ring Slow", "Launching upgrade tool...Succeeded" }, expectedErrors:null); @@ -136,12 +136,12 @@ public void ProjFSPreCheck() expectedReturn: ReturnCode.GenericError, expectedOutput: new List { - "ERROR: ProjFS configuration does not support `gvfs upgrade`.", + "ERROR: `gvfs upgrade` is not supported because you have previously installed an out of band ProjFS driver.", "Check your team's documentation for how to upgrade." }, expectedErrors: new List { - "ProjFS configuration does not support `gvfs upgrade`." + "`gvfs upgrade` is not supported because you have previously installed an out of band ProjFS driver." }); } diff --git a/GVFS/GVFS.Upgrader/UpgradeOrchestrator.cs b/GVFS/GVFS.Upgrader/UpgradeOrchestrator.cs index deb0cec57..df18ceb2d 100644 --- a/GVFS/GVFS.Upgrader/UpgradeOrchestrator.cs +++ b/GVFS/GVFS.Upgrader/UpgradeOrchestrator.cs @@ -60,6 +60,7 @@ public void Execute() string error = null; ProductUpgrader.RingType ring = ProductUpgrader.RingType.Invalid; + string mountError = null; if (!this.TryLoadUpgradeRing(out ring, out error) || ring == ProductUpgrader.RingType.None) { @@ -85,7 +86,6 @@ public void Execute() } finally { - string mountError = null; if (!this.TryMountRepositories(out mountError)) { mountError = Environment.NewLine + "WARNING: " + mountError; @@ -103,7 +103,7 @@ public void Execute() } else { - this.output.WriteLine(Environment.NewLine + "Upgrade completed successfully!"); + this.output.WriteLine($"{Environment.NewLine}{(string.IsNullOrEmpty(mountError) ? "U" : "Repository mount failed. But u")}pgrade completed successfully!"); } if (this.input == Console.In) @@ -229,6 +229,8 @@ private bool TryRunUpgrade(out Version newVersion, out string consoleError) }, $"Installing GVFS version: {newGVFSVersion}")) { + this.mount = false; + consoleError = errorMessage; return false; } diff --git a/GVFS/GVFS/CommandLine/UpgradeVerb.cs b/GVFS/GVFS/CommandLine/UpgradeVerb.cs index 282dacea2..edd1af8c3 100644 --- a/GVFS/GVFS/CommandLine/UpgradeVerb.cs +++ b/GVFS/GVFS/CommandLine/UpgradeVerb.cs @@ -91,15 +91,16 @@ private bool TryRunProductUpgrade() { string errorOutputFormat = Environment.NewLine + "ERROR: {0}"; string error = null; + string cannotInstallReason = null; Version newestVersion = null; ProductUpgrader.RingType ring = ProductUpgrader.RingType.Invalid; - bool isInstallable = this.TryCheckUpgradeInstallable(out error); + bool isInstallable = this.TryCheckUpgradeInstallable(out cannotInstallReason); if (this.Confirmed && !isInstallable) { - this.ReportInfoToConsole($"Cannot install upgrade on this machine."); - this.Output.WriteLine(errorOutputFormat, error); - this.tracer.RelatedError($"{nameof(this.TryRunProductUpgrade)}: Upgrade is not installable. {error}"); + this.ReportInfoToConsole($"Cannot upgrade GVFS on this machine."); + this.Output.WriteLine(errorOutputFormat, cannotInstallReason); + this.tracer.RelatedError($"{nameof(this.TryRunProductUpgrade)}: Upgrade is not installable. {cannotInstallReason}"); return false; } @@ -132,7 +133,7 @@ private bool TryRunProductUpgrade() return true; } - string upgradeAvailableMessage = $"New GVFS version available: {newestVersion.ToString()}"; + string upgradeAvailableMessage = $"New GVFS version {newestVersion.ToString()} available in ring {ring}"; if (this.Confirmed) { this.ReportInfoToConsole(upgradeAvailableMessage); @@ -156,15 +157,19 @@ private bool TryRunProductUpgrade() if (isInstallable) { string message = string.Join( - Environment.NewLine + Environment.NewLine, - upgradeAvailableMessage, + Environment.NewLine, GVFSConstants.UpgradeVerbMessages.UnmountRepoWarning, GVFSConstants.UpgradeVerbMessages.UpgradeInstallAdvice); - this.ReportInfoToConsole(message); + this.ReportInfoToConsole(upgradeAvailableMessage + Environment.NewLine + Environment.NewLine + message + Environment.NewLine); } else { this.ReportInfoToConsole($"{Environment.NewLine}{upgradeAvailableMessage}"); + + if (!string.IsNullOrEmpty(cannotInstallReason)) + { + this.ReportInfoToConsole($"{Environment.NewLine}{cannotInstallReason}"); + } } } diff --git a/Readme.md b/Readme.md index 5af238112..133b93754 100644 --- a/Readme.md +++ b/Readme.md @@ -50,7 +50,7 @@ If you'd like to build your own VFS for Git Windows installer: build will fail, and the second and subsequent builds will succeed. This is because the build requires a prebuild code generation step. For details, see the build script in the previous step. -The installer can now be found at `C:\Repos\VFSForGit\BuildOutput\GVFS.Installer\bin\x64\[Debug|Release]\VFSGit..exe` +The installer can now be found at `C:\Repos\VFSForGit\BuildOutput\GVFS.Installer\bin\x64\[Debug|Release]\SetupGVFS..exe` ## Trying out VFS for Git From 46f9bc5b51cf31af648d2026fddf70e26f33ed4b Mon Sep 17 00:00:00 2001 From: Jessica Schumaker Date: Thu, 4 Oct 2018 11:30:15 -0400 Subject: [PATCH 214/272] Repair Functional Tests: test for confirm on and off Enrich existing tests to include confirm off --- .../EnlistmentPerTestCase/RepairTests.cs | 49 +++++++++++-------- .../MultiEnlistmentTests/SharedCacheTests.cs | 4 +- .../Tools/GVFSFunctionalTestEnlistment.cs | 4 +- .../GVFS.FunctionalTests/Tools/GVFSProcess.cs | 5 +- 4 files changed, 36 insertions(+), 26 deletions(-) diff --git a/GVFS/GVFS.FunctionalTests/Tests/EnlistmentPerTestCase/RepairTests.cs b/GVFS/GVFS.FunctionalTests/Tests/EnlistmentPerTestCase/RepairTests.cs index b53d2caf8..7f44dcf80 100644 --- a/GVFS/GVFS.FunctionalTests/Tests/EnlistmentPerTestCase/RepairTests.cs +++ b/GVFS/GVFS.FunctionalTests/Tests/EnlistmentPerTestCase/RepairTests.cs @@ -17,8 +17,8 @@ public class RepairTests : TestsWithEnlistmentPerTestCase public void NoFixesNeeded() { this.Enlistment.UnmountGVFS(); - - this.Enlistment.Repair(); + this.Enlistment.Repair(confirm: false); + this.Enlistment.Repair(confirm: true); } [TestCase] @@ -28,12 +28,11 @@ public void FixesCorruptHeadSha() string headFilePath = Path.Combine(this.Enlistment.RepoRoot, ".git", "HEAD"); File.WriteAllText(headFilePath, "0000"); - this.Enlistment.TryMountGVFS().ShouldEqual(false, "GVFS shouldn't mount when HEAD is corrupt"); - this.Enlistment.Repair(); + this.RepairWithoutConfirmShouldNotFix(); - this.Enlistment.MountGVFS(); + this.RepairWithConfirmShouldFix(); } [TestCase] @@ -43,12 +42,11 @@ public void FixesCorruptHeadSymRef() string headFilePath = Path.Combine(this.Enlistment.RepoRoot, ".git", "HEAD"); File.WriteAllText(headFilePath, "ref: refs"); - this.Enlistment.TryMountGVFS().ShouldEqual(false, "GVFS shouldn't mount when HEAD is corrupt"); - this.Enlistment.Repair(); + this.RepairWithoutConfirmShouldNotFix(); - this.Enlistment.MountGVFS(); + this.RepairWithConfirmShouldFix(); } [TestCase] @@ -58,12 +56,11 @@ public void FixesMissingGitIndex() string gitIndexPath = Path.Combine(this.Enlistment.RepoRoot, ".git", "index"); File.Delete(gitIndexPath); - this.Enlistment.TryMountGVFS().ShouldEqual(false, "GVFS shouldn't mount when git index is missing"); - this.Enlistment.Repair(); + this.RepairWithoutConfirmShouldNotFix(); - this.Enlistment.MountGVFS(); + this.RepairWithConfirmShouldFix(); } [TestCase] @@ -84,9 +81,9 @@ public void FixesGitIndexCorruptedWithBadData() this.Enlistment.TryMountGVFS(out output).ShouldEqual(false, "GVFS shouldn't mount when index is corrupt"); output.ShouldContain("Index validation failed"); - this.Enlistment.Repair(); + this.RepairWithoutConfirmShouldNotFix(); - this.Enlistment.MountGVFS(); + this.RepairWithConfirmShouldFix(); } [TestCase] @@ -108,9 +105,9 @@ public void FixesGitIndexContainingAllNulls() this.Enlistment.TryMountGVFS(out output).ShouldEqual(false, "GVFS shouldn't mount when index is corrupt"); output.ShouldContain("Index validation failed"); - this.Enlistment.Repair(); + this.RepairWithoutConfirmShouldNotFix(); - this.Enlistment.MountGVFS(); + this.RepairWithConfirmShouldFix(); } [TestCase] @@ -135,9 +132,9 @@ public void FixesGitIndexCorruptedByTruncation() this.Enlistment.TryMountGVFS(out output).ShouldEqual(false, "GVFS shouldn't mount when index is corrupt"); output.ShouldContain("Index validation failed"); - this.Enlistment.Repair(); + this.RepairWithoutConfirmShouldNotFix(); - this.Enlistment.MountGVFS(); + this.RepairWithConfirmShouldFix(); } [TestCase] @@ -150,11 +147,11 @@ public void FixesCorruptGitConfig() this.Enlistment.TryMountGVFS().ShouldEqual(false, "GVFS shouldn't mount when git config is missing"); - this.Enlistment.Repair(); + this.RepairWithoutConfirmShouldNotFix(); + this.Enlistment.Repair(confirm: true); ProcessResult result = GitProcess.InvokeProcess(this.Enlistment.RepoRoot, "remote add origin " + this.Enlistment.RepoUrl); - result.ExitCode.ShouldEqual(0, result.Errors); - + result.ExitCode.ShouldEqual(0, result.Errors); this.Enlistment.MountGVFS(); } @@ -170,5 +167,17 @@ private void CreateCorruptIndexAndRename(string indexPath, Action Date: Thu, 4 Oct 2018 09:54:27 -0700 Subject: [PATCH 215/272] PR Feedback: Use fsid&inode instead of path as key in mutex map --- .../PrjFSKext.xcodeproj/project.pbxproj | 2 + .../PrjFSKext/PrjFSKext/KauthHandler.cpp | 37 +++++++- .../PrjFSKext/PrjFSKext/Message_Kernel.cpp | 2 + .../PrjFSKext/VirtualizationRoots.cpp | 23 +++-- .../PrjFSKext/VirtualizationRoots.hpp | 5 +- .../PrjFSKext/PrjFSKext/VnodeUtilities.cpp | 2 +- .../PrjFSKext/PrjFSKext/VnodeUtilities.hpp | 9 +- ProjFS.Mac/PrjFSKext/public/FsidInode.h | 20 ++++ ProjFS.Mac/PrjFSKext/public/Message.h | 5 + ProjFS.Mac/PrjFSLib/PrjFSLib.cpp | 94 ++++++++++--------- 10 files changed, 130 insertions(+), 69 deletions(-) create mode 100644 ProjFS.Mac/PrjFSKext/public/FsidInode.h diff --git a/ProjFS.Mac/PrjFSKext/PrjFSKext.xcodeproj/project.pbxproj b/ProjFS.Mac/PrjFSKext/PrjFSKext.xcodeproj/project.pbxproj index f55c07403..74700506e 100644 --- a/ProjFS.Mac/PrjFSKext/PrjFSKext.xcodeproj/project.pbxproj +++ b/ProjFS.Mac/PrjFSKext/PrjFSKext.xcodeproj/project.pbxproj @@ -30,6 +30,7 @@ /* End PBXBuildFile section */ /* Begin PBXFileReference section */ + 263E066C21667C11005F756A /* FsidInode.h */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.c.h; path = FsidInode.h; sourceTree = ""; }; 4A08829C20D80B8300E17FEE /* PrjFSXattrs.h */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.c.h; path = PrjFSXattrs.h; sourceTree = ""; }; 4A63CB0B20AB009000157B95 /* VnodeUtilities.cpp */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.cpp.cpp; path = VnodeUtilities.cpp; sourceTree = ""; }; 4A63CB0C20AB009000157B95 /* VnodeUtilities.hpp */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.cpp.h; path = VnodeUtilities.hpp; sourceTree = ""; }; @@ -90,6 +91,7 @@ 4ABB734C20C1A65B00DC0D17 /* PrjFSProviderClientShared.h */, 4A08829C20D80B8300E17FEE /* PrjFSXattrs.h */, C6BDD37A208C2FD700CB7E58 /* Message.h */, + 263E066C21667C11005F756A /* FsidInode.h */, ); path = public; sourceTree = ""; diff --git a/ProjFS.Mac/PrjFSKext/PrjFSKext/KauthHandler.cpp b/ProjFS.Mac/PrjFSKext/PrjFSKext/KauthHandler.cpp index dbf425d53..a0ed82e25 100644 --- a/ProjFS.Mac/PrjFSKext/PrjFSKext/KauthHandler.cpp +++ b/ProjFS.Mac/PrjFSKext/PrjFSKext/KauthHandler.cpp @@ -7,6 +7,7 @@ #include "PrjFSCommon.h" #include "VirtualizationRoots.hpp" +#include "VnodeUtilities.hpp" #include "KauthHandler.hpp" #include "KextLog.hpp" #include "Message.h" @@ -48,6 +49,7 @@ static bool TrySendRequestAndWaitForResponse( const VirtualizationRoot* root, MessageType messageType, const vnode_t vnode, + const FsidInode& vnodeFsidInode, const char* vnodePath, int pid, const char* procname, @@ -66,6 +68,7 @@ static bool ShouldHandleVnodeOpEvent( VirtualizationRoot** root, vtype* vnodeType, uint32_t* vnodeFileFlags, + FsidInode* vnodeFsidInode, int* pid, char procname[MAXCOMLEN + 1], int* kauthResult); @@ -78,6 +81,7 @@ static bool ShouldHandleFileOpEvent( // Out params: VirtualizationRoot** root, + FsidInode* vnodeFsidInode, int* pid); // Structs @@ -264,6 +268,7 @@ static int HandleVnodeOperation( VirtualizationRoot* root = nullptr; vtype vnodeType; uint32_t currentVnodeFileFlags; + FsidInode vnodeFsidInode; int pid; char procname[MAXCOMLEN + 1]; bool isDeleteAction = false; @@ -284,6 +289,7 @@ static int HandleVnodeOperation( &root, &vnodeType, ¤tVnodeFileFlags, + &vnodeFsidInode, &pid, procname, &kauthResult)) @@ -302,6 +308,7 @@ static int HandleVnodeOperation( MessageType_KtoU_NotifyDirectoryPreDelete : MessageType_KtoU_NotifyFilePreDelete, currentVnode, + vnodeFsidInode, vnodePath, pid, procname, @@ -332,6 +339,7 @@ static int HandleVnodeOperation( MessageType_KtoU_RecursivelyEnumerateDirectory : MessageType_KtoU_EnumerateDirectory, currentVnode, + vnodeFsidInode, vnodePath, pid, procname, @@ -362,6 +370,7 @@ static int HandleVnodeOperation( root, MessageType_KtoU_HydrateFile, currentVnode, + vnodeFsidInode, vnodePath, pid, procname, @@ -410,12 +419,14 @@ static int HandleFileOpOperation( } VirtualizationRoot* root = nullptr; + FsidInode vnodeFsidInode; int pid; if (!ShouldHandleFileOpEvent( context, currentVnodeFromPath, action, &root, + &vnodeFsidInode, &pid)) { goto CleanupAndReturn; @@ -440,6 +451,7 @@ static int HandleFileOpOperation( root, messageType, currentVnodeFromPath, + vnodeFsidInode, newPath, pid, procname, @@ -467,12 +479,14 @@ static int HandleFileOpOperation( } VirtualizationRoot* root = nullptr; + FsidInode vnodeFsidInode; int pid; if (!ShouldHandleFileOpEvent( context, currentVnode, action, &root, + &vnodeFsidInode, &pid)) { goto CleanupAndReturn; @@ -489,6 +503,7 @@ static int HandleFileOpOperation( root, MessageType_KtoU_NotifyFileModified, currentVnode, + vnodeFsidInode, path, pid, procname, @@ -506,6 +521,7 @@ static int HandleFileOpOperation( root, MessageType_KtoU_NotifyFileCreated, currentVnode, + vnodeFsidInode, path, pid, procname, @@ -541,6 +557,7 @@ static bool ShouldHandleVnodeOpEvent( VirtualizationRoot** root, vtype* vnodeType, uint32_t* vnodeFileFlags, + FsidInode* vnodeFsidInode, int* pid, char procname[MAXCOMLEN + 1], int* kauthResult) @@ -591,7 +608,8 @@ static bool ShouldHandleVnodeOpEvent( } } - *root = VirtualizationRoots_FindForVnode(vnode); + *vnodeFsidInode = Vnode_GetFsidAndInode(vnode, context); + *root = VirtualizationRoots_FindForVnode(vnode, *vnodeFsidInode); if (nullptr == *root) { @@ -629,6 +647,7 @@ static bool ShouldHandleFileOpEvent( // Out params: VirtualizationRoot** root, + FsidInode* vnodeFsidInode, int* pid) { vtype vnodeType = vnode_vtype(vnode); @@ -636,8 +655,9 @@ static bool ShouldHandleFileOpEvent( { return false; } - - *root = VirtualizationRoots_FindForVnode(vnode); + + *vnodeFsidInode = Vnode_GetFsidAndInode(vnode, context); + *root = VirtualizationRoots_FindForVnode(vnode, *vnodeFsidInode); if (nullptr == *root) { return false; @@ -662,6 +682,7 @@ static bool TrySendRequestAndWaitForResponse( const VirtualizationRoot* root, MessageType messageType, const vnode_t vnode, + const FsidInode& vnodeFsidInode, const char* vnodePath, int pid, const char* procname, @@ -686,7 +707,15 @@ static bool TrySendRequestAndWaitForResponse( int nextMessageId = OSIncrementAtomic(&s_nextMessageId); Message messageSpec = {}; - Message_Init(&messageSpec, &(message.request), nextMessageId, messageType, pid, procname, relativePath); + Message_Init( + &messageSpec, + &(message.request), + nextMessageId, + messageType, + vnodeFsidInode, + pid, + procname, + relativePath); bool isShuttingDown = false; Mutex_Acquire(s_outstandingMessagesMutex); diff --git a/ProjFS.Mac/PrjFSKext/PrjFSKext/Message_Kernel.cpp b/ProjFS.Mac/PrjFSKext/PrjFSKext/Message_Kernel.cpp index 1c0d80ced..aebb99964 100644 --- a/ProjFS.Mac/PrjFSKext/PrjFSKext/Message_Kernel.cpp +++ b/ProjFS.Mac/PrjFSKext/PrjFSKext/Message_Kernel.cpp @@ -6,12 +6,14 @@ void Message_Init( MessageHeader* header, uint64_t messageId, MessageType messageType, + const FsidInode& fsidInode, int32_t pid, const char* procname, const char* path) { header->messageId = messageId; header->messageType = messageType; + header->fsidInode = fsidInode; header->pid = pid; if (nullptr != procname) diff --git a/ProjFS.Mac/PrjFSKext/PrjFSKext/VirtualizationRoots.cpp b/ProjFS.Mac/PrjFSKext/PrjFSKext/VirtualizationRoots.cpp index 443a8d30f..46a5704d9 100644 --- a/ProjFS.Mac/PrjFSKext/PrjFSKext/VirtualizationRoots.cpp +++ b/ProjFS.Mac/PrjFSKext/PrjFSKext/VirtualizationRoots.cpp @@ -20,9 +20,9 @@ static const size_t MaxVirtualizationRoots = 128; static VirtualizationRoot s_virtualizationRoots[MaxVirtualizationRoots] = {}; -static int16_t FindRootForVnode_Locked(vnode_t vnode, uint32_t vid, VnodeFsidInode fileId); +static int16_t FindRootForVnode_Locked(vnode_t vnode, uint32_t vid, FsidInode fileId); static int16_t FindUnusedIndex_Locked(); -static int16_t InsertVirtualizationRoot_Locked(PrjFSProviderUserClient* userClient, pid_t clientPID, vnode_t vnode, uint32_t vid, VnodeFsidInode persistentIds, const char* path); +static int16_t InsertVirtualizationRoot_Locked(PrjFSProviderUserClient* userClient, pid_t clientPID, vnode_t vnode, uint32_t vid, FsidInode persistentIds, const char* path); kern_return_t VirtualizationRoots_Init() { @@ -56,7 +56,7 @@ kern_return_t VirtualizationRoots_Cleanup() return KERN_FAILURE; } -VirtualizationRoot* VirtualizationRoots_FindForVnode(vnode_t vnode) +VirtualizationRoot* VirtualizationRoots_FindForVnode(vnode_t vnode, const FsidInode& vnodeFsidInode) { VirtualizationRoot* root = nullptr; @@ -64,7 +64,7 @@ VirtualizationRoot* VirtualizationRoots_FindForVnode(vnode_t vnode) // Search up the tree until we hit a known virtualization root or THE root of the file system while (nullptr == root && NULLVP != vnode && !vnode_isvroot(vnode)) { - int16_t rootIndex = VirtualizationRoots_LookupVnode(vnode, nullptr); + int16_t rootIndex = VirtualizationRoots_LookupVnode(vnode, /*context*/ nullptr, vnodeFsidInode); if (rootIndex >= 0) { root = &s_virtualizationRoots[rootIndex]; @@ -84,16 +84,15 @@ VirtualizationRoot* VirtualizationRoots_FindForVnode(vnode_t vnode) return root; } -int16_t VirtualizationRoots_LookupVnode(vnode_t vnode, vfs_context_t context) +int16_t VirtualizationRoots_LookupVnode(vnode_t vnode, vfs_context_t context, const FsidInode& vnodeFsidInode) { - VnodeFsidInode fsidInode = Vnode_GetFsidAndInode(vnode, context); uint32_t vid = vnode_vid(vnode); int16_t rootIndex; RWLock_AcquireShared(s_rwLock); { - rootIndex = FindRootForVnode_Locked(vnode, vid, fsidInode); + rootIndex = FindRootForVnode_Locked(vnode, vid, vnodeFsidInode); } RWLock_ReleaseShared(s_rwLock); @@ -112,12 +111,12 @@ int16_t VirtualizationRoots_LookupVnode(vnode_t vnode, vfs_context_t context) RWLock_AcquireExclusive(s_rwLock); { // Vnode may already have been inserted as a root in the interim - rootIndex = FindRootForVnode_Locked(vnode, vid, fsidInode); + rootIndex = FindRootForVnode_Locked(vnode, vid, vnodeFsidInode); if (rootIndex < 0) { // Insert new offline root - rootIndex = InsertVirtualizationRoot_Locked(nullptr, 0, vnode, vid, fsidInode, path); + rootIndex = InsertVirtualizationRoot_Locked(nullptr, 0, vnode, vid, vnodeFsidInode, path); // TODO: error handling assert(rootIndex >= 0); @@ -149,7 +148,7 @@ static bool FsidsAreEqual(fsid_t a, fsid_t b) return a.val[0] == b.val[0] && a.val[1] == b.val[1]; } -static int16_t FindRootForVnode_Locked(vnode_t vnode, uint32_t vid, VnodeFsidInode fileId) +static int16_t FindRootForVnode_Locked(vnode_t vnode, uint32_t vid, FsidInode fileId) { for (int16_t i = 0; i < MaxVirtualizationRoots; ++i) { @@ -177,7 +176,7 @@ static int16_t FindRootForVnode_Locked(vnode_t vnode, uint32_t vid, VnodeFsidIno } // Returns negative value if it failed, or inserted index on success -static int16_t InsertVirtualizationRoot_Locked(PrjFSProviderUserClient* userClient, pid_t clientPID, vnode_t vnode, uint32_t vid, VnodeFsidInode persistentIds, const char* path) +static int16_t InsertVirtualizationRoot_Locked(PrjFSProviderUserClient* userClient, pid_t clientPID, vnode_t vnode, uint32_t vid, FsidInode persistentIds, const char* path) { // New root int16_t rootIndex = FindUnusedIndex_Locked(); @@ -230,7 +229,7 @@ VirtualizationRootResult VirtualizationRoot_RegisterProviderForPath(PrjFSProvide } else { - VnodeFsidInode vnodeIds = Vnode_GetFsidAndInode(virtualizationRootVNode, vfsContext); + FsidInode vnodeIds = Vnode_GetFsidAndInode(virtualizationRootVNode, vfsContext); uint32_t rootVid = vnode_vid(virtualizationRootVNode); RWLock_AcquireExclusive(s_rwLock); diff --git a/ProjFS.Mac/PrjFSKext/PrjFSKext/VirtualizationRoots.hpp b/ProjFS.Mac/PrjFSKext/PrjFSKext/VirtualizationRoots.hpp index 49389c10c..3d1c86aef 100644 --- a/ProjFS.Mac/PrjFSKext/PrjFSKext/VirtualizationRoots.hpp +++ b/ProjFS.Mac/PrjFSKext/PrjFSKext/VirtualizationRoots.hpp @@ -1,5 +1,6 @@ #pragma once +#include "FsidInode.h" #include "PrjFSClasses.hpp" #include "kernel-header-wrappers/vnode.h" @@ -27,7 +28,7 @@ struct VirtualizationRoot kern_return_t VirtualizationRoots_Init(void); kern_return_t VirtualizationRoots_Cleanup(void); -VirtualizationRoot* VirtualizationRoots_FindForVnode(vnode_t vnode); +VirtualizationRoot* VirtualizationRoots_FindForVnode(vnode_t vnode, const FsidInode& vnodeFsidInode); struct VirtualizationRootResult { @@ -41,4 +42,4 @@ struct Message; errno_t ActiveProvider_SendMessage(int32_t rootIndex, const Message message); bool VirtualizationRoot_VnodeIsOnAllowedFilesystem(vnode_t vnode); -int16_t VirtualizationRoots_LookupVnode(vnode_t vnode, vfs_context_t context); +int16_t VirtualizationRoots_LookupVnode(vnode_t vnode, vfs_context_t context, const FsidInode& vnodeFsidInode); diff --git a/ProjFS.Mac/PrjFSKext/PrjFSKext/VnodeUtilities.cpp b/ProjFS.Mac/PrjFSKext/PrjFSKext/VnodeUtilities.cpp index f12908751..c73aa5bfe 100644 --- a/ProjFS.Mac/PrjFSKext/PrjFSKext/VnodeUtilities.cpp +++ b/ProjFS.Mac/PrjFSKext/PrjFSKext/VnodeUtilities.cpp @@ -5,7 +5,7 @@ extern "C" int mac_vnop_getxattr(struct vnode *, const char *, char *, size_t, size_t *); -VnodeFsidInode Vnode_GetFsidAndInode(vnode_t vnode, vfs_context_t context) +FsidInode Vnode_GetFsidAndInode(vnode_t vnode, vfs_context_t context) { vnode_attr attrs; VATTR_INIT(&attrs); diff --git a/ProjFS.Mac/PrjFSKext/PrjFSKext/VnodeUtilities.hpp b/ProjFS.Mac/PrjFSKext/PrjFSKext/VnodeUtilities.hpp index 7ab01a053..068c94efd 100644 --- a/ProjFS.Mac/PrjFSKext/PrjFSKext/VnodeUtilities.hpp +++ b/ProjFS.Mac/PrjFSKext/PrjFSKext/VnodeUtilities.hpp @@ -2,6 +2,7 @@ #include #include +#include "FsidInode.h" struct SizeOrError { @@ -10,10 +11,4 @@ struct SizeOrError }; SizeOrError Vnode_ReadXattr(vnode_t vnode, const char* xattrName, void* buffer, size_t bufferSize, vfs_context_t context); - -struct VnodeFsidInode -{ - fsid_t fsid; - uint64_t inode; -}; -VnodeFsidInode Vnode_GetFsidAndInode(vnode_t vnode, vfs_context_t context); +FsidInode Vnode_GetFsidAndInode(vnode_t vnode, vfs_context_t context); diff --git a/ProjFS.Mac/PrjFSKext/public/FsidInode.h b/ProjFS.Mac/PrjFSKext/public/FsidInode.h new file mode 100644 index 000000000..caf2f7715 --- /dev/null +++ b/ProjFS.Mac/PrjFSKext/public/FsidInode.h @@ -0,0 +1,20 @@ +// +// FsidInode.h +// PrjFSKext +// +// Created by William Baker on 10/4/18. +// Copyright © 2018 GVFS. All rights reserved. +// + +#ifndef FsidInode_h +#define FsidInode_h + +#include + +struct FsidInode +{ + fsid_t fsid; + uint64_t inode; +}; + +#endif /* FsidInode_h */ diff --git a/ProjFS.Mac/PrjFSKext/public/Message.h b/ProjFS.Mac/PrjFSKext/public/Message.h index 43a99ab06..7257cb385 100644 --- a/ProjFS.Mac/PrjFSKext/public/Message.h +++ b/ProjFS.Mac/PrjFSKext/public/Message.h @@ -3,6 +3,7 @@ #include #include "PrjFSCommon.h" +#include "FsidInode.h" typedef enum { @@ -39,6 +40,9 @@ struct MessageHeader // The message type indicates the type of request or response uint32_t messageType; // values of type MessageType + // fsid and inode of the file + FsidInode fsidInode; + // For messages from kernel to user mode, indicates the PID of the process that initiated the I/O int32_t pid; char procname[MAXCOMLEN + 1]; @@ -59,6 +63,7 @@ void Message_Init( MessageHeader* header, uint64_t messageId, MessageType messageType, + const FsidInode& fsidInode, int32_t pid, const char* procname, const char* path); diff --git a/ProjFS.Mac/PrjFSLib/PrjFSLib.cpp b/ProjFS.Mac/PrjFSLib/PrjFSLib.cpp index a901cfed2..132ccd383 100644 --- a/ProjFS.Mac/PrjFSLib/PrjFSLib.cpp +++ b/ProjFS.Mac/PrjFSLib/PrjFSLib.cpp @@ -55,6 +55,32 @@ struct _PrjFS_FileHandle FILE* file; }; +struct FsidInodeCompare +{ + bool operator() (const FsidInode& lhs, const FsidInode& rhs) const + { + if (lhs.fsid.val[0] != rhs.fsid.val[0]) + { + return lhs.fsid.val[0] < rhs.fsid.val[0]; + } + + if (lhs.fsid.val[1] != rhs.fsid.val[1]) + { + return lhs.fsid.val[1] < rhs.fsid.val[1]; + } + + return lhs.inode < rhs.inode; + } +}; + +struct MutexAndUseCount +{ + shared_ptr mutex; + int useCount; +}; + +typedef map FileMutexMap; + // Function prototypes static bool SetBitInFileFlags(const char* path, uint32_t bit, bool value); static bool IsBitSetInFileFlags(const char* path, uint32_t bit); @@ -90,6 +116,9 @@ static void ClearMachNotification(mach_port_t port); static const char* NotificationTypeToString(PrjFS_NotificationType notificationType); #endif +static FileMutexMap::iterator CheckoutFileMutexIterator(const FsidInode& fsidInode); +static void ReturnFileMutexIterator(FileMutexMap::iterator lockIterator); + // State static io_connect_t s_kernelServiceConnection = IO_OBJECT_NULL; static string s_virtualizationRootFullPath; @@ -97,27 +126,9 @@ static PrjFS_Callbacks s_callbacks; static dispatch_queue_t s_messageQueueDispatchQueue; static dispatch_queue_t s_kernelRequestHandlingConcurrentQueue; -struct CaseInsensitiveStringCompare -{ - bool operator() (const std::string& lhs, const std::string& rhs) const - { - return strcasecmp(lhs.c_str(), rhs.c_str()) < 0; - } -}; - -struct MutexAndUseCount -{ - shared_ptr mutex; - int useCount; -}; - -// Map of relative path -> MutexAndUseCount for that path, plus mutex to protect the map itself. -typedef map PathToMutexMap; -PathToMutexMap s_inProgressExpansions; -mutex s_inProgressExpansionsMutex; - -static shared_ptr CheckoutPathMutex(const string& fullPath); -static void ReturnPathMutex(const string& fullPath, const shared_ptr& mutex); +// Map of FsidInode -> MutexAndUseCount for that FsidInode, plus mutex to protect the map itself. +FileMutexMap s_fileLocks; +mutex s_fileLocksMutex; // The full API is defined in the header, but only the minimal set of functions needed // for the initial MirrorProvider implementation are listed here. Calling any other function @@ -643,9 +654,9 @@ static PrjFS_Result HandleEnumerateDirectoryRequest(const MessageHeader* request } PrjFS_Result result; - shared_ptr expansionMutex = CheckoutPathMutex(fullPath); + FileMutexMap::iterator mutexIterator = CheckoutFileMutexIterator(request->fsidInode); { - mutex_lock lock(*expansionMutex); + mutex_lock lock(*(mutexIterator->second.mutex)); if (!IsBitSetInFileFlags(fullPath, FileFlags_IsEmpty)) { result = PrjFS_Result_Success; @@ -670,7 +681,7 @@ static PrjFS_Result HandleEnumerateDirectoryRequest(const MessageHeader* request } CleanupAndReturn: - ReturnPathMutex(fullPath, expansionMutex); + ReturnFileMutexIterator(mutexIterator); return result; } @@ -755,10 +766,10 @@ static PrjFS_Result HandleHydrateFileRequest(const MessageHeader* request, const PrjFS_Result result; PrjFS_FileHandle fileHandle; - shared_ptr hydrationMutex = CheckoutPathMutex(fullPath); + FileMutexMap::iterator mutexIterator = CheckoutFileMutexIterator(request->fsidInode); { - mutex_lock lock(*hydrationMutex); + mutex_lock lock(*(mutexIterator->second.mutex)); if (!IsBitSetInFileFlags(fullPath, FileFlags_IsEmpty)) { result = PrjFS_Result_Success; @@ -820,7 +831,7 @@ static PrjFS_Result HandleHydrateFileRequest(const MessageHeader* request, const } CleanupAndReturn: - ReturnPathMutex(fullPath, hydrationMutex); + ReturnFileMutexIterator(mutexIterator); return result; } @@ -1057,33 +1068,30 @@ static const char* NotificationTypeToString(PrjFS_NotificationType notificationT } #endif -static shared_ptr CheckoutPathMutex(const string& fullPath) +static FileMutexMap::iterator CheckoutFileMutexIterator(const FsidInode& fsidInode) { - mutex_lock lock(s_inProgressExpansionsMutex); - PathToMutexMap::iterator iter = s_inProgressExpansions.find(fullPath); - if (iter == s_inProgressExpansions.end()) + mutex_lock lock(s_fileLocksMutex); + FileMutexMap::iterator iter = s_fileLocks.find(fsidInode); + if (iter == s_fileLocks.end()) { - pair newEntry = s_inProgressExpansions.insert( - PathToMutexMap::value_type(fullPath, { make_shared(), 1 })); + pair newEntry = s_fileLocks.insert( + FileMutexMap::value_type(fsidInode, { make_shared(), 1 })); assert(newEntry.second); - return newEntry.first->second.mutex; + return newEntry.first; } else { iter->second.useCount++; - return iter->second.mutex; + return iter; } } -static void ReturnPathMutex(const string& fullPath, const shared_ptr& mutex) +static void ReturnFileMutexIterator(FileMutexMap::iterator lockIterator) { - mutex_lock lock(s_inProgressExpansionsMutex); - PathToMutexMap::iterator iter = s_inProgressExpansions.find(fullPath); - assert(iter != s_inProgressExpansions.end()); - assert(iter->second.mutex.get() == mutex.get()); - iter->second.useCount--; - if (iter->second.useCount == 0) + mutex_lock lock(s_fileLocksMutex); + lockIterator->second.useCount--; + if (lockIterator->second.useCount == 0) { - s_inProgressExpansions.erase(iter); + s_fileLocks.erase(lockIterator); } } From 5ab446331bbb33ae61477855d58cae0efcdd664d Mon Sep 17 00:00:00 2001 From: Kyle Rader Date: Thu, 30 Aug 2018 15:31:08 -0700 Subject: [PATCH 216/272] Add Mac Setup instructions needed for building on Mac --- Readme.md | 41 +++++++++++++++++++++++++++++++++++++---- 1 file changed, 37 insertions(+), 4 deletions(-) diff --git a/Readme.md b/Readme.md index 133b93754..a41a9c1e7 100644 --- a/Readme.md +++ b/Readme.md @@ -30,10 +30,10 @@ built executables, and releases may still refer to the old GVFS name. See https: * VFS for Git requires Windows 10 Anniversary Update (Windows 10 version 1607) or later * Run the latest GVFS and Git for Windows installers from https://github.com/Microsoft/VFSForGit/releases -## Building VFS for Git +## Building VFS for Git on Windows If you'd like to build your own VFS for Git Windows installer: -* Install Visual Studio 2017 Community Edition or higher (https://www.visualstudio.com/downloads/). +* Install Visual Studio 2017 Community Edition or higher (https://www.visualstudio.com/downloads/). * Include the following workloads: * .NET desktop development * Desktop development with C++ @@ -46,15 +46,47 @@ If you'd like to build your own VFS for Git Windows installer: * Create a folder to clone into, e.g. `C:\Repos\VFSForGit` * Clone this repo into the `src` subfolder, e.g. `C:\Repos\VFSForGit\src` * Run `\src\Scripts\BuildGVFSForWindows.bat` -* You can also build in Visual Studio by opening `src\GVFS.sln` (do not upgrade any projects) and building. However, the very first +* You can also build in Visual Studio by opening `src\GVFS.sln` (do not upgrade any projects) and building. However, the very first build will fail, and the second and subsequent builds will succeed. This is because the build requires a prebuild code generation step. For details, see the build script in the previous step. The installer can now be found at `C:\Repos\VFSForGit\BuildOutput\GVFS.Installer\bin\x64\[Debug|Release]\SetupGVFS..exe` +## Building VFS for Git on Mac + +### First setup your Mac + +* Ensure you have Xcode installed and have accepted the terms of use (Launch Xcode at least once). +* Install [Visual Studio for Mac ](https://visualstudio.microsoft.com/vs/mac) +* If you still do not have the `dotnet` cli `>= v2.1.300` installed [manually install it](https://www.microsoft.com/net/download/dotnet-core/2.1) +* [Install Homebrew](https://brew.sh/) +* Install Java with + ``` + brew cask install java + ``` + +* Install the Git credential manager for Mac [by following their Homebrew instructions](https://github.com/Microsoft/Git-Credential-Manager-for-Mac-and-Linux/blob/master/Install.md#installing-on-mac-using-homebrew-or-on-linux-using-linuxbrew-recommended) + +* If you are not currently an Apple Developer you will need to become one so that you can either use Microsoft certs or create your own certs for signing purposes. (If you are a Microsoft employee you must use your alias@microsoft.com when creating your Apple account.) + +* Create a `VFS` directory and Clone the VFS repo into a directory called `src` inside it: + ``` + mkdir VFS && cd VFS && git clone https://github.com/Microsoft/VFSForGit.git src && cd src + ``` + +* Disable "System Integrity Protection" (for loading unsigned kexts) by booting into recovery mode (`[Win/⌘] + R` while booting). + Once booted into recovery mode open Utilities -> Terminal to launch a termnial. Enter: + ``` + csrutil disable + ``` + to disable SIP. + +* Now `cd` into + + ## Trying out VFS for Git -* VFS for Git will work with any git service that supports the GVFS [protocol](Protocol.md). For example, you can create a repo in +* VFS for Git will work with any git service that supports the GVFS [protocol](Protocol.md). For example, you can create a repo in Visual Studio Team Services (https://www.visualstudio.com/team-services/), and push some contents to it. There are two constraints: * Your repo must not enable any clean/smudge filters * Your repo must have a `.gitattributes` file in the root that includes the line `* -text` @@ -63,6 +95,7 @@ Visual Studio Team Services (https://www.visualstudio.com/team-services/), and p * Run git commands as you normally would * `gvfs unmount` when done + # Licenses The VFS for Git source code in this repo is available under the MIT license. See [License.md](License.md). From 84fdbf89392259ca00608d5ed8bdcb8c2faf291b Mon Sep 17 00:00:00 2001 From: Kyle Rader Date: Wed, 19 Sep 2018 20:21:50 -0700 Subject: [PATCH 217/272] Add initial set of build steps for Mac OS --- ProjFS.Mac/Scripts/Build.sh | 4 +-- Readme.md | 54 ++++++++++++++++++++++++++++++++-- Scripts/Mac/BuildGVFSForMac.sh | 8 ++--- 3 files changed, 57 insertions(+), 9 deletions(-) diff --git a/ProjFS.Mac/Scripts/Build.sh b/ProjFS.Mac/Scripts/Build.sh index 2f298f876..333883c37 100755 --- a/ProjFS.Mac/Scripts/Build.sh +++ b/ProjFS.Mac/Scripts/Build.sh @@ -8,11 +8,11 @@ fi SCRIPTDIR=$(dirname ${BASH_SOURCE[0]}) SRCDIR=$SCRIPTDIR/../.. ROOTDIR=$SRCDIR/.. -PACKAGES=$ROOTDIR/packages +PACKAGES=$ROOTDIR/packages PROJFS=$SRCDIR/ProjFS.Mac -xcodebuild -sdk macosx10.13 -configuration $CONFIGURATION -workspace $PROJFS/PrjFS.xcworkspace build -scheme PrjFS -derivedDataPath $ROOTDIR/BuildOutput/ProjFS.Mac/Native || exit 1 +xcodebuild -configuration $CONFIGURATION -workspace $PROJFS/PrjFS.xcworkspace build -scheme PrjFS -derivedDataPath $ROOTDIR/BuildOutput/ProjFS.Mac/Native || exit 1 dotnet restore $PROJFS/PrjFSLib.Mac.Managed/PrjFSLib.Mac.Managed.csproj /p:Configuration=$CONFIGURATION /p:Platform=x64 --packages $PACKAGES || exit 1 dotnet build $PROJFS/PrjFSLib.Mac.Managed/PrjFSLib.Mac.Managed.csproj /p:Configuration=$CONFIGURATION /p:Platform=x64 || exit 1 diff --git a/Readme.md b/Readme.md index a41a9c1e7..8f0dc078e 100644 --- a/Readme.md +++ b/Readme.md @@ -56,8 +56,8 @@ The installer can now be found at `C:\Repos\VFSForGit\BuildOutput\GVFS.Installer ### First setup your Mac -* Ensure you have Xcode installed and have accepted the terms of use (Launch Xcode at least once). -* Install [Visual Studio for Mac ](https://visualstudio.microsoft.com/vs/mac) +* Ensure you have Xcode installed and have accepted the terms of use and launched Xcode at least once. +* Install [Visual Studio for Mac ](https://visualstudio.microsoft.com/vs/mac). (This will also install the `dotnet` CLI). * If you still do not have the `dotnet` cli `>= v2.1.300` installed [manually install it](https://www.microsoft.com/net/download/dotnet-core/2.1) * [Install Homebrew](https://brew.sh/) * Install Java with @@ -81,7 +81,55 @@ The installer can now be found at `C:\Repos\VFSForGit\BuildOutput\GVFS.Installer ``` to disable SIP. -* Now `cd` into +* Now `cd` back into your VFS src directory and run + + ``` + Scripts/Mac/BuildGVFSForMac.sh [Release] + ``` + + **Troubleshooting** + + If you get + ``` + xcodebuild: error: SDK "macosx10.13" cannot be located. + ``` + You may have the "XCode Command Line Tools" installed (helpfully by Mac OS) instead of full XCode. + Make sure + ``` + xcode-select -p + ``` + + shows `/Applications/Xcode.app/Contents/Developer`. If it does not, install Xcode and then launch it (you can close it afterwards.) + +* Now setup the git credential manager by running + + ``` + Scripts/Mac/PrepFunctionalTests.sh + ``` + +* Now you have to load the ProjFS Kext. + + ``` + ProjFS.Mac/Scripts/LoadPrjFSKext.sh [Release] + ``` + +* Now you can put your built gvfs program on your path. A simple way to do that is by adding `Path/to/VFS/Publish` to your path. + + Check you have it by running + + ``` + command -v gvfs + ``` + + You should see a path to the gvfs executable. + +* Try cloning! Now that you have `gvfs` ready you can try cloning a VFS for Git enabled repository! + + ``` + gvfs clone URL_TO_REPOSITORY [--cache-url] --local-cache-path ~/.gvfsCache + ``` + + Note you may have a cache server URL to use. Note the current use of `--local-cache-path`. That argument prevent gvfs from running into a permissions error trying to put the cache at the root of your Mac's hard-drive. ## Trying out VFS for Git diff --git a/Scripts/Mac/BuildGVFSForMac.sh b/Scripts/Mac/BuildGVFSForMac.sh index 2eb875c77..678bb89aa 100755 --- a/Scripts/Mac/BuildGVFSForMac.sh +++ b/Scripts/Mac/BuildGVFSForMac.sh @@ -14,20 +14,20 @@ popd SRCDIR=$SCRIPTDIR/../.. ROOTDIR=$SRCDIR/.. -BUILDOUTPUT=$ROOTDIR/BuildOutput +BUILDOUTPUT=$ROOTDIR/BuildOutput PUBLISHDIR=$ROOTDIR/Publish if [ ! -d $BUILDOUTPUT ]; then mkdir $BUILDOUTPUT fi -PACKAGES=$ROOTDIR/packages +PACKAGES=$ROOTDIR/packages # Build the ProjFS kext and libraries $SRCDIR/ProjFS.Mac/Scripts/Build.sh $CONFIGURATION || exit 1 # Create the directory where we'll do pre build tasks -BUILDDIR=$BUILDOUTPUT/GVFS.Build +BUILDDIR=$BUILDOUTPUT/GVFS.Build if [ ! -d $BUILDDIR ]; then mkdir $BUILDDIR || exit 1 fi @@ -44,7 +44,7 @@ dotnet build $SRCDIR/GVFS.sln --runtime osx-x64 --framework netcoreapp2.1 --conf dotnet publish $SRCDIR/GVFS.sln /p:Configuration=$DOTNETCONFIGURATION /p:Platform=x64 --runtime osx-x64 --framework netcoreapp2.1 --self-contained --output $PUBLISHDIR /maxcpucount:1 || exit 1 NATIVEDIR=$SRCDIR/GVFS/GVFS.Native.Mac -xcodebuild -sdk macosx10.13 -configuration $CONFIGURATION -workspace $NATIVEDIR/GVFS.Native.Mac.xcworkspace build -scheme GVFS.Native.Mac -derivedDataPath $ROOTDIR/BuildOutput/GVFS.Native.Mac || exit 1 +xcodebuild -configuration $CONFIGURATION -workspace $NATIVEDIR/GVFS.Native.Mac.xcworkspace build -scheme GVFS.Native.Mac -derivedDataPath $ROOTDIR/BuildOutput/GVFS.Native.Mac || exit 1 echo 'Copying native binaries to Publish directory' cp $BUILDOUTPUT/GVFS.Native.Mac/Build/Products/$CONFIGURATION/GVFS.ReadObjectHook $PUBLISHDIR || exit 1 From 871f87c59a56e8566e10424a0f4c44a5e09da459 Mon Sep 17 00:00:00 2001 From: Kyle Rader Date: Thu, 20 Sep 2018 14:54:48 -0700 Subject: [PATCH 218/272] Update mac os build subtitle --- Readme.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Readme.md b/Readme.md index 8f0dc078e..aada7ffbd 100644 --- a/Readme.md +++ b/Readme.md @@ -54,7 +54,7 @@ The installer can now be found at `C:\Repos\VFSForGit\BuildOutput\GVFS.Installer ## Building VFS for Git on Mac -### First setup your Mac +VFS for macOS is still in progress. You can build it, but this will not create a macOS VFS installer the same way the current Windows build will. * Ensure you have Xcode installed and have accepted the terms of use and launched Xcode at least once. * Install [Visual Studio for Mac ](https://visualstudio.microsoft.com/vs/mac). (This will also install the `dotnet` CLI). From 88b1c55254a966586c974f0455c4639b2f1080d4 Mon Sep 17 00:00:00 2001 From: Kyle Rader Date: Thu, 20 Sep 2018 15:18:57 -0700 Subject: [PATCH 219/272] Remove steps covered by the prep script --- Readme.md | 27 +++++++++++---------------- 1 file changed, 11 insertions(+), 16 deletions(-) diff --git a/Readme.md b/Readme.md index aada7ffbd..ee8c0562b 100644 --- a/Readme.md +++ b/Readme.md @@ -57,15 +57,17 @@ The installer can now be found at `C:\Repos\VFSForGit\BuildOutput\GVFS.Installer VFS for macOS is still in progress. You can build it, but this will not create a macOS VFS installer the same way the current Windows build will. * Ensure you have Xcode installed and have accepted the terms of use and launched Xcode at least once. -* Install [Visual Studio for Mac ](https://visualstudio.microsoft.com/vs/mac). (This will also install the `dotnet` CLI). -* If you still do not have the `dotnet` cli `>= v2.1.300` installed [manually install it](https://www.microsoft.com/net/download/dotnet-core/2.1) -* [Install Homebrew](https://brew.sh/) -* Install Java with + +* Disable "System Integrity Protection" (for loading unsigned kexts) by booting into recovery mode (`[Win/⌘] + R` while booting). + Once booted into recovery mode open Utilities -> Terminal to launch a termnial. Enter: ``` - brew cask install java + csrutil disable ``` + to disable SIP. -* Install the Git credential manager for Mac [by following their Homebrew instructions](https://github.com/Microsoft/Git-Credential-Manager-for-Mac-and-Linux/blob/master/Install.md#installing-on-mac-using-homebrew-or-on-linux-using-linuxbrew-recommended) +* Install [Visual Studio for Mac ](https://visualstudio.microsoft.com/vs/mac). (This will also install the `dotnet` CLI). + +* If you still do not have the `dotnet` cli `>= v2.1.300` installed [manually install it](https://www.microsoft.com/net/download/dotnet-core/2.1) * If you are not currently an Apple Developer you will need to become one so that you can either use Microsoft certs or create your own certs for signing purposes. (If you are a Microsoft employee you must use your alias@microsoft.com when creating your Apple account.) @@ -74,20 +76,13 @@ VFS for macOS is still in progress. You can build it, but this will not create a mkdir VFS && cd VFS && git clone https://github.com/Microsoft/VFSForGit.git src && cd src ``` -* Disable "System Integrity Protection" (for loading unsigned kexts) by booting into recovery mode (`[Win/⌘] + R` while booting). - Once booted into recovery mode open Utilities -> Terminal to launch a termnial. Enter: - ``` - csrutil disable - ``` - to disable SIP. - * Now `cd` back into your VFS src directory and run ``` Scripts/Mac/BuildGVFSForMac.sh [Release] ``` - **Troubleshooting** + _Troubleshooting if this fails_ If you get ``` @@ -126,10 +121,10 @@ VFS for macOS is still in progress. You can build it, but this will not create a * Try cloning! Now that you have `gvfs` ready you can try cloning a VFS for Git enabled repository! ``` - gvfs clone URL_TO_REPOSITORY [--cache-url] --local-cache-path ~/.gvfsCache + gvfs clone URL_TO_REPOSITORY --local-cache-path ~/.gvfsCache ``` - Note you may have a cache server URL to use. Note the current use of `--local-cache-path`. That argument prevent gvfs from running into a permissions error trying to put the cache at the root of your Mac's hard-drive. + Note the current use of `--local-cache-path`. That argument prevent gvfs from running into a permissions error trying to put the cache at the root of your Mac's hard-drive. ## Trying out VFS for Git From fbc892a2df053d4322a40cc78420794dd84a6b56 Mon Sep 17 00:00:00 2001 From: Kyle Rader Date: Thu, 20 Sep 2018 15:22:09 -0700 Subject: [PATCH 220/272] Remove deprecated java hack for git-credential-manager --- Scripts/Mac/PrepFunctionalTests.sh | 7 ------- 1 file changed, 7 deletions(-) diff --git a/Scripts/Mac/PrepFunctionalTests.sh b/Scripts/Mac/PrepFunctionalTests.sh index f1a0c4260..fb35bde65 100755 --- a/Scripts/Mac/PrepFunctionalTests.sh +++ b/Scripts/Mac/PrepFunctionalTests.sh @@ -36,13 +36,6 @@ fi git-credential-manager install -# If our Java version is 9+ (the formatting of 'java -version' changed in Java 9), work around -# https://github.com/Microsoft/Git-Credential-Manager-for-Mac-and-Linux/issues/71 -JAVAVERSION="$(java -version 2>&1 | egrep -o '"[[:digit:]]+.[[:digit:]]+.[[:digit:]]+"' | xargs)" -if [[ ! -z $JAVAVERSION ]]; then - git config --global credential.helper "!/usr/bin/java -Ddebug=false --add-modules java.xml.bind -Djava.net.useSystemProxies=true -jar /usr/local/Cellar/git-credential-manager/2.0.3/libexec/git-credential-manager-2.0.3.jar" || exit 1 -fi - # If we're running on an agent where the PAT environment variable is set and a URL is passed into the script, add it to the keychain. PATURL=$1 PAT=$2 From 96218db5912168e5bd64406c724983de2696a70a Mon Sep 17 00:00:00 2001 From: Kyle Rader Date: Fri, 21 Sep 2018 11:23:34 -0700 Subject: [PATCH 221/272] Grammar --- Readme.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Readme.md b/Readme.md index ee8c0562b..9338be913 100644 --- a/Readme.md +++ b/Readme.md @@ -56,7 +56,7 @@ The installer can now be found at `C:\Repos\VFSForGit\BuildOutput\GVFS.Installer VFS for macOS is still in progress. You can build it, but this will not create a macOS VFS installer the same way the current Windows build will. -* Ensure you have Xcode installed and have accepted the terms of use and launched Xcode at least once. +* Ensure you have Xcode installed, have accepted the terms of use, and launched Xcode at least once. * Disable "System Integrity Protection" (for loading unsigned kexts) by booting into recovery mode (`[Win/⌘] + R` while booting). Once booted into recovery mode open Utilities -> Terminal to launch a termnial. Enter: From 54b73152e6282149a02dc9740b2b3786ecf3e09d Mon Sep 17 00:00:00 2001 From: Kyle Rader Date: Fri, 21 Sep 2018 11:58:56 -0700 Subject: [PATCH 222/272] Style updates --- Readme.md | 22 +++++++++++++--------- 1 file changed, 13 insertions(+), 9 deletions(-) diff --git a/Readme.md b/Readme.md index 9338be913..354418433 100644 --- a/Readme.md +++ b/Readme.md @@ -54,29 +54,29 @@ The installer can now be found at `C:\Repos\VFSForGit\BuildOutput\GVFS.Installer ## Building VFS for Git on Mac -VFS for macOS is still in progress. You can build it, but this will not create a macOS VFS installer the same way the current Windows build will. +VFS for Git on Mac is still in progress. You can build it, but this will not create a macOS VFS installer the same way the current Windows build will. * Ensure you have Xcode installed, have accepted the terms of use, and launched Xcode at least once. -* Disable "System Integrity Protection" (for loading unsigned kexts) by booting into recovery mode (`[Win/⌘] + R` while booting). +* Disable the "System Integrity Protection" (for loading unsigned Kexts) by booting into recovery mode (`[Win/⌘] + R` while booting). Once booted into recovery mode open Utilities -> Terminal to launch a termnial. Enter: ``` csrutil disable ``` - to disable SIP. + to disable SIP. Then click the Apple logo in the top left and restart. * Install [Visual Studio for Mac ](https://visualstudio.microsoft.com/vs/mac). (This will also install the `dotnet` CLI). -* If you still do not have the `dotnet` cli `>= v2.1.300` installed [manually install it](https://www.microsoft.com/net/download/dotnet-core/2.1) +* If you still do not have the `dotnet` cli `>= v2.1.300` installed [manually install it]. You can check what version you have with `dotnet --version`.(https://www.microsoft.com/net/download/dotnet-core/2.1) -* If you are not currently an Apple Developer you will need to become one so that you can either use Microsoft certs or create your own certs for signing purposes. (If you are a Microsoft employee you must use your alias@microsoft.com when creating your Apple account.) +* You will need to manage and sign your own certificate. [Apple Developer Cert Docs](https://developer.apple.com/documentation/security/certificate_key_and_trust_services/certificates). -* Create a `VFS` directory and Clone the VFS repo into a directory called `src` inside it: +* Create a `VFS` directory and Clone VFSForGit into a directory called `src` inside it: ``` mkdir VFS && cd VFS && git clone https://github.com/Microsoft/VFSForGit.git src && cd src ``` -* Now `cd` back into your VFS src directory and run +* From the src directory run ``` Scripts/Mac/BuildGVFSForMac.sh [Release] @@ -96,7 +96,11 @@ VFS for macOS is still in progress. You can build it, but this will not create a shows `/Applications/Xcode.app/Contents/Developer`. If it does not, install Xcode and then launch it (you can close it afterwards.) -* Now setup the git credential manager by running +* Prep your machine to use VFS for Git. The following are all done by the script below.wwwww + * install Homebrew + * install and setup the git credential manager (with `brew`) + * install/update Java (with `brew`) + * install a VFS for Git aware version of Git ``` Scripts/Mac/PrepFunctionalTests.sh @@ -124,7 +128,7 @@ VFS for macOS is still in progress. You can build it, but this will not create a gvfs clone URL_TO_REPOSITORY --local-cache-path ~/.gvfsCache ``` - Note the current use of `--local-cache-path`. That argument prevent gvfs from running into a permissions error trying to put the cache at the root of your Mac's hard-drive. + Note the current use of `--local-cache-path`. That argument prevents VFS for Git from running into a permissions error trying to put the cache at the root of your Mac's hard-drive because porting of this feature is still in progress. ## Trying out VFS for Git From 4fe765d05f4e78b71a96737cf93437e5b6686663 Mon Sep 17 00:00:00 2001 From: Kyle Rader Date: Fri, 21 Sep 2018 15:27:40 -0700 Subject: [PATCH 223/272] Simplify steps language --- Readme.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/Readme.md b/Readme.md index 354418433..498a30433 100644 --- a/Readme.md +++ b/Readme.md @@ -112,9 +112,9 @@ VFS for Git on Mac is still in progress. You can build it, but this will not cre ProjFS.Mac/Scripts/LoadPrjFSKext.sh [Release] ``` -* Now you can put your built gvfs program on your path. A simple way to do that is by adding `Path/to/VFS/Publish` to your path. +* Add your built gvfs program to your path. A simple way to do that is by adding `Path/to/VFS/Publish` to your path. - Check you have it by running + Confirm you have it by running ``` command -v gvfs @@ -122,7 +122,7 @@ VFS for Git on Mac is still in progress. You can build it, but this will not cre You should see a path to the gvfs executable. -* Try cloning! Now that you have `gvfs` ready you can try cloning a VFS for Git enabled repository! +* Try cloning a VFS for Git enabled repository! ``` gvfs clone URL_TO_REPOSITORY --local-cache-path ~/.gvfsCache From b824382829fb9424e55bfba40015ec320db9b464 Mon Sep 17 00:00:00 2001 From: Kyle Rader Date: Wed, 26 Sep 2018 13:04:50 -0700 Subject: [PATCH 224/272] Update grammar issues --- Readme.md | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/Readme.md b/Readme.md index 498a30433..43dce75f2 100644 --- a/Readme.md +++ b/Readme.md @@ -54,9 +54,9 @@ The installer can now be found at `C:\Repos\VFSForGit\BuildOutput\GVFS.Installer ## Building VFS for Git on Mac -VFS for Git on Mac is still in progress. You can build it, but this will not create a macOS VFS installer the same way the current Windows build will. +Note that VFS for Git on Mac is under active development. -* Ensure you have Xcode installed, have accepted the terms of use, and launched Xcode at least once. +* Ensure you have Xcode installed, have accepted the terms of use, and have launched Xcode at least once. * Disable the "System Integrity Protection" (for loading unsigned Kexts) by booting into recovery mode (`[Win/⌘] + R` while booting). Once booted into recovery mode open Utilities -> Terminal to launch a termnial. Enter: @@ -71,9 +71,9 @@ VFS for Git on Mac is still in progress. You can build it, but this will not cre * You will need to manage and sign your own certificate. [Apple Developer Cert Docs](https://developer.apple.com/documentation/security/certificate_key_and_trust_services/certificates). -* Create a `VFS` directory and Clone VFSForGit into a directory called `src` inside it: +* Create a `VFSForGit` directory and Clone VFSForGit into a directory called `src` inside it: ``` - mkdir VFS && cd VFS && git clone https://github.com/Microsoft/VFSForGit.git src && cd src + mkdir VFSForGit && cd VFSForGit && git clone https://github.com/Microsoft/VFSForGit.git src && cd src ``` * From the src directory run From 01b68ac525c741e31d3ac41e12e871a5b704efbb Mon Sep 17 00:00:00 2001 From: Kyle Rader Date: Wed, 26 Sep 2018 13:07:19 -0700 Subject: [PATCH 225/272] Fix Spelling --- Readme.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/Readme.md b/Readme.md index 43dce75f2..1b87e3bbb 100644 --- a/Readme.md +++ b/Readme.md @@ -59,7 +59,8 @@ Note that VFS for Git on Mac is under active development. * Ensure you have Xcode installed, have accepted the terms of use, and have launched Xcode at least once. * Disable the "System Integrity Protection" (for loading unsigned Kexts) by booting into recovery mode (`[Win/⌘] + R` while booting). - Once booted into recovery mode open Utilities -> Terminal to launch a termnial. Enter: + Once booted into recovery mode open Utilities -> Terminal to launch a terminal. Enter: + ``` csrutil disable ``` From 53f1defea4b3b9fb2df385a97e3b637dd3a8b1d6 Mon Sep 17 00:00:00 2001 From: Kyle Rader Date: Thu, 4 Oct 2018 12:32:06 -0700 Subject: [PATCH 226/272] More style fixes --- Readme.md | 56 +++++++++++++++++++++++++++++++++++-------------------- 1 file changed, 36 insertions(+), 20 deletions(-) diff --git a/Readme.md b/Readme.md index 1b87e3bbb..d6ddc27fe 100644 --- a/Readme.md +++ b/Readme.md @@ -58,29 +58,36 @@ Note that VFS for Git on Mac is under active development. * Ensure you have Xcode installed, have accepted the terms of use, and have launched Xcode at least once. -* Disable the "System Integrity Protection" (for loading unsigned Kexts) by booting into recovery mode (`[Win/⌘] + R` while booting). - Once booted into recovery mode open Utilities -> Terminal to launch a terminal. Enter: - - ``` - csrutil disable - ``` - to disable SIP. Then click the Apple logo in the top left and restart. - * Install [Visual Studio for Mac ](https://visualstudio.microsoft.com/vs/mac). (This will also install the `dotnet` CLI). * If you still do not have the `dotnet` cli `>= v2.1.300` installed [manually install it]. You can check what version you have with `dotnet --version`.(https://www.microsoft.com/net/download/dotnet-core/2.1) -* You will need to manage and sign your own certificate. [Apple Developer Cert Docs](https://developer.apple.com/documentation/security/certificate_key_and_trust_services/certificates). +* You will need to manage and sign your own certificate. + + If you're using Xcode for the first time, you may have to login to Xcode with your Apple ID to generate a codesigning certificate. You can do this by launching Xcode.app, opening the PrjFS.xcworkspace and trying to build. You can find the signing options in the General tab of the project's settings. * Create a `VFSForGit` directory and Clone VFSForGit into a directory called `src` inside it: ``` - mkdir VFSForGit && cd VFSForGit && git clone https://github.com/Microsoft/VFSForGit.git src && cd src + mkdir VFSForGit + cd VFSForGit + git clone https://github.com/Microsoft/VFSForGit.git src + cd src + ``` + +* Prep your machine to use VFS for Git. The following are all done by the script below. + * install Homebrew + * install and setup the Git Credential Manager (with `brew`) + * install/update Java (with `brew`) + * install a VFS for Git aware version of Git + + ``` + Scripts/Mac/PrepFunctionalTests.sh ``` * From the src directory run ``` - Scripts/Mac/BuildGVFSForMac.sh [Release] + Scripts/Mac/BuildGVFSForMac.sh [Debug|Release] ``` _Troubleshooting if this fails_ @@ -97,23 +104,32 @@ Note that VFS for Git on Mac is under active development. shows `/Applications/Xcode.app/Contents/Developer`. If it does not, install Xcode and then launch it (you can close it afterwards.) -* Prep your machine to use VFS for Git. The following are all done by the script below.wwwww - * install Homebrew - * install and setup the git credential manager (with `brew`) - * install/update Java (with `brew`) - * install a VFS for Git aware version of Git +* For the time being, only for active development, you will have to disable the SIP (System Integrity Protection) in order to load the kext). + + **This is dangerous and very bad for the security of your machine. Do not do this on any production machine! If you no longer need to developer VFS for Git on Mac we recommend re-enabling SIP ASAP.** + + To disable SIP boot into recovery mode (`[Win/⌘] + R` while booting your Mac). + Once booted into recovery mode open Utilities -> Terminal to launch a terminal. Enter: ``` - Scripts/Mac/PrepFunctionalTests.sh + csrutil disable + # use "csrutil enable" to re-enable when you no longer need to build VFS for Git on Mac ``` + Then click the Apple logo in the top left and restart. * Now you have to load the ProjFS Kext. ``` - ProjFS.Mac/Scripts/LoadPrjFSKext.sh [Release] + ProjFS.Mac/Scripts/LoadPrjFSKext.sh [Debug|Release] + ``` + +* Add your built VFS for Git executable (`gvfs`) program to your path. A simple way to do that is by adding + + ``` + /VFSForGit/Publish ``` -* Add your built gvfs program to your path. A simple way to do that is by adding `Path/to/VFS/Publish` to your path. + to your `PATH`. Confirm you have it by running @@ -129,7 +145,7 @@ Note that VFS for Git on Mac is under active development. gvfs clone URL_TO_REPOSITORY --local-cache-path ~/.gvfsCache ``` - Note the current use of `--local-cache-path`. That argument prevents VFS for Git from running into a permissions error trying to put the cache at the root of your Mac's hard-drive because porting of this feature is still in progress. + Note the current use of `--local-cache-path`. Without this argument VFS for Git will encounter a permissions error when it attempts to create its cache at the root of your hard-drive. Automatic picking of the cache path has not yet been ported to VFS for Git on Mac. ## Trying out VFS for Git From 04b92037a4697d98e29e81d9aedd75a006590285 Mon Sep 17 00:00:00 2001 From: Kyle Rader Date: Thu, 4 Oct 2018 13:11:00 -0700 Subject: [PATCH 227/272] More grammar resolutions --- Readme.md | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/Readme.md b/Readme.md index d6ddc27fe..ca80a3f57 100644 --- a/Readme.md +++ b/Readme.md @@ -62,9 +62,7 @@ Note that VFS for Git on Mac is under active development. * If you still do not have the `dotnet` cli `>= v2.1.300` installed [manually install it]. You can check what version you have with `dotnet --version`.(https://www.microsoft.com/net/download/dotnet-core/2.1) -* You will need to manage and sign your own certificate. - - If you're using Xcode for the first time, you may have to login to Xcode with your Apple ID to generate a codesigning certificate. You can do this by launching Xcode.app, opening the PrjFS.xcworkspace and trying to build. You can find the signing options in the General tab of the project's settings. +* If you're using Xcode for the first time, you may have to login to Xcode with your Apple ID to generate a codesigning certificate. You can do this by launching Xcode.app, opening the PrjFS.xcworkspace and trying to build. You can find the signing options in the General tab of the project's settings. * Create a `VFSForGit` directory and Clone VFSForGit into a directory called `src` inside it: ``` @@ -104,9 +102,9 @@ Note that VFS for Git on Mac is under active development. shows `/Applications/Xcode.app/Contents/Developer`. If it does not, install Xcode and then launch it (you can close it afterwards.) -* For the time being, only for active development, you will have to disable the SIP (System Integrity Protection) in order to load the kext). +* In order to build VFS for Git on Mac (and PrjFSKext) you will have to disable the SIP (System Integrity Protection) in order to load the kext). - **This is dangerous and very bad for the security of your machine. Do not do this on any production machine! If you no longer need to developer VFS for Git on Mac we recommend re-enabling SIP ASAP.** + **This is dangerous and very bad for the security of your machine. Do not do this on any production machine! If you no longer need to develop VFS for Git on Mac we recommend re-enabling SIP ASAP.** To disable SIP boot into recovery mode (`[Win/⌘] + R` while booting your Mac). Once booted into recovery mode open Utilities -> Terminal to launch a terminal. Enter: From 718b5d0464d47df28e267b52c472f699c2682f1f Mon Sep 17 00:00:00 2001 From: Jameson Miller Date: Thu, 4 Oct 2018 14:25:01 -0400 Subject: [PATCH 228/272] Add original description on NamedPipe protocol --- GVFS/GVFS.Common/NamedPipes/NamedPipeServer.cs | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/GVFS/GVFS.Common/NamedPipes/NamedPipeServer.cs b/GVFS/GVFS.Common/NamedPipes/NamedPipeServer.cs index 83ac690e3..07c2bff4a 100644 --- a/GVFS/GVFS.Common/NamedPipes/NamedPipeServer.cs +++ b/GVFS/GVFS.Common/NamedPipes/NamedPipeServer.cs @@ -6,6 +6,22 @@ namespace GVFS.Common.NamedPipes { + /// + /// The server side of a Named Pipe used for interprocess communication. + /// + /// Named Pipe protocol: + /// The client / server process sends a "message" (or line) of data as a + /// sequence of bytes terminated by a 0x3 byte (ASCII control code for + /// End of text). The sender of a message must wait for a response from + /// the other side before sending another message. Text is encoded as + /// UTF-8 to be sent as bytes across the wire. + /// + /// This format was chosen so that: + /// 1) A reasonable range of values can be transmitted across the pipe, + /// including null and bytes that represent newline characters. + /// 2) It would be easy to implement in multiple places, as we + /// have managed and native implementations. + /// public class NamedPipeServer : IDisposable { // TODO(Mac) the limit is much shorter on macOS From 6f00c207c062c70a1d1fae45104beb6bb5429ad1 Mon Sep 17 00:00:00 2001 From: Derrick Stolee Date: Thu, 4 Oct 2018 11:10:26 -0400 Subject: [PATCH 229/272] SecondCloneSucceedsWithMissingTrees: Attempt 2 The previous approach to this test relied on deleting packfiles from the shared object cache. That failed on Windows because the .idx files have restrictive permissions that cause the delete to fail. Instead, use the loose object downloads to hydrate the commit and root tree for the WellKnownBranch and then do a clone pointing at that branch. This should trigger the failure case. --- .../MultiEnlistmentTests/SharedCacheTests.cs | 19 +++++++++++++++++++ .../TestsWithMultiEnlistment.cs | 11 +++++++++-- .../Tools/GVFSFunctionalTestEnlistment.cs | 8 ++++++-- 3 files changed, 34 insertions(+), 4 deletions(-) diff --git a/GVFS/GVFS.FunctionalTests/Tests/MultiEnlistmentTests/SharedCacheTests.cs b/GVFS/GVFS.FunctionalTests/Tests/MultiEnlistmentTests/SharedCacheTests.cs index 7ec39205a..f4026dccc 100644 --- a/GVFS/GVFS.FunctionalTests/Tests/MultiEnlistmentTests/SharedCacheTests.cs +++ b/GVFS/GVFS.FunctionalTests/Tests/MultiEnlistmentTests/SharedCacheTests.cs @@ -268,6 +268,25 @@ public void MountUsesNewLocalCacheKeyWhenLocalCacheDeleted() this.AlternatesFileShouldHaveGitObjectsRoot(enlistment); } + [TestCase] + public void SecondCloneSucceedsWithMissingTrees() + { + string newCachePath = Path.Combine(this.localCacheParentPath, ".customGvfsCache2"); + GVFSFunctionalTestEnlistment enlistment1 = this.CreateNewEnlistment(localCacheRoot: newCachePath, skipPrefetch: true); + File.ReadAllText(Path.Combine(enlistment1.RepoRoot, WellKnownFile)); + this.AlternatesFileShouldHaveGitObjectsRoot(enlistment1); + + // This Git command loads the commit and root tree for WellKnownCommitSha, + // but does not download any more reachable objects. + string command = "cat-file -p origin/" + WellKnownBranch + "^{tree}"; + ProcessResult result = GitHelpers.InvokeGitAgainstGVFSRepo(enlistment1.RepoRoot, command); + result.ExitCode.ShouldEqual(0, $"git {command} failed with error: " + result.Errors); + + // If we did not properly check the failed checkout at this step, then clone will fail during checkout. + GVFSFunctionalTestEnlistment enlistment2 = this.CreateNewEnlistment(localCacheRoot: newCachePath, branch: WellKnownBranch, skipPrefetch: true); + File.ReadAllText(Path.Combine(enlistment2.RepoRoot, WellKnownFile)); + } + // Override OnTearDownEnlistmentsDeleted rathern than using [TearDown] as the enlistments need to be unmounted before // localCacheParentPath can be deleted (as the SQLite blob sizes database cannot be deleted while GVFS is mounted) protected override void OnTearDownEnlistmentsDeleted() diff --git a/GVFS/GVFS.FunctionalTests/Tests/MultiEnlistmentTests/TestsWithMultiEnlistment.cs b/GVFS/GVFS.FunctionalTests/Tests/MultiEnlistmentTests/TestsWithMultiEnlistment.cs index f0afd9399..3c9a35327 100644 --- a/GVFS/GVFS.FunctionalTests/Tests/MultiEnlistmentTests/TestsWithMultiEnlistment.cs +++ b/GVFS/GVFS.FunctionalTests/Tests/MultiEnlistmentTests/TestsWithMultiEnlistment.cs @@ -28,9 +28,16 @@ protected virtual void OnTearDownEnlistmentsDeleted() { } - protected GVFSFunctionalTestEnlistment CreateNewEnlistment(string localCacheRoot = null, string branch = null) + protected GVFSFunctionalTestEnlistment CreateNewEnlistment( + string localCacheRoot = null, + string branch = null, + bool skipPrefetch = false) { - GVFSFunctionalTestEnlistment output = GVFSFunctionalTestEnlistment.CloneAndMount(GVFSTestConfig.PathToGVFS, branch, localCacheRoot); + GVFSFunctionalTestEnlistment output = GVFSFunctionalTestEnlistment.CloneAndMount( + GVFSTestConfig.PathToGVFS, + branch, + localCacheRoot, + skipPrefetch); this.enlistmentsToDelete.Add(output); return output; } diff --git a/GVFS/GVFS.FunctionalTests/Tools/GVFSFunctionalTestEnlistment.cs b/GVFS/GVFS.FunctionalTests/Tools/GVFSFunctionalTestEnlistment.cs index 427fef7b2..eb6ea827c 100644 --- a/GVFS/GVFS.FunctionalTests/Tools/GVFSFunctionalTestEnlistment.cs +++ b/GVFS/GVFS.FunctionalTests/Tools/GVFSFunctionalTestEnlistment.cs @@ -89,10 +89,14 @@ public static GVFSFunctionalTestEnlistment CloneAndMountWithPerRepoCache(string return CloneAndMount(pathToGvfs, enlistmentRoot, null, localCache, skipPrefetch); } - public static GVFSFunctionalTestEnlistment CloneAndMount(string pathToGvfs, string commitish = null, string localCacheRoot = null) + public static GVFSFunctionalTestEnlistment CloneAndMount( + string pathToGvfs, + string commitish = null, + string localCacheRoot = null, + bool skipPrefetch = false) { string enlistmentRoot = GVFSFunctionalTestEnlistment.GetUniqueEnlistmentRoot(); - return CloneAndMount(pathToGvfs, enlistmentRoot, commitish, localCacheRoot); + return CloneAndMount(pathToGvfs, enlistmentRoot, commitish, localCacheRoot, skipPrefetch); } public static GVFSFunctionalTestEnlistment CloneAndMountEnlistmentWithSpacesInPath(string pathToGvfs, string commitish = null) From 43287ecf8d7f821a2e286fc6c67a2e1eaed2c0c2 Mon Sep 17 00:00:00 2001 From: Derrick Stolee Date: Thu, 4 Oct 2018 13:41:36 -0400 Subject: [PATCH 230/272] SharedCacheTests: Update WellKnownCommitSha This doesn't match, as we expect it to! --- .../Tests/MultiEnlistmentTests/SharedCacheTests.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/GVFS/GVFS.FunctionalTests/Tests/MultiEnlistmentTests/SharedCacheTests.cs b/GVFS/GVFS.FunctionalTests/Tests/MultiEnlistmentTests/SharedCacheTests.cs index f4026dccc..c7b3072f2 100644 --- a/GVFS/GVFS.FunctionalTests/Tests/MultiEnlistmentTests/SharedCacheTests.cs +++ b/GVFS/GVFS.FunctionalTests/Tests/MultiEnlistmentTests/SharedCacheTests.cs @@ -21,7 +21,7 @@ public class SharedCacheTests : TestsWithMultiEnlistment // This branch and commit sha should point to the same place. private const string WellKnownBranch = "FunctionalTests/20170602"; - private const string WellKnownCommitSha = "79dc4233df4d9a7e053662bff95df498f640022e"; + private const string WellKnownCommitSha = "42eb6632beffae26893a3d6e1a9f48d652327c6f"; private string localCachePath; private string localCacheParentPath; From 02d13b7ea0fb456c27573e9949df9879f349280b Mon Sep 17 00:00:00 2001 From: Al Ameen Shah Date: Thu, 4 Oct 2018 11:22:36 -0400 Subject: [PATCH 231/272] - Fixing merge issues with master - Added NoConfig RingType. NoConfig & None rings would be similar. During upgrade no error messaging will be displayed. RingType Invalid would continue to be treated as error. - Rephrased mount error messaging. - Making gvfs config keys case-insensitive - Removed spinner for "Launching upgrade tool" - Show pre-upgrade warning about repos being unmounted & mounted when upgrade is run from non-elevated command prompt as well - Log version info from UpgradeVerb after successfull upgrade check. --- GVFS/GVFS.Common/GVFSConstants.cs | 6 ++- GVFS/GVFS.Common/LocalGVFSConfig.cs | 4 +- GVFS/GVFS.Common/ProductUpgrader.cs | 17 +++++++-- GVFS/GVFS.Upgrader/UpgradeOrchestrator.cs | 16 ++++---- GVFS/GVFS/CommandLine/ConfigVerb.cs | 10 ++--- GVFS/GVFS/CommandLine/GVFSVerb.cs | 2 +- GVFS/GVFS/CommandLine/UpgradeVerb.cs | 46 +++++++---------------- start | 1 - 8 files changed, 45 insertions(+), 57 deletions(-) delete mode 100644 start diff --git a/GVFS/GVFS.Common/GVFSConstants.cs b/GVFS/GVFS.Common/GVFSConstants.cs index 393004b5c..a387f8937 100644 --- a/GVFS/GVFS.Common/GVFSConstants.cs +++ b/GVFS/GVFS.Common/GVFSConstants.cs @@ -229,8 +229,10 @@ public static class UpgradeVerbMessages public const string GVFSUpgrade = "`gvfs upgrade`"; public const string GVFSUpgradeConfirm = "`gvfs upgrade --confirm`"; public const string GVFSUpgradeOptionalConfirm = "`gvfs upgrade [--confirm]`"; - public const string NoneRingConsoleAlert = "Upgrade ring set to \"None\". No upgrade check was performed."; - public const string InvalidRingConsoleAlert = "Upgrade ring is not set. No upgrade check was performed."; + public const string NoUpgradeCheckPerformed = "No upgrade check was performed."; + public const string NoneRingConsoleAlert = "Upgrade ring set to \"None\". " + NoUpgradeCheckPerformed; + public const string NoRingConfigConsoleAlert = "Upgrade ring is not set. " + NoUpgradeCheckPerformed; + public const string InvalidRingConsoleAlert = "Upgrade ring set to unknown value. " + NoUpgradeCheckPerformed; public const string SetUpgradeRingCommand = "To set or change upgrade ring, run `gvfs config " + LocalGVFSConfig.UpgradeRing + " [\"Fast\"|\"Slow\"|\"None\"]` from a command prompt."; public const string ReminderNotification = "A new version of GVFS is available. Run " + UpgradeVerbMessages.GVFSUpgrade + " to start the upgrade."; public const string UnmountRepoWarning = "Upgrade will unmount and remount gvfs repos, ensure you are at a stopping point."; diff --git a/GVFS/GVFS.Common/LocalGVFSConfig.cs b/GVFS/GVFS.Common/LocalGVFSConfig.cs index 722b28081..980d5956a 100644 --- a/GVFS/GVFS.Common/LocalGVFSConfig.cs +++ b/GVFS/GVFS.Common/LocalGVFSConfig.cs @@ -35,7 +35,7 @@ public bool TryGetConfig( try { - this.allSettings.TryGetValue(key, out value); + this.allSettings.TryGetValue(key.ToUpper(), out value); error = null; return true; } @@ -67,7 +67,7 @@ public bool TrySetConfig( try { - this.allSettings.SetValueAndFlush(key, value); + this.allSettings.SetValueAndFlush(key.ToUpper(), value); error = null; return true; } diff --git a/GVFS/GVFS.Common/ProductUpgrader.cs b/GVFS/GVFS.Common/ProductUpgrader.cs index 33a584fe8..636278ea8 100644 --- a/GVFS/GVFS.Common/ProductUpgrader.cs +++ b/GVFS/GVFS.Common/ProductUpgrader.cs @@ -59,12 +59,16 @@ public ProductUpgrader( public enum RingType { - // The values here should be ascending. + // The values here should be ascending. + // Invalid - User has set an incorrect ring + // NoConfig - User has Not set any ring yet + // None - User has set a valid "None" ring // (Fast should be greater than Slow, // Slow should be greater than None, None greater than Invalid.) // This is required for the correct implementation of Ring based // upgrade logic. Invalid = 0, + NoConfig = None - 1, None = 10, Slow = None + 1, Fast = Slow + 1, @@ -281,13 +285,18 @@ public virtual bool TryLoadRingConfig(out string error) error = null; return true; } - else + + if (string.IsNullOrEmpty(ringConfig)) { - error = "Invalid upgrade ring `" + ringConfig + "` specified in gvfs config."; + this.Ring = RingType.NoConfig; + error = null; + return true; } + + error = "Invalid upgrade ring `" + ringConfig + "` specified in gvfs config." + Environment.NewLine; } - error += Environment.NewLine + GVFSConstants.UpgradeVerbMessages.SetUpgradeRingCommand; + error += GVFSConstants.UpgradeVerbMessages.SetUpgradeRingCommand; this.Ring = RingType.Invalid; return false; } diff --git a/GVFS/GVFS.Upgrader/UpgradeOrchestrator.cs b/GVFS/GVFS.Upgrader/UpgradeOrchestrator.cs index df18ceb2d..66a729269 100644 --- a/GVFS/GVFS.Upgrader/UpgradeOrchestrator.cs +++ b/GVFS/GVFS.Upgrader/UpgradeOrchestrator.cs @@ -62,17 +62,17 @@ public void Execute() ProductUpgrader.RingType ring = ProductUpgrader.RingType.Invalid; string mountError = null; - if (!this.TryLoadUpgradeRing(out ring, out error) || ring == ProductUpgrader.RingType.None) + if (!this.TryLoadUpgradeRing(out ring, out error)) + { + this.output.WriteLine(GVFSConstants.UpgradeVerbMessages.InvalidRingConsoleAlert); + } + else if (ring == ProductUpgrader.RingType.None || ring == ProductUpgrader.RingType.NoConfig) { string message = ring == ProductUpgrader.RingType.None ? GVFSConstants.UpgradeVerbMessages.NoneRingConsoleAlert : - GVFSConstants.UpgradeVerbMessages.InvalidRingConsoleAlert; + GVFSConstants.UpgradeVerbMessages.NoRingConfigConsoleAlert; this.output.WriteLine(message); - - if (ring == ProductUpgrader.RingType.None) - { - this.output.WriteLine(GVFSConstants.UpgradeVerbMessages.SetUpgradeRingCommand); - } + this.output.WriteLine(GVFSConstants.UpgradeVerbMessages.SetUpgradeRingCommand); } else { @@ -103,7 +103,7 @@ public void Execute() } else { - this.output.WriteLine($"{Environment.NewLine}{(string.IsNullOrEmpty(mountError) ? "U" : "Repository mount failed. But u")}pgrade completed successfully!"); + this.output.WriteLine($"{Environment.NewLine}Upgrade completed successfully{(string.IsNullOrEmpty(mountError) ? "." : ", but one or more repositories will need to be mounted manually.")}"); } if (this.input == Console.In) diff --git a/GVFS/GVFS/CommandLine/ConfigVerb.cs b/GVFS/GVFS/CommandLine/ConfigVerb.cs index c978888da..6c64eb3d4 100644 --- a/GVFS/GVFS/CommandLine/ConfigVerb.cs +++ b/GVFS/GVFS/CommandLine/ConfigVerb.cs @@ -13,17 +13,15 @@ public class ConfigVerb : GVFSVerb.ForNoEnlistment [Value( 0, Required = true, - Default = "", - MetaName = "GVFS config setting name", - HelpText = "Name of GVFS config setting that is to be set or read")] + MetaName = "Setting name", + HelpText = "Name of setting that is to be set or read")] public string Key { get; set; } [Value( 1, Required = false, - Default = "", - MetaName = "GVFS config setting value", - HelpText = "Value of GVFS config setting to be set")] + MetaName = "Setting value", + HelpText = "Value of setting to be set")] public string Value { get; set; } protected override string VerbName diff --git a/GVFS/GVFS/CommandLine/GVFSVerb.cs b/GVFS/GVFS/CommandLine/GVFSVerb.cs index d38a28ed4..56b2eca61 100644 --- a/GVFS/GVFS/CommandLine/GVFSVerb.cs +++ b/GVFS/GVFS/CommandLine/GVFSVerb.cs @@ -260,7 +260,7 @@ protected ServerGVFSConfig QueryGVFSConfig(ITracer tracer, GVFSEnlistment enlist { this.ReportErrorAndExit(tracer, "Unable to query /gvfs/config"); } - + return serverGVFSConfig; } diff --git a/GVFS/GVFS/CommandLine/UpgradeVerb.cs b/GVFS/GVFS/CommandLine/UpgradeVerb.cs index edd1af8c3..b26cd1e66 100644 --- a/GVFS/GVFS/CommandLine/UpgradeVerb.cs +++ b/GVFS/GVFS/CommandLine/UpgradeVerb.cs @@ -112,10 +112,10 @@ private bool TryRunProductUpgrade() return false; } - if (ring == ProductUpgrader.RingType.None) + if (ring == ProductUpgrader.RingType.None || ring == ProductUpgrader.RingType.NoConfig) { this.tracer.RelatedInfo($"{nameof(this.TryRunProductUpgrade)}: {GVFSConstants.UpgradeVerbMessages.NoneRingConsoleAlert}"); - this.ReportInfoToConsole(GVFSConstants.UpgradeVerbMessages.NoneRingConsoleAlert); + this.ReportInfoToConsole(ring == ProductUpgrader.RingType.None ? GVFSConstants.UpgradeVerbMessages.NoneRingConsoleAlert : GVFSConstants.UpgradeVerbMessages.NoRingConfigConsoleAlert); this.ReportInfoToConsole(GVFSConstants.UpgradeVerbMessages.SetUpgradeRingCommand); return true; } @@ -154,23 +154,11 @@ private bool TryRunProductUpgrade() } else { - if (isInstallable) - { - string message = string.Join( - Environment.NewLine, + string message = string.Join( + Environment.NewLine, GVFSConstants.UpgradeVerbMessages.UnmountRepoWarning, GVFSConstants.UpgradeVerbMessages.UpgradeInstallAdvice); - this.ReportInfoToConsole(upgradeAvailableMessage + Environment.NewLine + Environment.NewLine + message + Environment.NewLine); - } - else - { - this.ReportInfoToConsole($"{Environment.NewLine}{upgradeAvailableMessage}"); - - if (!string.IsNullOrEmpty(cannotInstallReason)) - { - this.ReportInfoToConsole($"{Environment.NewLine}{cannotInstallReason}"); - } - } + this.ReportInfoToConsole(upgradeAvailableMessage + Environment.NewLine + Environment.NewLine + message + Environment.NewLine); } return true; @@ -221,23 +209,15 @@ private bool TryRunInstaller(out string consoleError) string upgraderPath = null; string errorMessage = null; - bool preUpgradeSuccess = this.ShowStatusWhileRunning( - () => - { - if (this.TryCopyUpgradeTool(out upgraderPath, out errorMessage) && - this.TryLaunchUpgradeTool(upgraderPath, out errorMessage)) - { - return true; - } + this.ReportInfoToConsole("Launching upgrade tool..."); - return false; - }, - "Launching upgrade tool", - suppressGvfsLogMessage: true); - - if (!preUpgradeSuccess) + if (!this.TryCopyUpgradeTool(out upgraderPath, out consoleError)) + { + return false; + } + + if (!this.TryLaunchUpgradeTool(upgraderPath, out errorMessage)) { - consoleError = errorMessage; return false; } @@ -310,7 +290,7 @@ private bool TryCheckUpgradeAvailable( latestVersion = version; - activity.RelatedInfo("Successfully checked server for GVFS upgrades."); + activity.RelatedInfo($"Successfully checked server for GVFS upgrades. New version available {latestVersion}"); } return true; diff --git a/start b/start deleted file mode 100644 index c342dc54f..000000000 --- a/start +++ /dev/null @@ -1 +0,0 @@ -prjflt From 5dc0e3517446bcf9e2bf624ece6cf5378740919d Mon Sep 17 00:00:00 2001 From: William Baker Date: Thu, 4 Oct 2018 14:27:43 -0700 Subject: [PATCH 232/272] Remove autogenerated Xcode comment --- ProjFS.Mac/PrjFSKext/public/FsidInode.h | 8 -------- 1 file changed, 8 deletions(-) diff --git a/ProjFS.Mac/PrjFSKext/public/FsidInode.h b/ProjFS.Mac/PrjFSKext/public/FsidInode.h index caf2f7715..d05bb2646 100644 --- a/ProjFS.Mac/PrjFSKext/public/FsidInode.h +++ b/ProjFS.Mac/PrjFSKext/public/FsidInode.h @@ -1,11 +1,3 @@ -// -// FsidInode.h -// PrjFSKext -// -// Created by William Baker on 10/4/18. -// Copyright © 2018 GVFS. All rights reserved. -// - #ifndef FsidInode_h #define FsidInode_h From 253c9f5c7b58cf606ca66aead2ac55fdafdad348 Mon Sep 17 00:00:00 2001 From: William Baker Date: Thu, 4 Oct 2018 14:45:51 -0700 Subject: [PATCH 233/272] Mac: Update MoveAndOverwriteFile to use native API --- GVFS/GVFS.Common/NativeMethods.Shared.cs | 10 +++++----- GVFS/GVFS.Platform.Mac/MacFileSystem.cs | 13 +++++++------ 2 files changed, 12 insertions(+), 11 deletions(-) diff --git a/GVFS/GVFS.Common/NativeMethods.Shared.cs b/GVFS/GVFS.Common/NativeMethods.Shared.cs index e7ed7f286..7a796b15e 100644 --- a/GVFS/GVFS.Common/NativeMethods.Shared.cs +++ b/GVFS/GVFS.Common/NativeMethods.Shared.cs @@ -143,6 +143,11 @@ public static string GetFinalPathName(string path) } } + public static void ThrowLastWin32Exception(string message) + { + throw new Win32Exception(Marshal.GetLastWin32Error(), message); + } + [DllImport("kernel32.dll", SetLastError = true)] public static extern SafeFileHandle OpenProcess( ProcessAccessFlags processAccess, @@ -153,11 +158,6 @@ public static extern SafeFileHandle OpenProcess( [return: MarshalAs(UnmanagedType.Bool)] public static extern bool GetExitCodeProcess(SafeFileHandle hProcess, out uint lpExitCode); - private static void ThrowLastWin32Exception(string message) - { - throw new Win32Exception(Marshal.GetLastWin32Error(), message); - } - [DllImport("kernel32.dll", SetLastError = true, CharSet = CharSet.Unicode)] private static extern SafeFileHandle CreateFile( [In] string lpFileName, diff --git a/GVFS/GVFS.Platform.Mac/MacFileSystem.cs b/GVFS/GVFS.Platform.Mac/MacFileSystem.cs index 4e72868a9..96dec6a59 100644 --- a/GVFS/GVFS.Platform.Mac/MacFileSystem.cs +++ b/GVFS/GVFS.Platform.Mac/MacFileSystem.cs @@ -1,4 +1,5 @@ -using GVFS.Common.FileSystem; +using GVFS.Common; +using GVFS.Common.FileSystem; using System.IO; using System.Runtime.InteropServices; @@ -15,13 +16,10 @@ public void FlushFileBuffers(string path) public void MoveAndOverwriteFile(string sourceFileName, string destinationFilename) { - // TODO(Mac): Use native API - if (File.Exists(destinationFilename)) + if (Rename(sourceFileName, destinationFilename) != 0) { - File.Delete(destinationFilename); + NativeMethods.ThrowLastWin32Exception($"Failed to renname {sourceFileName} to {destinationFilename}"); } - - File.Move(sourceFileName, destinationFilename); } public void CreateHardLink(string newFileName, string existingFileName) @@ -42,5 +40,8 @@ public bool TryGetNormalizedPath(string path, out string normalizedPath, out str [DllImport("libc", EntryPoint = "chmod", SetLastError = true)] private static extern int Chmod(string pathname, int mode); + + [DllImport("libc", EntryPoint = "rename", SetLastError = true)] + private static extern int Rename(string oldPath, string newPath); } } From 7aebab2206759401ab72b6adefeeb91ffb491db5 Mon Sep 17 00:00:00 2001 From: Al Ameen Shah Date: Thu, 4 Oct 2018 18:02:14 -0400 Subject: [PATCH 234/272] - Fixing UT failure. Launching upgrade tool was moved out of the spinner, UT was still expecting the Succeeded message. --- .../GVFS.UnitTests.Windows/Windows/Upgrader/UpgradeVerbTests.cs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/GVFS/GVFS.UnitTests.Windows/Windows/Upgrader/UpgradeVerbTests.cs b/GVFS/GVFS.UnitTests.Windows/Windows/Upgrader/UpgradeVerbTests.cs index f5e560465..6edb15419 100644 --- a/GVFS/GVFS.UnitTests.Windows/Windows/Upgrader/UpgradeVerbTests.cs +++ b/GVFS/GVFS.UnitTests.Windows/Windows/Upgrader/UpgradeVerbTests.cs @@ -81,7 +81,7 @@ public void LaunchInstaller() expectedOutput: new List { "New GVFS version " + NewerThanLocalVersion + " available in ring Slow", - "Launching upgrade tool...Succeeded" + "Launching upgrade tool..." }, expectedErrors:null); From df5ea192dbafe0c2f27f9b4eddc8908570229760 Mon Sep 17 00:00:00 2001 From: Jameson Miller Date: Thu, 4 Oct 2018 14:00:40 -0400 Subject: [PATCH 235/272] Simplify NamedPipeStreamReader logic The existing NamedPipe protocol expected that the sender of a message would wait for a response from the receiver before sending another message. It did not allow for multiple messages to be sent without the client responding. However, this is not a correct assumption, as there is at least 1 place where a component will send multiple messages without waiting for a response from the receiver (#333). This change updates the NamedPipeStreamReader to handle multiple messages sent sequentially, and simplifies the overall logic in the process. Tests are also included which exhibit this behavior. --- .../GVFS.Common/NamedPipes/NamedPipeServer.cs | 4 +- .../NamedPipes/NamedPipeStreamReader.cs | 63 +++++++------------ .../NamedPipeStreamReaderWriterTests.cs | 61 +++++++++--------- 3 files changed, 56 insertions(+), 72 deletions(-) diff --git a/GVFS/GVFS.Common/NamedPipes/NamedPipeServer.cs b/GVFS/GVFS.Common/NamedPipes/NamedPipeServer.cs index 07c2bff4a..5f93c602d 100644 --- a/GVFS/GVFS.Common/NamedPipes/NamedPipeServer.cs +++ b/GVFS/GVFS.Common/NamedPipes/NamedPipeServer.cs @@ -12,9 +12,7 @@ namespace GVFS.Common.NamedPipes /// Named Pipe protocol: /// The client / server process sends a "message" (or line) of data as a /// sequence of bytes terminated by a 0x3 byte (ASCII control code for - /// End of text). The sender of a message must wait for a response from - /// the other side before sending another message. Text is encoded as - /// UTF-8 to be sent as bytes across the wire. + /// End of text). Text is encoded as UTF-8 to be sent as bytes across the wire. /// /// This format was chosen so that: /// 1) A reasonable range of values can be transmitted across the pipe, diff --git a/GVFS/GVFS.Common/NamedPipes/NamedPipeStreamReader.cs b/GVFS/GVFS.Common/NamedPipes/NamedPipeStreamReader.cs index 17ced2e20..63e8f5f69 100644 --- a/GVFS/GVFS.Common/NamedPipes/NamedPipeStreamReader.cs +++ b/GVFS/GVFS.Common/NamedPipes/NamedPipeStreamReader.cs @@ -11,23 +11,16 @@ namespace GVFS.Common.NamedPipes /// public class NamedPipeStreamReader { - private const int DefaultBufferSize = 1024; + private const int InitialListSize = 1024; private const byte TerminatorByte = 0x3; + private readonly byte[] buffer; - private int bufferSize; - private byte[] buffer; private Stream stream; - public NamedPipeStreamReader(Stream stream, int bufferSize) - { - this.stream = stream; - this.bufferSize = bufferSize; - this.buffer = new byte[this.bufferSize]; - } - public NamedPipeStreamReader(Stream stream) - : this(stream, DefaultBufferSize) { + this.stream = stream; + this.buffer = new byte[1]; } /// @@ -36,38 +29,23 @@ public NamedPipeStreamReader(Stream stream) /// The message read from the stream, or null if the end of the input stream has been reached. public string ReadMessage() { - int bytesRead = this.Read(); - if (bytesRead == 0) + byte currentByte; + + bool streamOpen = this.TryReadByte(out currentByte); + if (!streamOpen) { // The end of the stream has been reached - return null to indicate this. return null; } - // If we have read in the entire message in the first read (mainline scenario), - // then just process the data directly from the buffer. - if (this.buffer[bytesRead - 1] == TerminatorByte) - { - return Encoding.UTF8.GetString(this.buffer, 0, bytesRead - 1); - } + List bytes = new List(InitialListSize); - // We need to process multiple chunks - collect data from multiple chunks - // into a single list - List bytes = new List(this.bufferSize * 2); - - while (true) + do { - bool encounteredTerminatorByte = this.buffer[bytesRead - 1] == TerminatorByte; - int lengthToCopy = encounteredTerminatorByte ? bytesRead - 1 : bytesRead; + bytes.Add(currentByte); + streamOpen = this.TryReadByte(out currentByte); - bytes.AddRange(new ArraySegment(this.buffer, 0, lengthToCopy)); - if (encounteredTerminatorByte) - { - break; - } - - bytesRead = this.Read(); - - if (bytesRead == 0) + if (!streamOpen) { // We have read a partial message (the last byte received does not indicate that // this was the end of the message), but the stream has been closed. Throw an exception @@ -76,17 +54,24 @@ public string ReadMessage() throw new IOException("Incomplete message read from stream. The end of the stream was reached without the expected terminating byte."); } } + while (currentByte != TerminatorByte); return Encoding.UTF8.GetString(bytes.ToArray()); } /// - /// Read the next chunk of bytes from the stream. + /// Read a byte from the stream. /// - /// The number of bytes read. - private int Read() + /// The byte read from the stream + /// True if byte read, false if end of stream has been reached + private bool TryReadByte(out byte readByte) { - return this.stream.Read(this.buffer, 0, this.buffer.Length); + this.buffer[0] = 0; + + int numBytesRead = this.stream.Read(this.buffer, 0, 1); + readByte = this.buffer[0]; + + return numBytesRead == 1; } } } diff --git a/GVFS/GVFS.UnitTests/Common/NamedPipeStreamReaderWriterTests.cs b/GVFS/GVFS.UnitTests/Common/NamedPipeStreamReaderWriterTests.cs index 7e630fea6..36acb1d37 100644 --- a/GVFS/GVFS.UnitTests/Common/NamedPipeStreamReaderWriterTests.cs +++ b/GVFS/GVFS.UnitTests/Common/NamedPipeStreamReaderWriterTests.cs @@ -9,8 +9,6 @@ namespace GVFS.UnitTests.Common [TestFixture] public class NamedPipeStreamReaderWriterTests { - private const int BufferSize = 256; - private MemoryStream stream; private NamedPipeStreamWriter streamWriter; private NamedPipeStreamReader streamReader; @@ -20,11 +18,10 @@ public void Setup() { this.stream = new MemoryStream(); this.streamWriter = new NamedPipeStreamWriter(this.stream); - this.streamReader = new NamedPipeStreamReader(this.stream, BufferSize); + this.streamReader = new NamedPipeStreamReader(this.stream); } [Test] - [Description("Verify that we can transmit multiple messages")] public void CanWriteAndReadMessages() { string firstMessage = @"This is a new message"; @@ -41,24 +38,6 @@ public void CanWriteAndReadMessages() } [Test] - [Description("Verify that we can transmit a message that contains content that is the size of a NamedPipeStreamReader's buffer")] - public void CanSendBufferSizedContent() - { - string longMessage = new string('T', BufferSize); - this.TestTransmitMessage(longMessage); - } - - [Test] - [Description("Verify that we can transmit message that is the same size a NamedPipeStreamReader's buffer")] - public void CanSendBufferSizedMessage() - { - int numBytesInMessageTerminator = 1; - string longMessage = new string('T', BufferSize - numBytesInMessageTerminator); - this.TestTransmitMessage(longMessage); - } - - [Test] - [Description("Verify that the expected exception is thrown if message is not terminated with expected byte.")] [Category(CategoryConstants.ExceptionExpected)] public void ReadingPartialMessgeThrows() { @@ -71,19 +50,23 @@ public void ReadingPartialMessgeThrows() } [Test] - [Description("Verify that we can transmit message that is larger than the buffer")] - public void CanSendMultiBufferSizedMessage() + public void CanSendMessagesWithNewLines() { - string longMessage = new string('T', BufferSize * 3); - this.TestTransmitMessage(longMessage); + string messageWithNewLines = "This is a \nstringwith\nnewlines"; + this.TestTransmitMessage(messageWithNewLines); } [Test] - [Description("Verify that we can transmit message that newline characters")] - public void CanSendNewLines() + public void CanSendMultipleMessagesSequentially() { - string messageWithNewLines = "This is a \nstringwith\nnewlines"; - this.TestTransmitMessage(messageWithNewLines); + string[] messages = new string[] + { + "This is a new message", + "This is another message", + "This is the third message in a series of messages" + }; + + this.TestTransmitMessages(messages); } private void TestTransmitMessage(string message) @@ -96,6 +79,24 @@ private void TestTransmitMessage(string message) string readMessage = this.streamReader.ReadMessage(); readMessage.ShouldEqual(message, "The message read from the stream reader is not the same as the message that was sent."); } + + private void TestTransmitMessages(string[] messages) + { + long pos = this.ReadStreamPosition(); + + foreach (string message in messages) + { + this.streamWriter.WriteMessage(message); + } + + this.SetStreamPosition(pos); + + foreach (string message in messages) + { + string readMessage = this.streamReader.ReadMessage(); + readMessage.ShouldEqual(message, "The message read from the stream reader is not the same as the message that was sent."); + } + } private long ReadStreamPosition() { From 755dda6ff536e3a3471f0e8d1781edf40a40df80 Mon Sep 17 00:00:00 2001 From: Phil Dennis-Jordan Date: Mon, 1 Oct 2018 16:51:55 +0200 Subject: [PATCH 236/272] Mac kext: Handle named streams MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Occasionally, the vnode auth handler is called on a vnode representing a named stream/named fork. (Typically, the resource fork.) Forks don’t have their own xattrs, etc. so some of the logic we apply fails. We don’t care specifically about named forks, so treat any access to them as an access to the main file instead. Note: we need to balance the vnode_getparent() call with vnode_put() --- ProjFS.Mac/PrjFSKext/PrjFSKext/KauthHandler.cpp | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/ProjFS.Mac/PrjFSKext/PrjFSKext/KauthHandler.cpp b/ProjFS.Mac/PrjFSKext/PrjFSKext/KauthHandler.cpp index a0ed82e25..b0729babd 100644 --- a/ProjFS.Mac/PrjFSKext/PrjFSKext/KauthHandler.cpp +++ b/ProjFS.Mac/PrjFSKext/PrjFSKext/KauthHandler.cpp @@ -260,6 +260,18 @@ static int HandleVnodeOperation( // arg2 is the (vnode_t) parent vnode int* kauthError = reinterpret_cast(arg3); int kauthResult = KAUTH_RESULT_DEFER; + bool putVnodeWhenDone = false; + + // A lot of our file checks such as attribute tests behave oddly if the vnode + // refers to a named fork/stream; apply the logic as if the vnode operation was + // occurring on the file itself. (/path/to/file/..namedfork/rsrc) + if (vnode_isnamedstream(currentVnode)) + { + vnode_t mainFileFork = vnode_getparent(currentVnode); + assert(NULLVP != mainFileFork); + currentVnode = mainFileFork; + putVnodeWhenDone = true; + } const char* vnodePath = nullptr; char vnodePathBuffer[PrjFSMaxPath]; @@ -384,6 +396,11 @@ static int HandleVnodeOperation( } CleanupAndReturn: + if (putVnodeWhenDone) + { + vnode_put(currentVnode); + } + atomic_fetch_sub(&s_numActiveKauthEvents, 1); return kauthResult; } From f147f8c02634c9d73299c4cac461783dfae1745a Mon Sep 17 00:00:00 2001 From: Phil Dennis-Jordan Date: Fri, 28 Sep 2018 13:04:06 +0200 Subject: [PATCH 237/272] Mac kext: Replaces some C-style casts with C++ style. --- ProjFS.Mac/PrjFSKext/PrjFSKext/KauthHandler.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/ProjFS.Mac/PrjFSKext/PrjFSKext/KauthHandler.cpp b/ProjFS.Mac/PrjFSKext/PrjFSKext/KauthHandler.cpp index b0729babd..c78267278 100644 --- a/ProjFS.Mac/PrjFSKext/PrjFSKext/KauthHandler.cpp +++ b/ProjFS.Mac/PrjFSKext/PrjFSKext/KauthHandler.cpp @@ -425,7 +425,7 @@ static int HandleFileOpOperation( KAUTH_FILEOP_LINK == action) { // arg0 is the (const char *) fromPath (or the file being linked to) - const char* newPath = (const char*)arg1; + const char* newPath = reinterpret_cast(arg1); // TODO(Mac): We need to handle failures to lookup the vnode. If we fail to lookup the vnode // it's possible that we'll miss notifications @@ -481,7 +481,7 @@ static int HandleFileOpOperation( else if (KAUTH_FILEOP_CLOSE == action) { vnode_t currentVnode = reinterpret_cast(arg0); - const char* path = (const char*)arg1; + const char* path = reinterpret_cast(arg1); int closeFlags = static_cast(arg2); if (vnode_isdir(currentVnode)) From d21d165d15db312ee4b83a2fb3c8cf4d3f6707e4 Mon Sep 17 00:00:00 2001 From: Phil Dennis-Jordan Date: Mon, 1 Oct 2018 16:42:08 +0200 Subject: [PATCH 238/272] Mac kext: Enables warning for shadowed variables. Shadowing variables frequently is a source of bugs, so this enables the corresponding compiler warning. --- ProjFS.Mac/PrjFSKext/PrjFSKext.xcodeproj/project.pbxproj | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/ProjFS.Mac/PrjFSKext/PrjFSKext.xcodeproj/project.pbxproj b/ProjFS.Mac/PrjFSKext/PrjFSKext.xcodeproj/project.pbxproj index 74700506e..dbd7e2453 100644 --- a/ProjFS.Mac/PrjFSKext/PrjFSKext.xcodeproj/project.pbxproj +++ b/ProjFS.Mac/PrjFSKext/PrjFSKext.xcodeproj/project.pbxproj @@ -408,6 +408,7 @@ "$(inherited)", "MACH_ASSERT=1", ); + GCC_WARN_SHADOW = YES; INFOPLIST_FILE = PrjFSKext/Info.plist; MODULE_NAME = io.gvfs.PrjFSKext; MODULE_START = PrjFSKext_Start; @@ -430,7 +431,11 @@ buildSettings = { CODE_SIGN_STYLE = Automatic; DEVELOPMENT_TEAM = UBF8T346G9; - GCC_PREPROCESSOR_DEFINITIONS = "MACH_ASSERT=1"; + GCC_PREPROCESSOR_DEFINITIONS = ( + "MACH_ASSERT=1", + "$(inherited)", + ); + GCC_WARN_SHADOW = YES; INFOPLIST_FILE = PrjFSKext/Info.plist; MODULE_NAME = io.gvfs.PrjFSKext; MODULE_START = PrjFSKext_Start; From e50e488ede2723162593b9a316a83915d29b1dae Mon Sep 17 00:00:00 2001 From: Phil Dennis-Jordan Date: Mon, 1 Oct 2018 13:52:41 +0200 Subject: [PATCH 239/272] Mac kext: Adds typed array memory allocation function. --- ProjFS.Mac/PrjFSKext/PrjFSKext/Memory.hpp | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/ProjFS.Mac/PrjFSKext/PrjFSKext/Memory.hpp b/ProjFS.Mac/PrjFSKext/PrjFSKext/Memory.hpp index eddfdf0fc..0749a6553 100644 --- a/ProjFS.Mac/PrjFSKext/PrjFSKext/Memory.hpp +++ b/ProjFS.Mac/PrjFSKext/PrjFSKext/Memory.hpp @@ -7,4 +7,17 @@ kern_return_t Memory_Cleanup(); void* Memory_Alloc(uint32_t size); void Memory_Free(void* buffer, uint32_t size); +template +T* Memory_AllocArray(uint32_t arrayLength) +{ + size_t allocBytes = arrayLength * sizeof(T); + if (allocBytes > UINT32_MAX) + { + return nullptr; + } + + return static_cast(Memory_Alloc(static_cast(allocBytes))); +} + + #endif /* Memory_h */ From 3211ed1a87322fdf3173fc7a2c02527853f42674 Mon Sep 17 00:00:00 2001 From: Phil Dennis-Jordan Date: Fri, 28 Sep 2018 13:03:57 +0200 Subject: [PATCH 240/272] Mac kext: Use only VirtualizationRoot handles, not direct pointers This change moves the VirtualizationRoot structure definition out of the header file, and unifies the API to only use integer handles, which internally are array indices for non-negative values, and negative values from an enum of special cases. Where indices were previously used outside the VirtualizationRoot implementation functions, the terminology has been changed to "handles." An immediate advantage of the move away from pointers is an improvement in the ability to control thread safety of the root structures; additionally, unlike pointers, handles remain valid outside of held locks, so in a follow-on change, we can reallocate the array for resizing. --- .../PrjFSKext/PrjFSKext/KauthHandler.cpp | 70 ++++---- .../PrjFSKext/PrjFSProviderUserClient.cpp | 20 ++- .../PrjFSKext/PrjFSProviderUserClient.hpp | 5 +- .../PrjFSKext/VirtualizationRoots.cpp | 155 ++++++++++++++---- .../PrjFSKext/VirtualizationRoots.hpp | 43 +++-- 5 files changed, 186 insertions(+), 107 deletions(-) diff --git a/ProjFS.Mac/PrjFSKext/PrjFSKext/KauthHandler.cpp b/ProjFS.Mac/PrjFSKext/PrjFSKext/KauthHandler.cpp index c78267278..111c9355d 100644 --- a/ProjFS.Mac/PrjFSKext/PrjFSKext/KauthHandler.cpp +++ b/ProjFS.Mac/PrjFSKext/PrjFSKext/KauthHandler.cpp @@ -42,11 +42,9 @@ static inline bool ActionBitIsSet(kauth_action_t action, kauth_action_t mask); static bool IsFileSystemCrawler(char* procname); -static const char* GetRelativePath(const char* path, const char* root); - static void Sleep(int seconds, void* channel); static bool TrySendRequestAndWaitForResponse( - const VirtualizationRoot* root, + VirtualizationRootHandle root, MessageType messageType, const vnode_t vnode, const FsidInode& vnodeFsidInode, @@ -65,7 +63,7 @@ static bool ShouldHandleVnodeOpEvent( kauth_action_t action, // Out params: - VirtualizationRoot** root, + VirtualizationRootHandle* root, vtype* vnodeType, uint32_t* vnodeFileFlags, FsidInode* vnodeFsidInode, @@ -80,7 +78,7 @@ static bool ShouldHandleFileOpEvent( kauth_action_t action, // Out params: - VirtualizationRoot** root, + VirtualizationRootHandle* root, FsidInode* vnodeFsidInode, int* pid); @@ -277,7 +275,7 @@ static int HandleVnodeOperation( char vnodePathBuffer[PrjFSMaxPath]; int vnodePathLength = PrjFSMaxPath; - VirtualizationRoot* root = nullptr; + VirtualizationRootHandle root = RootHandle_None; vtype vnodeType; uint32_t currentVnodeFileFlags; FsidInode vnodeFsidInode; @@ -435,7 +433,7 @@ static int HandleFileOpOperation( goto CleanupAndReturn; } - VirtualizationRoot* root = nullptr; + VirtualizationRootHandle root = RootHandle_None; FsidInode vnodeFsidInode; int pid; if (!ShouldHandleFileOpEvent( @@ -495,7 +493,7 @@ static int HandleFileOpOperation( goto CleanupAndReturn; } - VirtualizationRoot* root = nullptr; + VirtualizationRootHandle root = RootHandle_None; FsidInode vnodeFsidInode; int pid; if (!ShouldHandleFileOpEvent( @@ -571,7 +569,7 @@ static bool ShouldHandleVnodeOpEvent( kauth_action_t action, // Out params: - VirtualizationRoot** root, + VirtualizationRootHandle* root, vtype* vnodeType, uint32_t* vnodeFileFlags, FsidInode* vnodeFsidInode, @@ -579,8 +577,8 @@ static bool ShouldHandleVnodeOpEvent( char procname[MAXCOMLEN + 1], int* kauthResult) { - *root = nullptr; *kauthResult = KAUTH_RESULT_DEFER; + *root = RootHandle_None; if (!VirtualizationRoot_VnodeIsOnAllowedFilesystem(vnode)) { @@ -626,19 +624,22 @@ static bool ShouldHandleVnodeOpEvent( } *vnodeFsidInode = Vnode_GetFsidAndInode(vnode, context); - *root = VirtualizationRoots_FindForVnode(vnode, *vnodeFsidInode); + *root = VirtualizationRoot_FindForVnode(vnode, *vnodeFsidInode); - if (nullptr == *root) + if (RootHandle_ProviderTemporaryDirectory == *root) + { + *kauthResult = KAUTH_RESULT_DEFER; + return false; + } + else if (RootHandle_None == *root) { KextLog_FileNote(vnode, "No virtualization root found for file with set flag."); *kauthResult = KAUTH_RESULT_DEFER; return false; } - else if (nullptr == (*root)->providerUserClient) + else if (!VirtualizationRoot_IsOnline(*root)) { - // There is no registered provider for this root - // TODO(Mac): Protect files in the worktree from modification (and prevent // the creation of new files) when the provider is offline @@ -647,7 +648,7 @@ static bool ShouldHandleVnodeOpEvent( } // If the calling process is the provider, we must exit right away to avoid deadlocks - if (*pid == (*root)->providerPid) + if (VirtualizationRoot_PIDMatchesProvider(*root, *pid)) { *kauthResult = KAUTH_RESULT_DEFER; return false; @@ -663,10 +664,12 @@ static bool ShouldHandleFileOpEvent( kauth_action_t action, // Out params: - VirtualizationRoot** root, + VirtualizationRootHandle* root, FsidInode* vnodeFsidInode, int* pid) { + *root = RootHandle_None; + vtype vnodeType = vnode_vtype(vnode); if (ShouldIgnoreVnodeType(vnodeType, vnode)) { @@ -674,21 +677,17 @@ static bool ShouldHandleFileOpEvent( } *vnodeFsidInode = Vnode_GetFsidAndInode(vnode, context); - *root = VirtualizationRoots_FindForVnode(vnode, *vnodeFsidInode); - if (nullptr == *root) - { - return false; - } - else if (nullptr == (*root)->providerUserClient) + *root = VirtualizationRoot_FindForVnode(vnode, *vnodeFsidInode); + if (!VirtualizationRoot_IsValidRootHandle(*root)) { - // There is no registered provider for this root + // This VNode is not part of a root return false; } - // If the calling process is the provider, we must exit right away to avoid deadlocks *pid = GetPid(context); - if (*pid == (*root)->providerPid) + if (VirtualizationRoot_PIDMatchesProvider(*root, *pid)) { + // If the calling process is the provider, we must exit right away to avoid deadlocks return false; } @@ -696,7 +695,7 @@ static bool ShouldHandleFileOpEvent( } static bool TrySendRequestAndWaitForResponse( - const VirtualizationRoot* root, + VirtualizationRootHandle root, MessageType messageType, const vnode_t vnode, const FsidInode& vnodeFsidInode, @@ -719,7 +718,7 @@ static bool TrySendRequestAndWaitForResponse( return false; } - const char* relativePath = GetRelativePath(vnodePath, root->path); + const char* relativePath = VirtualizationRoot_GetRootRelativePath(root, vnodePath); int nextMessageId = OSIncrementAtomic(&s_nextMessageId); @@ -748,7 +747,7 @@ static bool TrySendRequestAndWaitForResponse( // TODO(Mac): Should we pass in the root directly, rather than root->index? // The index seems more like a private implementation detail. - if (!isShuttingDown && 0 != ActiveProvider_SendMessage(root->index, messageSpec)) + if (!isShuttingDown && 0 != ActiveProvider_SendMessage(root, messageSpec)) { // TODO: appropriately handle unresponsive providers @@ -885,19 +884,6 @@ static bool IsFileSystemCrawler(char* procname) return false; } -static const char* GetRelativePath(const char* path, const char* root) -{ - assert(strlen(path) >= strlen(root)); - - const char* relativePath = path + strlen(root); - if (relativePath[0] == '/') - { - relativePath++; - } - - return relativePath; -} - static bool ShouldIgnoreVnodeType(vtype vnodeType, vnode_t vnode) { switch (vnodeType) diff --git a/ProjFS.Mac/PrjFSKext/PrjFSKext/PrjFSProviderUserClient.cpp b/ProjFS.Mac/PrjFSKext/PrjFSKext/PrjFSProviderUserClient.cpp index 6e9b75427..83b2fcb19 100644 --- a/ProjFS.Mac/PrjFSKext/PrjFSKext/PrjFSProviderUserClient.cpp +++ b/ProjFS.Mac/PrjFSKext/PrjFSKext/PrjFSProviderUserClient.cpp @@ -19,7 +19,11 @@ static const IOExternalMethodDispatch ProviderUserClientDispatch[] = { [ProviderSelector_RegisterVirtualizationRootPath] = { - &PrjFSProviderUserClient::registerVirtualizationRoot, 0, kIOUCVariableStructureSize, 1, 0 + .function = &PrjFSProviderUserClient::registerVirtualizationRoot, + .checkScalarInputCount = 0, + .checkStructureInputSize = kIOUCVariableStructureSize, // null-terminated string: virtualisation root path + .checkScalarOutputCount = 1, // returned errno + .checkStructureOutputSize = 0 }, [ProviderSelector_KernelMessageResponse] = { @@ -37,7 +41,7 @@ bool PrjFSProviderUserClient::initWithTask( UInt32 type, OSDictionary* properties) { - this->virtualizationRootIndex = -1; + this->virtualizationRootHandle = RootHandle_None; this->pid = proc_selfpid(); if (!this->super::initWithTask(owningTask, securityToken, type, properties)) @@ -92,9 +96,9 @@ void PrjFSProviderUserClient::free() // the connection. IOReturn PrjFSProviderUserClient::clientClose() { - int32_t root = this->virtualizationRootIndex; - this->virtualizationRootIndex = -1; - if (-1 != root) + VirtualizationRootHandle root = this->virtualizationRootHandle; + this->virtualizationRootHandle = RootHandle_None; + if (RootHandle_None != root) { ActiveProvider_Disconnect(root); } @@ -210,7 +214,7 @@ IOReturn PrjFSProviderUserClient::registerVirtualizationRoot(const char* rootPat *outError = EINVAL; return kIOReturnSuccess; } - else if (this->virtualizationRootIndex != -1) + else if (this->virtualizationRootHandle != RootHandle_None) { // Already set *outError = EBUSY; @@ -220,11 +224,11 @@ IOReturn PrjFSProviderUserClient::registerVirtualizationRoot(const char* rootPat VirtualizationRootResult result = VirtualizationRoot_RegisterProviderForPath(this, this->pid, rootPath); if (0 == result.error) { - this->virtualizationRootIndex = result.rootIndex; + this->virtualizationRootHandle = result.root; // Sets the root index in the IORegistry for diagnostic purposes char location[5] = ""; - snprintf(location, sizeof(location), "%d", result.rootIndex); + snprintf(location, sizeof(location), "%d", result.root); this->setLocation(location); } diff --git a/ProjFS.Mac/PrjFSKext/PrjFSKext/PrjFSProviderUserClient.hpp b/ProjFS.Mac/PrjFSKext/PrjFSKext/PrjFSProviderUserClient.hpp index 12621ede4..de487108c 100644 --- a/ProjFS.Mac/PrjFSKext/PrjFSKext/PrjFSProviderUserClient.hpp +++ b/ProjFS.Mac/PrjFSKext/PrjFSKext/PrjFSProviderUserClient.hpp @@ -3,6 +3,7 @@ #include "PrjFSClasses.hpp" #include "Locks.hpp" #include "Message.h" +#include "VirtualizationRoots.hpp" #include struct MessageHeader; @@ -18,8 +19,8 @@ class PrjFSProviderUserClient : public IOUserClient Mutex dataQueueWriterMutex; public: pid_t pid; - // The root for which this is the provider; -1 prior to registration - int32_t virtualizationRootIndex; + // The root for which this is the provider; RootHandle_None prior to registration + VirtualizationRootHandle virtualizationRootHandle; // IOUserClient methods: virtual bool initWithTask( diff --git a/ProjFS.Mac/PrjFSKext/PrjFSKext/VirtualizationRoots.cpp b/ProjFS.Mac/PrjFSKext/PrjFSKext/VirtualizationRoots.cpp index 46a5704d9..ecb792448 100644 --- a/ProjFS.Mac/PrjFSKext/PrjFSKext/VirtualizationRoots.cpp +++ b/ProjFS.Mac/PrjFSKext/PrjFSKext/VirtualizationRoots.cpp @@ -12,6 +12,27 @@ #include "VnodeUtilities.hpp" +struct VirtualizationRoot +{ + bool inUse; + // If this is a nullptr, there is no active provider for this virtualization root (offline root) + PrjFSProviderUserClient* providerUserClient; + int providerPid; + // For an active root, this is retained (vnode_get), for an offline one, it is not, so it may be stale (check the vid) + vnode_t rootVNode; + uint32_t rootVNodeVid; + + // Mount point ID + persistent, on-disk ID for the root directory, so we can + // identify it if the vnode of an offline root gets recycled. + fsid_t rootFsid; + uint64_t rootInode; + + // TODO(Mac): this should eventually be entirely diagnostic and not used for decisions + char path[PrjFSMaxPath]; + + int32_t index; +}; + static RWLock s_rwLock = {}; // Arbitrary choice, but prevents user space attacker from causing @@ -20,9 +41,52 @@ static const size_t MaxVirtualizationRoots = 128; static VirtualizationRoot s_virtualizationRoots[MaxVirtualizationRoots] = {}; -static int16_t FindRootForVnode_Locked(vnode_t vnode, uint32_t vid, FsidInode fileId); -static int16_t FindUnusedIndex_Locked(); -static int16_t InsertVirtualizationRoot_Locked(PrjFSProviderUserClient* userClient, pid_t clientPID, vnode_t vnode, uint32_t vid, FsidInode persistentIds, const char* path); +// Looks up the vnode/vid and fsid/inode pairs among the known roots +static VirtualizationRootHandle FindRootAtVnode_Locked(vnode_t vnode, uint32_t vid, FsidInode fileId); + +// Looks up the vnode and fsid/inode pair among the known roots, and if not found, +// detects if there is a hitherto-unknown root at vnode by checking attributes. +static VirtualizationRootHandle FindOrDetectRootAtVnode(vnode_t vnode, vfs_context_t context, const FsidInode& vnodeFsidInode); + +static VirtualizationRootHandle FindUnusedIndex_Locked(); +static VirtualizationRootHandle InsertVirtualizationRoot_Locked(PrjFSProviderUserClient* userClient, pid_t clientPID, vnode_t vnode, uint32_t vid, FsidInode persistentIds, const char* path); + +bool VirtualizationRoot_IsOnline(VirtualizationRootHandle rootIndex) +{ + if (rootIndex < 0 || rootIndex >= MaxVirtualizationRoots) + { + return false; + } + + bool result; + RWLock_AcquireShared(s_rwLock); + { + result = (nullptr != s_virtualizationRoots[rootIndex].providerUserClient); + } + RWLock_ReleaseShared(s_rwLock); + + return result; +} + +bool VirtualizationRoot_PIDMatchesProvider(VirtualizationRootHandle rootIndex, pid_t pid) +{ + bool result; + RWLock_AcquireShared(s_rwLock); + { + result = + (rootIndex >= 0 && rootIndex < MaxVirtualizationRoots) + && (nullptr != s_virtualizationRoots[rootIndex].providerUserClient) + && pid == s_virtualizationRoots[rootIndex].providerPid; + } + RWLock_ReleaseShared(s_rwLock); + + return result; +} + +bool VirtualizationRoot_IsValidRootHandle(VirtualizationRootHandle rootIndex) +{ + return (rootIndex > RootHandle_None); +} kern_return_t VirtualizationRoots_Init() { @@ -37,7 +101,7 @@ kern_return_t VirtualizationRoots_Init() return KERN_FAILURE; } - for (uint32_t i = 0; i < MaxVirtualizationRoots; ++i) + for (VirtualizationRootHandle i = 0; i < MaxVirtualizationRoots; ++i) { s_virtualizationRoots[i].index = i; } @@ -56,18 +120,19 @@ kern_return_t VirtualizationRoots_Cleanup() return KERN_FAILURE; } -VirtualizationRoot* VirtualizationRoots_FindForVnode(vnode_t vnode, const FsidInode& vnodeFsidInode) +VirtualizationRootHandle VirtualizationRoot_FindForVnode(vnode_t vnode, const FsidInode& vnodeFsidInode) { - VirtualizationRoot* root = nullptr; + VirtualizationRootHandle rootHandle = RootHandle_None; vnode_get(vnode); // Search up the tree until we hit a known virtualization root or THE root of the file system - while (nullptr == root && NULLVP != vnode && !vnode_isvroot(vnode)) + while (RootHandle_None == rootHandle && NULLVP != vnode && !vnode_isvroot(vnode)) { - int16_t rootIndex = VirtualizationRoots_LookupVnode(vnode, /*context*/ nullptr, vnodeFsidInode); - if (rootIndex >= 0) + rootHandle = FindOrDetectRootAtVnode(vnode, nullptr /* vfs context */, vnodeFsidInode); + // Note: if FindOrDetectRootAtVnode returns a "special" handle other + // than RootHandle_None, we want to stop the search and return that. + if (rootHandle != RootHandle_None) { - root = &s_virtualizationRoots[rootIndex]; break; } @@ -81,22 +146,22 @@ VirtualizationRoot* VirtualizationRoots_FindForVnode(vnode_t vnode, const FsidIn vnode_put(vnode); } - return root; + return rootHandle; } -int16_t VirtualizationRoots_LookupVnode(vnode_t vnode, vfs_context_t context, const FsidInode& vnodeFsidInode) +static VirtualizationRootHandle FindOrDetectRootAtVnode(vnode_t vnode, vfs_context_t context, const FsidInode& vnodeFsidInode) { uint32_t vid = vnode_vid(vnode); - int16_t rootIndex; + VirtualizationRootHandle rootIndex; RWLock_AcquireShared(s_rwLock); { - rootIndex = FindRootForVnode_Locked(vnode, vid, vnodeFsidInode); + rootIndex = FindRootAtVnode_Locked(vnode, vid, vnodeFsidInode); } RWLock_ReleaseShared(s_rwLock); - if (rootIndex < 0) + if (rootIndex == RootHandle_None) { PrjFSVirtualizationRootXAttrData rootXattr = {}; SizeOrError xattrResult = Vnode_ReadXattr(vnode, PrjFSVirtualizationRootXAttrName, &rootXattr, sizeof(rootXattr), context); @@ -111,9 +176,9 @@ int16_t VirtualizationRoots_LookupVnode(vnode_t vnode, vfs_context_t context, co RWLock_AcquireExclusive(s_rwLock); { // Vnode may already have been inserted as a root in the interim - rootIndex = FindRootForVnode_Locked(vnode, vid, vnodeFsidInode); + rootIndex = FindRootAtVnode_Locked(vnode, vid, vnodeFsidInode); - if (rootIndex < 0) + if (RootHandle_None == rootIndex) { // Insert new offline root rootIndex = InsertVirtualizationRoot_Locked(nullptr, 0, vnode, vid, vnodeFsidInode, path); @@ -130,9 +195,9 @@ int16_t VirtualizationRoots_LookupVnode(vnode_t vnode, vfs_context_t context, co return rootIndex; } -static int16_t FindUnusedIndex_Locked() +static VirtualizationRootHandle FindUnusedIndex_Locked() { - for (int16_t i = 0; i < MaxVirtualizationRoots; ++i) + for (VirtualizationRootHandle i = 0; i < MaxVirtualizationRoots; ++i) { if (!s_virtualizationRoots[i].inUse) { @@ -140,7 +205,7 @@ static int16_t FindUnusedIndex_Locked() } } - return -1; + return RootHandle_None; } static bool FsidsAreEqual(fsid_t a, fsid_t b) @@ -148,9 +213,9 @@ static bool FsidsAreEqual(fsid_t a, fsid_t b) return a.val[0] == b.val[0] && a.val[1] == b.val[1]; } -static int16_t FindRootForVnode_Locked(vnode_t vnode, uint32_t vid, FsidInode fileId) +static VirtualizationRootHandle FindRootAtVnode_Locked(vnode_t vnode, uint32_t vid, FsidInode fileId) { - for (int16_t i = 0; i < MaxVirtualizationRoots; ++i) + for (VirtualizationRootHandle i = 0; i < MaxVirtualizationRoots; ++i) { VirtualizationRoot& rootEntry = s_virtualizationRoots[i]; if (!rootEntry.inUse) @@ -172,14 +237,13 @@ static int16_t FindRootForVnode_Locked(vnode_t vnode, uint32_t vid, FsidInode fi return i; } } - return -1; + return RootHandle_None; } // Returns negative value if it failed, or inserted index on success -static int16_t InsertVirtualizationRoot_Locked(PrjFSProviderUserClient* userClient, pid_t clientPID, vnode_t vnode, uint32_t vid, FsidInode persistentIds, const char* path) +static VirtualizationRootHandle InsertVirtualizationRoot_Locked(PrjFSProviderUserClient* userClient, pid_t clientPID, vnode_t vnode, uint32_t vid, FsidInode persistentIds, const char* path) { - // New root - int16_t rootIndex = FindUnusedIndex_Locked(); + VirtualizationRootHandle rootIndex = FindUnusedIndex_Locked(); if (rootIndex >= 0) { @@ -215,7 +279,7 @@ VirtualizationRootResult VirtualizationRoot_RegisterProviderForPath(PrjFSProvide vnode_t virtualizationRootVNode = NULLVP; vfs_context_t vfsContext = vfs_context_create(nullptr); - int32_t rootIndex = -1; + VirtualizationRootHandle rootIndex = RootHandle_None; errno_t err = vnode_lookup(virtualizationRootPath, 0 /* flags */, &virtualizationRootVNode, vfsContext); if (0 == err) { @@ -234,7 +298,7 @@ VirtualizationRootResult VirtualizationRoot_RegisterProviderForPath(PrjFSProvide RWLock_AcquireExclusive(s_rwLock); { - rootIndex = FindRootForVnode_Locked(virtualizationRootVNode, rootVid, vnodeIds); + rootIndex = FindRootAtVnode_Locked(virtualizationRootVNode, rootVid, vnodeIds); if (rootIndex >= 0) { // Reattaching to existing root @@ -242,7 +306,7 @@ VirtualizationRootResult VirtualizationRoot_RegisterProviderForPath(PrjFSProvide { // Only one provider per root err = EBUSY; - rootIndex = -1; + rootIndex = RootHandle_None; } else { @@ -292,7 +356,7 @@ VirtualizationRootResult VirtualizationRoot_RegisterProviderForPath(PrjFSProvide return VirtualizationRootResult { err, rootIndex }; } -void ActiveProvider_Disconnect(int32_t rootIndex) +void ActiveProvider_Disconnect(VirtualizationRootHandle rootIndex) { assert(rootIndex >= 0); assert(rootIndex <= MaxVirtualizationRoots); @@ -311,7 +375,7 @@ void ActiveProvider_Disconnect(int32_t rootIndex) RWLock_ReleaseExclusive(s_rwLock); } -errno_t ActiveProvider_SendMessage(int32_t rootIndex, const Message message) +errno_t ActiveProvider_SendMessage(VirtualizationRootHandle rootIndex, const Message message) { assert(rootIndex >= 0); assert(rootIndex < MaxVirtualizationRoots); @@ -356,3 +420,32 @@ bool VirtualizationRoot_VnodeIsOnAllowedFilesystem(vnode_t vnode) || 0 == strncmp("apfs", vfsStat->f_fstypename, sizeof(vfsStat->f_fstypename)); } +static const char* GetRelativePath(const char* path, const char* root) +{ + assert(strlen(path) >= strlen(root)); + + const char* relativePath = path + strlen(root); + if (relativePath[0] == '/') + { + relativePath++; + } + + return relativePath; +} + +const char* VirtualizationRoot_GetRootRelativePath(VirtualizationRootHandle rootIndex, const char* path) +{ + assert(rootIndex >= 0); + assert(rootIndex <= MaxVirtualizationRoots); + + const char* relativePath; + + RWLock_AcquireShared(s_rwLock); + { + assert(s_virtualizationRoots[rootIndex].inUse); + relativePath = GetRelativePath(path, s_virtualizationRoots[rootIndex].path); + } + RWLock_ReleaseShared(s_rwLock); + + return relativePath; +} diff --git a/ProjFS.Mac/PrjFSKext/PrjFSKext/VirtualizationRoots.hpp b/ProjFS.Mac/PrjFSKext/PrjFSKext/VirtualizationRoots.hpp index 3d1c86aef..2b19b0681 100644 --- a/ProjFS.Mac/PrjFSKext/PrjFSKext/VirtualizationRoots.hpp +++ b/ProjFS.Mac/PrjFSKext/PrjFSKext/VirtualizationRoots.hpp @@ -4,42 +4,37 @@ #include "PrjFSClasses.hpp" #include "kernel-header-wrappers/vnode.h" -struct VirtualizationRoot -{ - bool inUse; - // If this is a nullptr, there is no active provider for this virtualization root (offline root) - PrjFSProviderUserClient* providerUserClient; - int providerPid; - // For an active root, this is retained (vnode_get), for an offline one, it is not, so it may be stale (check the vid) - vnode_t rootVNode; - uint32_t rootVNodeVid; - - // Mount point ID + persistent, on-disk ID for the root directory, so we can - // identify it if the vnode of an offline root gets recycled. - fsid_t rootFsid; - uint64_t rootInode; - - // TODO(Mac): this should eventually be entirely diagnostic and not used for decisions - char path[PrjFSMaxPath]; +typedef int16_t VirtualizationRootHandle; - int32_t index; +// Zero and positive values indicate a handle for a valid virtualization +// root. Other values have special meanings: +enum VirtualizationRootSpecialHandle : VirtualizationRootHandle +{ + // Not in a virtualization root. + RootHandle_None = -1, + // Root/non-root state not known. Useful reset value for invalidating cached state. + RootHandle_Indeterminate = -2, + // Vnode is not in a virtualization root, but below a provider's registered temp directory + RootHandle_ProviderTemporaryDirectory = -3, }; kern_return_t VirtualizationRoots_Init(void); kern_return_t VirtualizationRoots_Cleanup(void); -VirtualizationRoot* VirtualizationRoots_FindForVnode(vnode_t vnode, const FsidInode& vnodeFsidInode); +VirtualizationRootHandle VirtualizationRoot_FindForVnode(vnode_t vnode, const FsidInode& vnodeFsidInode); struct VirtualizationRootResult { errno_t error; - int32_t rootIndex; + VirtualizationRootHandle root; }; VirtualizationRootResult VirtualizationRoot_RegisterProviderForPath(PrjFSProviderUserClient* userClient, pid_t clientPID, const char* virtualizationRootPath); -void ActiveProvider_Disconnect(int32_t rootIndex); +void ActiveProvider_Disconnect(VirtualizationRootHandle rootHandle); struct Message; -errno_t ActiveProvider_SendMessage(int32_t rootIndex, const Message message); +errno_t ActiveProvider_SendMessage(VirtualizationRootHandle rootHandle, const Message message); bool VirtualizationRoot_VnodeIsOnAllowedFilesystem(vnode_t vnode); - -int16_t VirtualizationRoots_LookupVnode(vnode_t vnode, vfs_context_t context, const FsidInode& vnodeFsidInode); +bool VirtualizationRoot_IsOnline(VirtualizationRootHandle rootHandle); +bool VirtualizationRoot_PIDMatchesProvider(VirtualizationRootHandle rootHandle, pid_t pid); +bool VirtualizationRoot_IsValidRootHandle(VirtualizationRootHandle rootHandle); +const char* VirtualizationRoot_GetRootRelativePath(VirtualizationRootHandle rootHandle, const char* path); From 0a0ac8838f2a2a9daddaed88682e411e421bd9fc Mon Sep 17 00:00:00 2001 From: Phil Dennis-Jordan Date: Mon, 1 Oct 2018 21:02:03 +0200 Subject: [PATCH 241/272] Mac kext: Dynamically alloc virtualisation root array & resize when full. --- .../PrjFSKext/VirtualizationRoots.cpp | 105 +++++++++++++----- 1 file changed, 76 insertions(+), 29 deletions(-) diff --git a/ProjFS.Mac/PrjFSKext/PrjFSKext/VirtualizationRoots.cpp b/ProjFS.Mac/PrjFSKext/PrjFSKext/VirtualizationRoots.cpp index ecb792448..88b0a97db 100644 --- a/ProjFS.Mac/PrjFSKext/PrjFSKext/VirtualizationRoots.cpp +++ b/ProjFS.Mac/PrjFSKext/PrjFSKext/VirtualizationRoots.cpp @@ -1,5 +1,6 @@ #include #include +#include #include "PrjFSCommon.h" #include "PrjFSXattrs.h" @@ -29,17 +30,13 @@ struct VirtualizationRoot // TODO(Mac): this should eventually be entirely diagnostic and not used for decisions char path[PrjFSMaxPath]; - - int32_t index; }; static RWLock s_rwLock = {}; -// Arbitrary choice, but prevents user space attacker from causing -// allocation of too much wired kernel memory. -static const size_t MaxVirtualizationRoots = 128; - -static VirtualizationRoot s_virtualizationRoots[MaxVirtualizationRoots] = {}; +// Current length of the s_virtualizationRoots array +static uint16_t s_maxVirtualizationRoots = 0; +static VirtualizationRoot* s_virtualizationRoots = nullptr; // Looks up the vnode/vid and fsid/inode pairs among the known roots static VirtualizationRootHandle FindRootAtVnode_Locked(vnode_t vnode, uint32_t vid, FsidInode fileId); @@ -51,9 +48,9 @@ static VirtualizationRootHandle FindOrDetectRootAtVnode(vnode_t vnode, vfs_conte static VirtualizationRootHandle FindUnusedIndex_Locked(); static VirtualizationRootHandle InsertVirtualizationRoot_Locked(PrjFSProviderUserClient* userClient, pid_t clientPID, vnode_t vnode, uint32_t vid, FsidInode persistentIds, const char* path); -bool VirtualizationRoot_IsOnline(VirtualizationRootHandle rootIndex) +bool VirtualizationRoot_IsOnline(VirtualizationRootHandle rootHandle) { - if (rootIndex < 0 || rootIndex >= MaxVirtualizationRoots) + if (rootHandle < 0) { return false; } @@ -61,22 +58,27 @@ bool VirtualizationRoot_IsOnline(VirtualizationRootHandle rootIndex) bool result; RWLock_AcquireShared(s_rwLock); { - result = (nullptr != s_virtualizationRoots[rootIndex].providerUserClient); + result = + rootHandle < s_maxVirtualizationRoots + && s_virtualizationRoots[rootHandle].inUse + && nullptr != s_virtualizationRoots[rootHandle].providerUserClient; } RWLock_ReleaseShared(s_rwLock); return result; } -bool VirtualizationRoot_PIDMatchesProvider(VirtualizationRootHandle rootIndex, pid_t pid) +bool VirtualizationRoot_PIDMatchesProvider(VirtualizationRootHandle rootHandle, pid_t pid) { bool result; RWLock_AcquireShared(s_rwLock); { result = - (rootIndex >= 0 && rootIndex < MaxVirtualizationRoots) - && (nullptr != s_virtualizationRoots[rootIndex].providerUserClient) - && pid == s_virtualizationRoots[rootIndex].providerPid; + rootHandle >= 0 + && rootHandle < s_maxVirtualizationRoots + && s_virtualizationRoots[rootHandle].inUse + && nullptr != s_virtualizationRoots[rootHandle].providerUserClient + && pid == s_virtualizationRoots[rootHandle].providerPid; } RWLock_ReleaseShared(s_rwLock); @@ -101,11 +103,20 @@ kern_return_t VirtualizationRoots_Init() return KERN_FAILURE; } - for (VirtualizationRootHandle i = 0; i < MaxVirtualizationRoots; ++i) + s_maxVirtualizationRoots = 128; + s_virtualizationRoots = Memory_AllocArray(s_maxVirtualizationRoots); + if (nullptr == s_virtualizationRoots) { - s_virtualizationRoots[i].index = i; + return KERN_RESOURCE_SHORTAGE; } + for (VirtualizationRootHandle i = 0; i < s_maxVirtualizationRoots; ++i) + { + s_virtualizationRoots[i] = VirtualizationRoot{ }; + } + + atomic_thread_fence(memory_order_seq_cst); + return KERN_SUCCESS; } @@ -197,7 +208,7 @@ static VirtualizationRootHandle FindOrDetectRootAtVnode(vnode_t vnode, vfs_conte static VirtualizationRootHandle FindUnusedIndex_Locked() { - for (VirtualizationRootHandle i = 0; i < MaxVirtualizationRoots; ++i) + for (VirtualizationRootHandle i = 0; i < s_maxVirtualizationRoots; ++i) { if (!s_virtualizationRoots[i].inUse) { @@ -208,6 +219,42 @@ static VirtualizationRootHandle FindUnusedIndex_Locked() return RootHandle_None; } +static VirtualizationRootHandle FindUnusedIndexOrGrow_Locked() +{ + VirtualizationRootHandle rootIndex = FindUnusedIndex_Locked(); + + if (RootHandle_None == rootIndex) + { + // No space, resize array + uint16_t newLength = MIN(s_maxVirtualizationRoots * 2u, INT16_MAX + 1u); + if (newLength <= s_maxVirtualizationRoots) + { + return RootHandle_None; + } + + VirtualizationRoot* grownArray = Memory_AllocArray(newLength); + if (nullptr == grownArray) + { + return RootHandle_None; + } + + uint32_t oldSizeBytes = sizeof(s_virtualizationRoots[0]) * s_maxVirtualizationRoots; + memcpy(grownArray, s_virtualizationRoots, oldSizeBytes); + Memory_Free(s_virtualizationRoots, oldSizeBytes); + s_virtualizationRoots = grownArray; + + for (uint16_t i = s_maxVirtualizationRoots; i < newLength; ++i) + { + s_virtualizationRoots[i] = VirtualizationRoot{ }; + } + + rootIndex = s_maxVirtualizationRoots; + s_maxVirtualizationRoots = newLength; + } + + return rootIndex; +} + static bool FsidsAreEqual(fsid_t a, fsid_t b) { return a.val[0] == b.val[0] && a.val[1] == b.val[1]; @@ -215,7 +262,7 @@ static bool FsidsAreEqual(fsid_t a, fsid_t b) static VirtualizationRootHandle FindRootAtVnode_Locked(vnode_t vnode, uint32_t vid, FsidInode fileId) { - for (VirtualizationRootHandle i = 0; i < MaxVirtualizationRoots; ++i) + for (VirtualizationRootHandle i = 0; i < s_maxVirtualizationRoots; ++i) { VirtualizationRoot& rootEntry = s_virtualizationRoots[i]; if (!rootEntry.inUse) @@ -243,17 +290,16 @@ static VirtualizationRootHandle FindRootAtVnode_Locked(vnode_t vnode, uint32_t v // Returns negative value if it failed, or inserted index on success static VirtualizationRootHandle InsertVirtualizationRoot_Locked(PrjFSProviderUserClient* userClient, pid_t clientPID, vnode_t vnode, uint32_t vid, FsidInode persistentIds, const char* path) { - VirtualizationRootHandle rootIndex = FindUnusedIndex_Locked(); + VirtualizationRootHandle rootIndex = FindUnusedIndexOrGrow_Locked(); - if (rootIndex >= 0) + if (RootHandle_None != rootIndex) { - assert(rootIndex < MaxVirtualizationRoots); + assert(rootIndex < s_maxVirtualizationRoots); VirtualizationRoot* root = &s_virtualizationRoots[rootIndex]; root->providerUserClient = userClient; root->providerPid = clientPID; root->inUse = true; - root->index = rootIndex; root->rootVNode = vnode; root->rootVNodeVid = vid; @@ -321,7 +367,7 @@ VirtualizationRootResult VirtualizationRoot_RegisterProviderForPath(PrjFSProvide rootIndex = InsertVirtualizationRoot_Locked(userClient, clientPID, virtualizationRootVNode, rootVid, vnodeIds, virtualizationRootPath); if (rootIndex >= 0) { - assert(rootIndex < MaxVirtualizationRoots); + assert(rootIndex < s_maxVirtualizationRoots); VirtualizationRoot* root = &s_virtualizationRoots[rootIndex]; strlcpy(root->path, virtualizationRootPath, sizeof(root->path)); @@ -359,10 +405,10 @@ VirtualizationRootResult VirtualizationRoot_RegisterProviderForPath(PrjFSProvide void ActiveProvider_Disconnect(VirtualizationRootHandle rootIndex) { assert(rootIndex >= 0); - assert(rootIndex <= MaxVirtualizationRoots); - RWLock_AcquireExclusive(s_rwLock); { + assert(rootIndex <= s_maxVirtualizationRoots); + VirtualizationRoot* root = &s_virtualizationRoots[rootIndex]; assert(nullptr != root->providerUserClient); @@ -378,19 +424,20 @@ void ActiveProvider_Disconnect(VirtualizationRootHandle rootIndex) errno_t ActiveProvider_SendMessage(VirtualizationRootHandle rootIndex, const Message message) { assert(rootIndex >= 0); - assert(rootIndex < MaxVirtualizationRoots); PrjFSProviderUserClient* userClient = nullptr; - RWLock_AcquireExclusive(s_rwLock); + RWLock_AcquireShared(s_rwLock); { + assert(rootIndex < s_maxVirtualizationRoots); + userClient = s_virtualizationRoots[rootIndex].providerUserClient; if (nullptr != userClient) { userClient->retain(); } } - RWLock_ReleaseExclusive(s_rwLock); + RWLock_ReleaseShared(s_rwLock); if (nullptr != userClient) { @@ -436,12 +483,12 @@ static const char* GetRelativePath(const char* path, const char* root) const char* VirtualizationRoot_GetRootRelativePath(VirtualizationRootHandle rootIndex, const char* path) { assert(rootIndex >= 0); - assert(rootIndex <= MaxVirtualizationRoots); const char* relativePath; RWLock_AcquireShared(s_rwLock); { + assert(rootIndex < s_maxVirtualizationRoots); assert(s_virtualizationRoots[rootIndex].inUse); relativePath = GetRelativePath(path, s_virtualizationRoots[rootIndex].path); } From 25dab5d4afac154b2c5d9bc00b7422de268c729e Mon Sep 17 00:00:00 2001 From: John Briggs Date: Thu, 4 Oct 2018 13:20:11 -0400 Subject: [PATCH 242/272] Select installation of VS that can build the project on Windows. - Pass all required components to vswhere in BuildGVFSForWindows.bat. - Remove unnecessary component from Readme.md. --- Readme.md | 1 - Scripts/BuildGVFSForWindows.bat | 12 +++++++++--- 2 files changed, 9 insertions(+), 4 deletions(-) diff --git a/Readme.md b/Readme.md index 133b93754..1a005324c 100644 --- a/Readme.md +++ b/Readme.md @@ -40,7 +40,6 @@ If you'd like to build your own VFS for Git Windows installer: * .NET Core cross-platform development * Include the following additional components: * .NET Core runtime - * C++/CLI support * Windows 10 SDK (10.0.10240.0) * Install the .NET Core 2.1 SDK (https://www.microsoft.com/net/download/dotnet-core/2.1) * Create a folder to clone into, e.g. `C:\Repos\VFSForGit` diff --git a/Scripts/BuildGVFSForWindows.bat b/Scripts/BuildGVFSForWindows.bat index fd3adee6b..ac3ec7a1d 100644 --- a/Scripts/BuildGVFSForWindows.bat +++ b/Scripts/BuildGVFSForWindows.bat @@ -19,14 +19,20 @@ SET vswhere=%~dp0..\..\packages\vswhere.%vswherever%\tools\vswhere.exe :: Use vswhere to find the latest VS installation (including prerelease installations) with the msbuild component. :: See https://github.com/Microsoft/vswhere/wiki/Find-MSBuild -for /f "usebackq tokens=*" %%i in (`%vswhere% -all -prerelease -latest -products * -requires Microsoft.Component.MSBuild -property installationPath`) do ( +for /f "usebackq tokens=*" %%i in (`%vswhere% -all -prerelease -latest -products * -requires Microsoft.Component.MSBuild Microsoft.VisualStudio.Workload.ManagedDesktop Microsoft.VisualStudio.Workload.NativeDesktop Microsoft.VisualStudio.Workload.NetCoreTools Microsoft.Component.NetFX.Core.Runtime Microsoft.VisualStudio.Component.Windows10SDK.10240 -property installationPath`) do ( set VsInstallDir=%%i ) +IF NOT DEFINED VsInstallDir ( + echo ERROR: Could not locate a Visual Studio installation with required components. + echo Refer to Readme.md for a list of the required Visual Studio components. + exit /b 10 +) + SET msbuild="%VsInstallDir%\MSBuild\15.0\Bin\amd64\msbuild.exe" IF NOT EXIST %msbuild% ( - echo Error: Could not find msbuild - exit /b 1 + echo ERROR: Could not find msbuild + exit /b 1 ) %msbuild% %~dp0\..\GVFS.sln /p:GVFSVersion=%GVFSVersion% /p:Configuration=%SolutionConfiguration% /p:Platform=x64 || exit /b 1 From c07d9eb6a7b5ae4df3bbf9a346a3c9697184690b Mon Sep 17 00:00:00 2001 From: Al Ameen Shah Date: Fri, 5 Oct 2018 17:42:23 -0400 Subject: [PATCH 243/272] - Removed wildcard matching while searching for downloaded installers - Added ToUpper() with ToUpperInvariant in LocalGVFSConfig - Fixed partial failure in build caused by UninstallGVFS.bat script. - Updated TryGetVersion in GitProcess to accept git path as input argument. --- GVFS/GVFS.Common/GVFSConstants.cs | 5 ++-- GVFS/GVFS.Common/Git/GitProcess.cs | 27 +++++++-------------- GVFS/GVFS.Common/LocalGVFSConfig.cs | 25 +++++++++++-------- GVFS/GVFS.Common/ProductUpgrader.Shared.cs | 28 +++++++++++++++------- GVFS/GVFS.Upgrader/UpgradeOrchestrator.cs | 5 ++-- GVFS/GVFS/CommandLine/DiagnoseVerb.cs | 7 +++--- GVFS/GVFS/CommandLine/GVFSVerb.cs | 5 ++-- Scripts/UninstallGVFS.bat | 2 +- 8 files changed, 56 insertions(+), 48 deletions(-) diff --git a/GVFS/GVFS.Common/GVFSConstants.cs b/GVFS/GVFS.Common/GVFSConstants.cs index a387f8937..8d2bd4989 100644 --- a/GVFS/GVFS.Common/GVFSConstants.cs +++ b/GVFS/GVFS.Common/GVFSConstants.cs @@ -83,12 +83,12 @@ public static class LogFileTypes public const string Dehydrate = "dehydrate"; public const string MountVerb = MountPrefix + "_verb"; public const string MountProcess = MountPrefix + "_process"; + public const string MountUpgrade = MountPrefix + "_repoupgrade"; public const string Prefetch = "prefetch"; public const string Repair = "repair"; public const string Service = "service"; public const string UpgradeVerb = UpgradePrefix + "_verb"; - public const string UpgradeProcess = UpgradePrefix + "_process"; - public const string MountUpgrade = MountPrefix + "_repoupgrade"; + public const string UpgradeProcess = UpgradePrefix + "_process"; } public static class DotGVFS @@ -228,7 +228,6 @@ public static class UpgradeVerbMessages { public const string GVFSUpgrade = "`gvfs upgrade`"; public const string GVFSUpgradeConfirm = "`gvfs upgrade --confirm`"; - public const string GVFSUpgradeOptionalConfirm = "`gvfs upgrade [--confirm]`"; public const string NoUpgradeCheckPerformed = "No upgrade check was performed."; public const string NoneRingConsoleAlert = "Upgrade ring set to \"None\". " + NoUpgradeCheckPerformed; public const string NoRingConfigConsoleAlert = "Upgrade ring is not set. " + NoUpgradeCheckPerformed; diff --git a/GVFS/GVFS.Common/Git/GitProcess.cs b/GVFS/GVFS.Common/Git/GitProcess.cs index 8701605ab..41e63763d 100644 --- a/GVFS/GVFS.Common/Git/GitProcess.cs +++ b/GVFS/GVFS.Common/Git/GitProcess.cs @@ -92,30 +92,21 @@ public static Result GetFromFileConfig(string gitBinPath, string configFile, str return new GitProcess(gitBinPath, workingDirectoryRoot: null, gvfsHooksRoot: null).InvokeGitOutsideEnlistment("config --file " + configFile + " " + settingName); } - public static bool TryGetVersion(out GitVersion gitVersion, out string error) + public static bool TryGetVersion(string gitBinPath, out GitVersion gitVersion, out string error) { - string gitPath = GVFSPlatform.Instance.GitInstallation.GetInstalledGitBinPath(); - if (gitPath != null) - { - GitProcess gitProcess = new GitProcess(gitPath, null, null); - Result result = gitProcess.InvokeGitOutsideEnlistment("--version"); - string version = result.Output; - - if (!GitVersion.TryParseGitVersionCommandResult(version, out gitVersion)) - { - error = "Unable to determine installed git version. " + version; - return false; - } + GitProcess gitProcess = new GitProcess(gitBinPath, null, null); + Result result = gitProcess.InvokeGitOutsideEnlistment("--version"); + string version = result.Output; - error = null; - return true; - } - else + if (result.HasErrors || !GitVersion.TryParseGitVersionCommandResult(version, out gitVersion)) { gitVersion = null; - error = "Unable to determine installed git path."; + error = "Unable to determine installed git version. " + version; return false; } + + error = null; + return true; } public virtual void RevokeCredential(string repoUrl) diff --git a/GVFS/GVFS.Common/LocalGVFSConfig.cs b/GVFS/GVFS.Common/LocalGVFSConfig.cs index 980d5956a..ac65b4589 100644 --- a/GVFS/GVFS.Common/LocalGVFSConfig.cs +++ b/GVFS/GVFS.Common/LocalGVFSConfig.cs @@ -21,21 +21,21 @@ public LocalGVFSConfig() } public bool TryGetConfig( - string key, + string name, out string value, out string error, ITracer tracer) { if (!this.TryLoadSettings(tracer, out error)) { - error = $"Error getting config value {key}. {error}"; + error = $"Error getting config value {name}. {error}"; value = null; return false; } try { - this.allSettings.TryGetValue(key.ToUpper(), out value); + this.allSettings.TryGetValue(this.KeyFromConfigName(name), out value); error = null; return true; } @@ -44,30 +44,30 @@ public bool TryGetConfig( const string ErrorFormat = "Error getting config value for {0}. Config file {1}. {2}"; if (tracer != null) { - tracer.RelatedError(ErrorFormat, key, this.configFile, exception.ToString()); + tracer.RelatedError(ErrorFormat, name, this.configFile, exception.ToString()); } - error = string.Format(ErrorFormat, key, this.configFile, exception.Message); + error = string.Format(ErrorFormat, name, this.configFile, exception.Message); value = null; return false; } } public bool TrySetConfig( - string key, + string name, string value, out string error, ITracer tracer) { if (!this.TryLoadSettings(tracer, out error)) { - error = $"Error setting config value {key}: {value}. {error}"; + error = $"Error setting config value {name}: {value}. {error}"; return false; } try { - this.allSettings.SetValueAndFlush(key.ToUpper(), value); + this.allSettings.SetValueAndFlush(this.KeyFromConfigName(name), value); error = null; return true; } @@ -76,15 +76,20 @@ public bool TrySetConfig( const string ErrorFormat = "Error setting config value {0}: {1}. Config file {2}. {3}"; if (tracer != null) { - tracer.RelatedError(ErrorFormat, key, value, this.configFile, exception.ToString()); + tracer.RelatedError(ErrorFormat, name, value, this.configFile, exception.ToString()); } - error = string.Format(ErrorFormat, key, value, this.configFile, exception.Message); + error = string.Format(ErrorFormat, name, value, this.configFile, exception.Message); value = null; return false; } } + private string KeyFromConfigName(string configName) + { + return configName.ToUpperInvariant(); + } + private bool TryLoadSettings(ITracer tracer, out string error) { if (this.allSettings == null) diff --git a/GVFS/GVFS.Common/ProductUpgrader.Shared.cs b/GVFS/GVFS.Common/ProductUpgrader.Shared.cs index 178f58038..51125717e 100644 --- a/GVFS/GVFS.Common/ProductUpgrader.Shared.cs +++ b/GVFS/GVFS.Common/ProductUpgrader.Shared.cs @@ -1,5 +1,4 @@ -using System; -using System.Diagnostics; +using System.Collections.Generic; using System.IO; namespace GVFS.Common @@ -19,12 +18,25 @@ public static bool IsLocalUpgradeAvailable() string downloadDirectory = GetAssetDownloadsPath(); if (Directory.Exists(downloadDirectory)) { - const string PotentialInstallerName = "*VFS*.*"; - string[] installers = Directory.GetFiles( - downloadDirectory, - PotentialInstallerName, - SearchOption.TopDirectoryOnly); - return installers.Length > 0; + // This method is used Only by Git hooks. Git hooks does not have access + // to GVFSPlatform to read platform specific file extensions. That is the + // reason possible installer file extensions are defined here. + HashSet extensions = new HashSet() { "EXE", "DMG", "RPM", "DEB" }; + HashSet installerNames = new HashSet() + { + GVFSInstallerFileNamePrefix.ToUpperInvariant(), + VFSForGitInstallerFileNamePrefix.ToUpperInvariant() + }; + + foreach (string file in Directory.EnumerateFiles(downloadDirectory, "*", SearchOption.TopDirectoryOnly)) + { + string[] components = Path.GetFileName(file).ToUpperInvariant().Split('.'); + int length = components.Length; + if (length >= 2 && installerNames.Contains(components[0]) && extensions.Contains(components[length - 1])) + { + return true; + } + } } return false; diff --git a/GVFS/GVFS.Upgrader/UpgradeOrchestrator.cs b/GVFS/GVFS.Upgrader/UpgradeOrchestrator.cs index 66a729269..f3faf5f37 100644 --- a/GVFS/GVFS.Upgrader/UpgradeOrchestrator.cs +++ b/GVFS/GVFS.Upgrader/UpgradeOrchestrator.cs @@ -414,9 +414,8 @@ private void LogInstalledVersionInfo() GitVersion installedGitVersion = null; string error = null; - if (GitProcess.TryGetVersion( - out installedGitVersion, - out error)) + string gitPath = GVFSPlatform.Instance.GitInstallation.GetInstalledGitBinPath(); + if (!string.IsNullOrEmpty(gitPath) && GitProcess.TryGetVersion(gitPath, out installedGitVersion, out error)) { metadata.Add(nameof(installedGitVersion), installedGitVersion.ToString()); } diff --git a/GVFS/GVFS/CommandLine/DiagnoseVerb.cs b/GVFS/GVFS/CommandLine/DiagnoseVerb.cs index a7979157d..c8a0c0a52 100644 --- a/GVFS/GVFS/CommandLine/DiagnoseVerb.cs +++ b/GVFS/GVFS/CommandLine/DiagnoseVerb.cs @@ -50,9 +50,10 @@ protected override void Execute(GVFSEnlistment enlistment) this.WriteMessage(string.Empty); this.WriteMessage("gvfs version " + ProcessHelper.GetCurrentProcessVersion()); - GitVersion gitVersion; - string error; - if (GitProcess.TryGetVersion(out gitVersion, out error)) + GitVersion gitVersion = null; + string error = null; + string gitPath = GVFSPlatform.Instance.GitInstallation.GetInstalledGitBinPath(); + if (!string.IsNullOrEmpty(gitPath) && GitProcess.TryGetVersion(gitPath, out gitVersion, out error)) { this.WriteMessage("git version " + gitVersion.ToString()); } diff --git a/GVFS/GVFS/CommandLine/GVFSVerb.cs b/GVFS/GVFS/CommandLine/GVFSVerb.cs index 56b2eca61..94bd33752 100644 --- a/GVFS/GVFS/CommandLine/GVFSVerb.cs +++ b/GVFS/GVFS/CommandLine/GVFSVerb.cs @@ -619,8 +619,9 @@ private void CheckFileSystemSupportsRequiredFeatures(ITracer tracer, Enlistment private void CheckGitVersion(ITracer tracer, GVFSEnlistment enlistment, out string version) { - GitVersion gitVersion; - if (!GitProcess.TryGetVersion(out gitVersion, out string _)) + string gitPath = GVFSPlatform.Instance.GitInstallation.GetInstalledGitBinPath(); + GitVersion gitVersion = null; + if (string.IsNullOrEmpty(gitPath) || !GitProcess.TryGetVersion(gitPath, out gitVersion, out string _)) { this.ReportErrorAndExit(tracer, "Error: Unable to retrieve the git version"); } diff --git a/Scripts/UninstallGVFS.bat b/Scripts/UninstallGVFS.bat index 27ecc7a9e..6a0451b50 100644 --- a/Scripts/UninstallGVFS.bat +++ b/Scripts/UninstallGVFS.bat @@ -17,6 +17,6 @@ for /F "delims=" %%f in ('dir "c:\Program Files\GVFS\unins*.exe" /B /S /O:-D') d :deleteGVFS rmdir /q/s "c:\Program Files\GVFS" -rmdir /q/s "C:\ProgramData\GVFS\GVFS.Upgrade" +if exist "C:\ProgramData\GVFS\GVFS.Upgrade" rmdir /q/s "C:\ProgramData\GVFS\GVFS.Upgrade" :end From 63a8405ccd02c8915ef0ee4ad963a12fd7e8a971 Mon Sep 17 00:00:00 2001 From: William Baker Date: Fri, 5 Oct 2018 14:47:08 -0700 Subject: [PATCH 244/272] Windows: Add delay and retry to attributes check in ExpandedFileAttributesAreUpdated --- .../BasicFileSystemTests.cs | 24 ++++++++++++++++++- 1 file changed, 23 insertions(+), 1 deletion(-) diff --git a/GVFS/GVFS.FunctionalTests/Tests/EnlistmentPerFixture/BasicFileSystemTests.cs b/GVFS/GVFS.FunctionalTests/Tests/EnlistmentPerFixture/BasicFileSystemTests.cs index 3c05f8151..37061f874 100644 --- a/GVFS/GVFS.FunctionalTests/Tests/EnlistmentPerFixture/BasicFileSystemTests.cs +++ b/GVFS/GVFS.FunctionalTests/Tests/EnlistmentPerFixture/BasicFileSystemTests.cs @@ -8,12 +8,17 @@ using System.Collections.Generic; using System.IO; using System.Linq; +using System.Threading; namespace GVFS.FunctionalTests.Tests.LongRunningEnlistment { [TestFixture] public class BasicFileSystemTests : TestsWithEnlistmentPerFixture { + private const int FileAttributeSparseFile = 0x00000200; + private const int FileAttributeReparsePoint = 0x00000400; + private const int FileAttributeRecallOnOpen = 0x00040000; + [TestCaseSource(typeof(FileRunnersAndFolders), FileRunnersAndFolders.TestRunners)] public void ShrinkFileContents(FileSystemRunner fileSystem, string parentFolder) { @@ -152,7 +157,24 @@ public void ExpandedFileAttributesAreUpdated() // Ignore the archive bit as it can be re-added to the file as part of its expansion to full FileAttributes attributes = info.Attributes & ~FileAttributes.Archive; - attributes.ShouldEqual(FileAttributes.Hidden, "Attributes do not match"); + + int retryCount = 0; + int maxRetries = 10; + while (attributes != FileAttributes.Hidden && retryCount < maxRetries) + { + // ProjFS attributes are remoted asynchronously when files are converted to full + FileAttributes attributesLessProjFS = attributes & (FileAttributes)~(FileAttributeSparseFile | FileAttributeRecallOnOpen | FileAttributeReparsePoint); + + attributesLessProjFS.ShouldEqual( + FileAttributes.Hidden, + $"Attributes (ignoring ProjFS attributes) do not match, expected: {FileAttributes.Hidden} actual: {attributesLessProjFS}"); + + ++retryCount; + Thread.Sleep(500); + attributes = new FileInfo(virtualFile).Attributes & ~FileAttributes.Archive; + } + + attributes.ShouldEqual(FileAttributes.Hidden, $"Attributes do not match, expected: {FileAttributes.Hidden} actual: {attributes}"); } [TestCase] From 320d04645f8a65ea597ff4cfe74a6f636684d2d3 Mon Sep 17 00:00:00 2001 From: William Baker Date: Fri, 5 Oct 2018 15:08:56 -0700 Subject: [PATCH 245/272] Use Refresh rather than creating a new FileInfo --- .../Tests/EnlistmentPerFixture/BasicFileSystemTests.cs | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/GVFS/GVFS.FunctionalTests/Tests/EnlistmentPerFixture/BasicFileSystemTests.cs b/GVFS/GVFS.FunctionalTests/Tests/EnlistmentPerFixture/BasicFileSystemTests.cs index 37061f874..db86eb26e 100644 --- a/GVFS/GVFS.FunctionalTests/Tests/EnlistmentPerFixture/BasicFileSystemTests.cs +++ b/GVFS/GVFS.FunctionalTests/Tests/EnlistmentPerFixture/BasicFileSystemTests.cs @@ -171,7 +171,9 @@ public void ExpandedFileAttributesAreUpdated() ++retryCount; Thread.Sleep(500); - attributes = new FileInfo(virtualFile).Attributes & ~FileAttributes.Archive; + + info.Refresh(); + attributes = info.Attributes & ~FileAttributes.Archive; } attributes.ShouldEqual(FileAttributes.Hidden, $"Attributes do not match, expected: {FileAttributes.Hidden} actual: {attributes}"); From c2910a10f4306e1985aa31b628899df21e76a0c4 Mon Sep 17 00:00:00 2001 From: Phil Dennis-Jordan Date: Wed, 20 Jun 2018 12:22:09 +0200 Subject: [PATCH 246/272] Mac kext: Scoped timing wrapper and user client for extracting profile data This adds a scoped time measurement class with splits. Probes are defined in an enum. When a ProfileSample instance is created, the Mach absolute time is taken, and when it is destroyed, another time stamp is taken, and the interval recorded with the sample aggregation probe ID specified during construction. Additionally, split time samples can be taken. The number of samples, total runtime, sum of squares, minimum, and maximum values are recorded. From these, mean and standard deviation can be derived. An external method for extracting the profiling data from the kernel has been added to the logging user client class. The simple log tool has been extended to fetch and dump this data to stdout every 15 seconds. --- .../PrjFSKext.xcodeproj/project.pbxproj | 6 + .../PrjFSKext/PerformanceTracing.cpp | 59 +++++++ .../PrjFSKext/PerformanceTracing.hpp | 155 ++++++++++++++++++ ProjFS.Mac/PrjFSKext/PrjFSKext/PrjFSKext.cpp | 3 + .../PrjFSKext/PrjFSLogUserClient.cpp | 43 +++++ .../PrjFSKext/PrjFSLogUserClient.hpp | 13 ++ ProjFS.Mac/PrjFSKext/public/PrjFSCommon.h | 25 +++ .../PrjFSKext/public/PrjFSLogClientShared.h | 8 + .../PrjFSLib.xcodeproj/project.pbxproj | 6 + .../PrjFSLib/prjfs-log/kext-perf-tracing.cpp | 65 ++++++++ .../PrjFSLib/prjfs-log/kext-perf-tracing.hpp | 5 + ProjFS.Mac/PrjFSLib/prjfs-log/prjfs-log.cpp | 34 +++- 12 files changed, 419 insertions(+), 3 deletions(-) create mode 100644 ProjFS.Mac/PrjFSKext/PrjFSKext/PerformanceTracing.cpp create mode 100644 ProjFS.Mac/PrjFSKext/PrjFSKext/PerformanceTracing.hpp create mode 100644 ProjFS.Mac/PrjFSLib/prjfs-log/kext-perf-tracing.cpp create mode 100644 ProjFS.Mac/PrjFSLib/prjfs-log/kext-perf-tracing.hpp diff --git a/ProjFS.Mac/PrjFSKext/PrjFSKext.xcodeproj/project.pbxproj b/ProjFS.Mac/PrjFSKext/PrjFSKext.xcodeproj/project.pbxproj index dbd7e2453..44b29d877 100644 --- a/ProjFS.Mac/PrjFSKext/PrjFSKext.xcodeproj/project.pbxproj +++ b/ProjFS.Mac/PrjFSKext/PrjFSKext.xcodeproj/project.pbxproj @@ -15,6 +15,7 @@ 4AC1D7C12091FA0400786861 /* PrjFSProviderUserClient.hpp in Headers */ = {isa = PBXBuildFile; fileRef = 4AC1D7BF2091FA0400786861 /* PrjFSProviderUserClient.hpp */; }; 4AC1D7C52091FBFC00786861 /* PrjFSLogUserClient.cpp in Sources */ = {isa = PBXBuildFile; fileRef = 4AC1D7C32091FBFC00786861 /* PrjFSLogUserClient.cpp */; }; 4AC1D7C62091FBFC00786861 /* PrjFSLogUserClient.hpp in Headers */ = {isa = PBXBuildFile; fileRef = 4AC1D7C42091FBFC00786861 /* PrjFSLogUserClient.hpp */; }; + 4AE187A821203A890026AC68 /* PerformanceTracing.cpp in Sources */ = {isa = PBXBuildFile; fileRef = 4AE187A721203A890026AC68 /* PerformanceTracing.cpp */; }; C6BDD366208BC60400CB7E58 /* VirtualizationRoots.hpp in Headers */ = {isa = PBXBuildFile; fileRef = C6BDD364208BC60400CB7E58 /* VirtualizationRoots.hpp */; }; C6BDD367208BC60400CB7E58 /* VirtualizationRoots.cpp in Sources */ = {isa = PBXBuildFile; fileRef = C6BDD365208BC60400CB7E58 /* VirtualizationRoots.cpp */; }; C6BDD36A208BC99100CB7E58 /* Locks.hpp in Headers */ = {isa = PBXBuildFile; fileRef = C6BDD368208BC99100CB7E58 /* Locks.hpp */; }; @@ -38,6 +39,7 @@ 4A9D139C208F675500376182 /* PrjFSService.hpp */ = {isa = PBXFileReference; fileEncoding = 4; indentWidth = 4; lastKnownFileType = sourcecode.cpp.h; path = PrjFSService.hpp; sourceTree = ""; tabWidth = 4; usesTabs = 0; }; 4AA0BA9920A4500600F33D1C /* vnode.h */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.c.h; path = vnode.h; sourceTree = ""; }; 4ABB733020B85DA500DC0D17 /* mount.h */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.c.h; path = mount.h; sourceTree = ""; }; + 4ABB734320B867E000DC0D17 /* PerformanceTracing.hpp */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.cpp.h; path = PerformanceTracing.hpp; sourceTree = ""; }; 4ABB734520BED11500DC0D17 /* PrjFSLogClientShared.h */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.c.h; path = PrjFSLogClientShared.h; sourceTree = ""; }; 4ABB734C20C1A65B00DC0D17 /* PrjFSProviderClientShared.h */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.c.h; path = PrjFSProviderClientShared.h; sourceTree = ""; }; 4AC1D7BE2091FA0400786861 /* PrjFSProviderUserClient.cpp */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.cpp.cpp; path = PrjFSProviderUserClient.cpp; sourceTree = ""; }; @@ -45,6 +47,7 @@ 4AC1D7C22091FA2300786861 /* PrjFSClasses.hpp */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.cpp.h; path = PrjFSClasses.hpp; sourceTree = ""; }; 4AC1D7C32091FBFC00786861 /* PrjFSLogUserClient.cpp */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.cpp.cpp; path = PrjFSLogUserClient.cpp; sourceTree = ""; }; 4AC1D7C42091FBFC00786861 /* PrjFSLogUserClient.hpp */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.cpp.h; path = PrjFSLogUserClient.hpp; sourceTree = ""; }; + 4AE187A721203A890026AC68 /* PerformanceTracing.cpp */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.cpp.cpp; path = PerformanceTracing.cpp; sourceTree = ""; }; C6BDD364208BC60400CB7E58 /* VirtualizationRoots.hpp */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.cpp.h; path = VirtualizationRoots.hpp; sourceTree = ""; }; C6BDD365208BC60400CB7E58 /* VirtualizationRoots.cpp */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.cpp.cpp; path = VirtualizationRoots.cpp; sourceTree = ""; }; C6BDD368208BC99100CB7E58 /* Locks.hpp */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.cpp.h; path = Locks.hpp; sourceTree = ""; }; @@ -124,6 +127,8 @@ C6C780CB207FD02400E7E054 /* KextLog.hpp */, C6C780CC207FD02400E7E054 /* KextLog.cpp */, C6C780B5207FC67200E7E054 /* Info.plist */, + 4ABB734320B867E000DC0D17 /* PerformanceTracing.hpp */, + 4AE187A721203A890026AC68 /* PerformanceTracing.cpp */, C6E9E116208BBB62004A5725 /* KauthHandler.hpp */, C6E9E117208BBB62004A5725 /* KauthHandler.cpp */, C6BDD364208BC60400CB7E58 /* VirtualizationRoots.hpp */, @@ -253,6 +258,7 @@ buildActionMask = 2147483647; files = ( C6BDD36B208BC99100CB7E58 /* Locks.cpp in Sources */, + 4AE187A821203A890026AC68 /* PerformanceTracing.cpp in Sources */, C6BDD367208BC60400CB7E58 /* VirtualizationRoots.cpp in Sources */, C6BDD374208C033200CB7E58 /* Memory.cpp in Sources */, 4AC1D7C52091FBFC00786861 /* PrjFSLogUserClient.cpp in Sources */, diff --git a/ProjFS.Mac/PrjFSKext/PrjFSKext/PerformanceTracing.cpp b/ProjFS.Mac/PrjFSKext/PrjFSKext/PerformanceTracing.cpp new file mode 100644 index 000000000..644cb574c --- /dev/null +++ b/ProjFS.Mac/PrjFSKext/PrjFSKext/PerformanceTracing.cpp @@ -0,0 +1,59 @@ +#include "PerformanceTracing.hpp" +#include +#include +#include + +PerfTracingProbe profile_probes[Probe_Count]; + +void PerfTracing_Init() +{ + for (size_t i = 0; i < Probe_Count; ++i) + { + PerfTracing_ProbeInit(&profile_probes[i]); + } +} + +void PerfTracing_ProbeInit(PerfTracingProbe* probe) +{ + *probe = PerfTracingProbe{ .min = UINT64_MAX }; +} + +IOReturn PerfTracing_ExportDataUserClient(IOExternalMethodArguments* arguments) +{ + if (arguments->structureOutput == nullptr || arguments->structureOutputSize != sizeof(profile_probes)) + { + return kIOReturnBadArgument; + } + + memcpy(arguments->structureOutput, profile_probes, sizeof(profile_probes)); + return kIOReturnSuccess; +} + +void PerfTracing_RecordSample(PerfTracingProbe* probe, uint64_t startTime, uint64_t endTime) +{ + uint64_t interval = endTime - startTime; + + atomic_fetch_add(&probe->numSamples1, 1); + atomic_fetch_add(&probe->sum, interval); + + __uint128_t intervalSquared = interval; + intervalSquared *= intervalSquared; + atomic_fetch_add(&probe->sumSquares, intervalSquared); + + // Update minimum sample if necessary + { + uint64_t oldMin = atomic_load(&probe->min); + while (interval < oldMin && !atomic_compare_exchange_weak(&probe->min, &oldMin, interval)) + {} + } + + // Update maximum sample if necessary + { + uint64_t oldMax = atomic_load(&probe->max); + while (interval > oldMax && !atomic_compare_exchange_weak(&probe->max, &oldMax, interval)) + {} + } + + atomic_fetch_add(&probe->numSamples2, 1); +} + diff --git a/ProjFS.Mac/PrjFSKext/PrjFSKext/PerformanceTracing.hpp b/ProjFS.Mac/PrjFSKext/PrjFSKext/PerformanceTracing.hpp new file mode 100644 index 000000000..4f0099814 --- /dev/null +++ b/ProjFS.Mac/PrjFSKext/PrjFSKext/PerformanceTracing.hpp @@ -0,0 +1,155 @@ +#pragma once + +#include "PrjFSCommon.h" + +#include +#include + +void PerfTracing_Init(); +void PerfTracing_ProbeInit(PerfTracingProbe* probe); +void PerfTracing_RecordSample(PerfTracingProbe* probe, uint64_t startTime, uint64_t endTime); + +struct IOExternalMethodArguments; +IOReturn PerfTracing_ExportDataUserClient(IOExternalMethodArguments* arguments); + +extern PerfTracingProbe profile_probes[Probe_Count]; +// A scope-based manual instrumentation profiling mechanism. +// In the simplest case, the time between construction and destruction of the ProfileSample +// is measured and registered with the probe specified during construction: +// +// void MyFunction() +// { +// ProfileSample functionSample(Probe_MyFunction); +// // ... The code for which we're measuring the runtime ... +// } // <-- functionSample goes out of scope here, so that's when timing ends +// +// +// To allow runtimes different code paths in the same scope to be recorded separately, the +// probe identity can be modified using SetProbe(): +// +// void MyFunction() +// { +// ProfileSample functionSample(Probe_MyFunction); +// // ... The code for which we're measuring the runtime ... +// if (specialCase) +// { +// // We want to be able to distinguish between the runtimes of MyFunction for this +// // special case vs "normal" runs. +// functionSample.SetProbe(Probe_MyFunctionSpecialCase); +// // ... do something potentially expensive here ... +// } +// // ... more code ... +// } // <-- Runtime to here will be recorded either under Probe_MyFunction or Probe_MyFunctionSpecialCase +// +// +// For tracing sub-sections of code, such as the special case logic above, in isolation, +// we have 2 options: taking additional scoped samples, or split timings. Scoped samples are +// easier to understand: +// +// void MyFunction() +// { +// ProfileSample functionSample(Probe_MyFunction); +// // ... +// if (specialCase) +// { +// // Measure only the special case code on its own: +// ProfileSample specialCaseSample(Probe_MyFunctionSpecialCase); +// // ... do something potentially expensive here ... +// } // <-- scope of specialCaseSample ends here +// // ... more code ... +// } // <-- end of Probe_MyFunction in all cases +// +// In the above example, the runtimes of all MyFunction() calls are recorded under Probe_MyFunction, +// while the special case code on its own is recorded in Probe_MyFunctionSpecialCase. +// +// Taking split timings meanwhile allows us to carve up scoped samples into constituent sub-samples, +// useful for drilling down to find the source of performance issues: +// +// void MyFunction() +// { +// ProfileSample functionSample(Probe_MyFunction); +// // ... +// // The time from the creation of functionSample to this point is recorded as Probe_MyFunctionPart1. +// functionSample.TakeSplitSample(Probe_MyFunctionPart1, Probe_MyFunctionRemainder); +// if (specialCase) +// { +// // ... do something potentially expensive here ... +// functionSample.TakeSplitSample(Probe_MyFunctionSpecialCase); // This measures time since the Part1 split +// } // <-- scope of specialCaseSample ends here +// // ... more code ... +// } // <-- end of Probe_MyFunction in all cases; the time since the last split (Probe_MyFunctionPart1 +// // or Probe_MyFunctionSpecialCase depending on code path) is recorded as Probe_MyFunctionRemainder. +// +// The end time stamp for a split is taken as the start of the next split, and the overall start and end +// stamps of the scoped sample are alse the start and end of the first and last (remainder) split, +// respectively. +// So in this case, the sum total runtime of all samples of Probe_MyFunction is exactly equal to the sum of +// Probe_MyFunctionPart1 + Probe_MyFunctionSpecialCase + Probe_MyFunctionRemainder. +// Note that Probe_MyFunctionSpecialCase may have a lower sample count. +// Note also that the "remainder" split is optional - if only the 1-argument version of TakeSplitSample +// is used, the split time to the end of scope is not recorded. (And like the scoped sample, it can be changed, +// in this case using SetFinalSplitProbe()) +class ProfileSample +{ + ProfileSample(const ProfileSample&) = delete; + ProfileSample() = delete; + + const uint64_t startTimestamp; + PrjFS_PerfCounter wholeSampleProbe; + PrjFS_PerfCounter finalSplitProbe; + uint64_t splitTimestamp; + +public: + inline ProfileSample(PrjFS_PerfCounter defaultProbe); + inline void SetProbe(PrjFS_PerfCounter probe); + inline void TakeSplitSample(PrjFS_PerfCounter splitProbe); + inline void TakeSplitSample(PrjFS_PerfCounter splitProbe, PrjFS_PerfCounter newFinalSplitProbe); + inline void SetFinalSplitProbe(PrjFS_PerfCounter newFinalSplitProbe); + inline ~ProfileSample(); +}; + +ProfileSample::~ProfileSample() +{ + uint64_t endTimestamp = mach_absolute_time(); + if (this->wholeSampleProbe != Probe_None) + { + PerfTracing_RecordSample(&profile_probes[this->wholeSampleProbe], this->startTimestamp, endTimestamp); + } + + if (this->finalSplitProbe != Probe_None) + { + PerfTracing_RecordSample(&profile_probes[this->finalSplitProbe], this->splitTimestamp, endTimestamp); + } +}; + +void ProfileSample::TakeSplitSample(PrjFS_PerfCounter splitProbe) +{ + uint64_t newSplitTimestamp = mach_absolute_time(); + PerfTracing_RecordSample(&profile_probes[splitProbe], this->splitTimestamp, newSplitTimestamp); + this->splitTimestamp = newSplitTimestamp; +} + +void ProfileSample::TakeSplitSample(PrjFS_PerfCounter splitProbe, PrjFS_PerfCounter newFinalSplitProbe) +{ + this->TakeSplitSample(splitProbe); + this->finalSplitProbe = newFinalSplitProbe; +} + +void ProfileSample::SetFinalSplitProbe(PrjFS_PerfCounter newFinalSplitProbe) +{ + this->finalSplitProbe = newFinalSplitProbe; +} + +ProfileSample::ProfileSample(PrjFS_PerfCounter defaultProbe) : + startTimestamp(mach_absolute_time()), + wholeSampleProbe(defaultProbe), + finalSplitProbe(Probe_None), + splitTimestamp(this->startTimestamp) +{ +} + +void ProfileSample::SetProbe(PrjFS_PerfCounter probe) +{ + this->wholeSampleProbe = probe; +} + diff --git a/ProjFS.Mac/PrjFSKext/PrjFSKext/PrjFSKext.cpp b/ProjFS.Mac/PrjFSKext/PrjFSKext/PrjFSKext.cpp index 508545bcd..97a8ee6e5 100644 --- a/ProjFS.Mac/PrjFSKext/PrjFSKext/PrjFSKext.cpp +++ b/ProjFS.Mac/PrjFSKext/PrjFSKext/PrjFSKext.cpp @@ -5,12 +5,15 @@ #include "KauthHandler.hpp" #include "Locks.hpp" #include "Memory.hpp" +#include "PerformanceTracing.hpp" extern "C" kern_return_t PrjFSKext_Start(kmod_info_t* ki, void* d); extern "C" kern_return_t PrjFSKext_Stop(kmod_info_t* ki, void* d); kern_return_t PrjFSKext_Start(kmod_info_t* ki, void* d) { + PerfTracing_Init(); + if (Locks_Init()) { goto CleanupAndFail; diff --git a/ProjFS.Mac/PrjFSKext/PrjFSKext/PrjFSLogUserClient.cpp b/ProjFS.Mac/PrjFSKext/PrjFSKext/PrjFSLogUserClient.cpp index b39b73bf9..69e59889d 100644 --- a/ProjFS.Mac/PrjFSKext/PrjFSKext/PrjFSLogUserClient.cpp +++ b/ProjFS.Mac/PrjFSKext/PrjFSKext/PrjFSLogUserClient.cpp @@ -2,6 +2,7 @@ #include "PrjFSLogClientShared.h" #include "KextLog.hpp" #include "PrjFSCommon.h" +#include "PerformanceTracing.hpp" #include @@ -9,6 +10,20 @@ OSDefineMetaClassAndStructors(PrjFSLogUserClient, IOUserClient); // Amount of memory to set aside for kernel -> userspace log messages. static const uint32_t LogMessageQueueCapacityBytes = 1024 * 1024; + +static const IOExternalMethodDispatch LogUserClientDispatch[] = +{ + [LogSelector_FetchProfilingData] = + { + .function = &PrjFSLogUserClient::fetchProfilingData, + .checkScalarInputCount = 0, + .checkStructureInputSize = 0, + .checkScalarOutputCount = 0, + .checkStructureOutputSize = Probe_Count * sizeof(PerfTracingProbe), // array of probes + }, +}; + + bool PrjFSLogUserClient::initWithTask( task_t owningTask, void* securityToken, @@ -129,3 +144,31 @@ void PrjFSLogUserClient::sendLogMessage(KextLog_MessageHeader* message, uint32_t Mutex_Release(this->dataQueueWriterMutex); } +IOReturn PrjFSLogUserClient::externalMethod( + uint32_t selector, + IOExternalMethodArguments* arguments, + IOExternalMethodDispatch* dispatch, + OSObject* target, + void* reference) +{ + IOExternalMethodDispatch local_dispatch = {}; + if (selector < sizeof(LogUserClientDispatch) / sizeof(LogUserClientDispatch[0])) + { + if (nullptr != LogUserClientDispatch[selector].function) + { + local_dispatch = LogUserClientDispatch[selector]; + dispatch = &local_dispatch; + target = this; + } + } + return this->super::externalMethod(selector, arguments, dispatch, target, reference); +} + +IOReturn PrjFSLogUserClient::fetchProfilingData( + OSObject* target, + void* reference, + IOExternalMethodArguments* arguments) +{ + return PerfTracing_ExportDataUserClient(arguments); +} + diff --git a/ProjFS.Mac/PrjFSKext/PrjFSKext/PrjFSLogUserClient.hpp b/ProjFS.Mac/PrjFSKext/PrjFSKext/PrjFSLogUserClient.hpp index a91d456a8..28c3bb66e 100644 --- a/ProjFS.Mac/PrjFSKext/PrjFSKext/PrjFSLogUserClient.hpp +++ b/ProjFS.Mac/PrjFSKext/PrjFSKext/PrjFSLogUserClient.hpp @@ -26,5 +26,18 @@ class PrjFSLogUserClient : public IOUserClient virtual IOReturn clientMemoryForType(UInt32 type, IOOptionBits* options, IOMemoryDescriptor** memory) override; virtual IOReturn registerNotificationPort(mach_port_t port, UInt32 type, io_user_reference_t refCon) override; + virtual IOReturn externalMethod( + uint32_t selector, + IOExternalMethodArguments* arguments, + IOExternalMethodDispatch* dispatch = nullptr, + OSObject* target = nullptr, + void* reference = nullptr) override; + + + static IOReturn fetchProfilingData( + OSObject* target, + void* reference, + IOExternalMethodArguments* arguments); + void sendLogMessage(KextLog_MessageHeader* message, uint32_t size); }; diff --git a/ProjFS.Mac/PrjFSKext/public/PrjFSCommon.h b/ProjFS.Mac/PrjFSKext/public/PrjFSCommon.h index f7478ece9..35b386b44 100644 --- a/ProjFS.Mac/PrjFSKext/public/PrjFSCommon.h +++ b/ProjFS.Mac/PrjFSKext/public/PrjFSCommon.h @@ -33,4 +33,29 @@ enum PrjFSServiceUserClientType UserClientType_Log, }; +enum PrjFS_PerfCounter : int32_t +{ + // Note: ensure that any changes to this list are reflected in the PerfCounterNames array of strings + + + Probe_Count, + + Probe_None = -1 +}; + +struct PerfTracingProbe +{ + _Atomic uint64_t numSamples1; + _Atomic uint64_t numSamples2; + // Units: Mach absolute time (squared for sumSquares) + // Sum of measured sample intervals + _Atomic uint64_t sum; + // Smallest encountered interval + _Atomic uint64_t min; + // Largest encountered interval + _Atomic uint64_t max; + // Sum-of-squares of measured time intervals (for stddev) + _Atomic __uint128_t sumSquares; +}; + #endif /* PrjFSCommon_h */ diff --git a/ProjFS.Mac/PrjFSKext/public/PrjFSLogClientShared.h b/ProjFS.Mac/PrjFSKext/public/PrjFSLogClientShared.h index ca847b4e9..957a86a73 100644 --- a/ProjFS.Mac/PrjFSKext/public/PrjFSLogClientShared.h +++ b/ProjFS.Mac/PrjFSKext/public/PrjFSLogClientShared.h @@ -2,6 +2,14 @@ #include +// External method selectors for log user clients +enum PrjFSLogUserClientSelector +{ + LogSelector_Invalid = 0, + + LogSelector_FetchProfilingData, +}; + enum PrjFSLogUserClientMemoryType { LogMemoryType_Invalid = 0, diff --git a/ProjFS.Mac/PrjFSLib/PrjFSLib.xcodeproj/project.pbxproj b/ProjFS.Mac/PrjFSLib/PrjFSLib.xcodeproj/project.pbxproj index 9bfd51e90..f88b6cc2b 100644 --- a/ProjFS.Mac/PrjFSLib/PrjFSLib.xcodeproj/project.pbxproj +++ b/ProjFS.Mac/PrjFSLib/PrjFSLib.xcodeproj/project.pbxproj @@ -9,6 +9,7 @@ /* Begin PBXBuildFile section */ 4A440DDE2093AD3300AADA76 /* IOKit.framework in Frameworks */ = {isa = PBXBuildFile; fileRef = 4A440DDD2093AD3300AADA76 /* IOKit.framework */; }; 4A8A1BEE20A0D5940024BC10 /* CoreFoundation.framework in Frameworks */ = {isa = PBXBuildFile; fileRef = 4A8A1BED20A0D5940024BC10 /* CoreFoundation.framework */; }; + 4A91E086215E76A90079FE1B /* kext-perf-tracing.cpp in Sources */ = {isa = PBXBuildFile; fileRef = 4A91E084215E76A90079FE1B /* kext-perf-tracing.cpp */; }; C6C780D120816BDC00E7E054 /* PrjFSLib.h in Headers */ = {isa = PBXBuildFile; fileRef = C6C780CF20816BDC00E7E054 /* PrjFSLib.h */; }; C6C780D220816BDC00E7E054 /* PrjFSLib.cpp in Sources */ = {isa = PBXBuildFile; fileRef = C6C780D020816BDC00E7E054 /* PrjFSLib.cpp */; }; D308478120B4431200F69E92 /* prjfs-log.cpp in Sources */ = {isa = PBXBuildFile; fileRef = D308478020B4431200F69E92 /* prjfs-log.cpp */; }; @@ -34,6 +35,8 @@ /* Begin PBXFileReference section */ 4A440DDD2093AD3300AADA76 /* IOKit.framework */ = {isa = PBXFileReference; lastKnownFileType = wrapper.framework; name = IOKit.framework; path = System/Library/Frameworks/IOKit.framework; sourceTree = SDKROOT; }; 4A8A1BED20A0D5940024BC10 /* CoreFoundation.framework */ = {isa = PBXFileReference; lastKnownFileType = wrapper.framework; name = CoreFoundation.framework; path = System/Library/Frameworks/CoreFoundation.framework; sourceTree = SDKROOT; }; + 4A91E084215E76A90079FE1B /* kext-perf-tracing.cpp */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.cpp.cpp; path = "kext-perf-tracing.cpp"; sourceTree = ""; }; + 4A91E085215E76A90079FE1B /* kext-perf-tracing.hpp */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.cpp.h; path = "kext-perf-tracing.hpp"; sourceTree = ""; }; C6C780C4207FC6AB00E7E054 /* libPrjFSLib.dylib */ = {isa = PBXFileReference; explicitFileType = "compiled.mach-o.dylib"; includeInIndex = 0; path = libPrjFSLib.dylib; sourceTree = BUILT_PRODUCTS_DIR; }; C6C780CF20816BDC00E7E054 /* PrjFSLib.h */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.c.h; path = PrjFSLib.h; sourceTree = ""; }; C6C780D020816BDC00E7E054 /* PrjFSLib.cpp */ = {isa = PBXFileReference; indentWidth = 4; lastKnownFileType = sourcecode.cpp.cpp; path = PrjFSLib.cpp; sourceTree = ""; tabWidth = 4; usesTabs = 0; }; @@ -103,6 +106,8 @@ isa = PBXGroup; children = ( D308478020B4431200F69E92 /* prjfs-log.cpp */, + 4A91E085215E76A90079FE1B /* kext-perf-tracing.hpp */, + 4A91E084215E76A90079FE1B /* kext-perf-tracing.cpp */, ); path = "prjfs-log"; sourceTree = ""; @@ -207,6 +212,7 @@ buildActionMask = 2147483647; files = ( D308478920B4432500F69E92 /* PrjFSUser.cpp in Sources */, + 4A91E086215E76A90079FE1B /* kext-perf-tracing.cpp in Sources */, D308478120B4431200F69E92 /* prjfs-log.cpp in Sources */, ); runOnlyForDeploymentPostprocessing = 0; diff --git a/ProjFS.Mac/PrjFSLib/prjfs-log/kext-perf-tracing.cpp b/ProjFS.Mac/PrjFSLib/prjfs-log/kext-perf-tracing.cpp new file mode 100644 index 000000000..30d46eda4 --- /dev/null +++ b/ProjFS.Mac/PrjFSLib/prjfs-log/kext-perf-tracing.cpp @@ -0,0 +1,65 @@ +#include "kext-perf-tracing.hpp" +#include "../../PrjFSKext/public/PrjFSCommon.h" +#include "../../PrjFSKext/public/PrjFSLogClientShared.h" +#include +#include +#include +#include +#include + +static mach_timebase_info_data_t s_machTimebase; + +static uint64_t nanosecondsFromAbsoluteTime(uint64_t machAbsoluteTime) +{ + return static_cast<__uint128_t>(machAbsoluteTime) * s_machTimebase.numer / s_machTimebase.denom; +} + +static const char* const PerfCounterNames[Probe_Count] = +{ +}; + +bool PrjFSLog_FetchAndPrintKextProfilingData(io_connect_t connection) +{ + static dispatch_once_t onceToken; + dispatch_once(&onceToken, ^{ + mach_timebase_info(&s_machTimebase); + }); + + PerfTracingProbe probes[Probe_Count]; + size_t out_size = sizeof(probes); + IOReturn ret = IOConnectCallStructMethod(connection, LogSelector_FetchProfilingData, nullptr, 0, probes, &out_size); + if (ret == kIOReturnUnsupported) + { + return false; + } + else if (ret == kIOReturnSuccess) + { + for (unsigned i = 0; i < Probe_Count; ++i) + { + double samples = probes[i].numSamples1; + double sum_abs = probes[i].sum; + double stddev_abs = samples > 1 ? sqrt((samples * probes[i].sumSquares - sum_abs * sum_abs) / (samples * (samples - 1))) : 0.0; + + double sum_ns = nanosecondsFromAbsoluteTime(sum_abs); + double stddev_ns = nanosecondsFromAbsoluteTime(stddev_abs); + double mean_ns = samples > 0 ? sum_ns / samples : 0; + printf("%2u %40s %8llu [%8llu] samples, total time: %15.0f ns, mean: %10.2f ns +/- %11.2f", + i, PerfCounterNames[i], probes[i].numSamples1, probes[i].numSamples2, sum_ns, mean_ns, stddev_ns); + if (probes[i].min != UINT64_MAX) + { + printf(", min: %7llu ns, max: %10llu ns\n", nanosecondsFromAbsoluteTime(probes[i].min), nanosecondsFromAbsoluteTime(probes[i].max)); + } + else + { + printf("\n"); + } + } + } + else + { + fprintf(stderr, "fetching profiling data from kernel failed: 0x%x\n", ret); + return false; + } + fflush(stdout); + return true; +} diff --git a/ProjFS.Mac/PrjFSLib/prjfs-log/kext-perf-tracing.hpp b/ProjFS.Mac/PrjFSLib/prjfs-log/kext-perf-tracing.hpp new file mode 100644 index 000000000..fedbf92da --- /dev/null +++ b/ProjFS.Mac/PrjFSLib/prjfs-log/kext-perf-tracing.hpp @@ -0,0 +1,5 @@ +#pragma once + +#include + +bool PrjFSLog_FetchAndPrintKextProfilingData(io_connect_t connection); diff --git a/ProjFS.Mac/PrjFSLib/prjfs-log/prjfs-log.cpp b/ProjFS.Mac/PrjFSLib/prjfs-log/prjfs-log.cpp index 281fc2bd3..aaa32b1c4 100644 --- a/ProjFS.Mac/PrjFSLib/prjfs-log/prjfs-log.cpp +++ b/ProjFS.Mac/PrjFSLib/prjfs-log/prjfs-log.cpp @@ -1,4 +1,5 @@ #include "../PrjFSUser.hpp" +#include "kext-perf-tracing.hpp" #include "../../PrjFSKext/public/PrjFSLogClientShared.h" #include #include @@ -7,10 +8,12 @@ #include static const char* KextLogLevelAsString(KextLog_Level level); -static uint64_t nanosecondsFromAbsoluteTime(uint64_t machAbsoluteTime); +static uint64_t NanosecondsFromAbsoluteTime(uint64_t machAbsoluteTime); +static dispatch_source_t StartKextProfilingDataPolling(io_connect_t connection); static mach_timebase_info_data_t s_machTimebase; + int main(int argc, const char * argv[]) { mach_timebase_info(&s_machTimebase); @@ -55,7 +58,7 @@ int main(int argc, const char * argv[]) const char* messageType = KextLogLevelAsString(message.level); int logStringLength = messageSize - sizeof(KextLog_MessageHeader) - 1; - uint64_t timeOffsetNS = nanosecondsFromAbsoluteTime(message.machAbsoluteTimestamp - machStartTime); + uint64_t timeOffsetNS = NanosecondsFromAbsoluteTime(message.machAbsoluteTimestamp - machStartTime); uint64_t timeOffsetMS = timeOffsetNS / NSEC_PER_MSEC; printf("(%d: %5llu.%03llu) %s: %.*s\n", lineCount, timeOffsetMS / 1000u, timeOffsetMS % 1000u, messageType, logStringLength, entry->data + sizeof(KextLog_MessageHeader)); @@ -66,8 +69,21 @@ int main(int argc, const char * argv[]) } }); dispatch_resume(dataQueue.dispatchSource); + + dispatch_source_t timer = nullptr; + if (PrjFSLog_FetchAndPrintKextProfilingData(connection)) + { + timer = StartKextProfilingDataPolling(connection); + } + CFRunLoopRun(); + if (nullptr != timer) + { + dispatch_cancel(timer); + dispatch_release(timer); + } + return 0; } @@ -86,7 +102,19 @@ static const char* KextLogLevelAsString(KextLog_Level level) } } -static uint64_t nanosecondsFromAbsoluteTime(uint64_t machAbsoluteTime) +static uint64_t NanosecondsFromAbsoluteTime(uint64_t machAbsoluteTime) { return static_cast<__uint128_t>(machAbsoluteTime) * s_machTimebase.numer / s_machTimebase.denom; } + +static dispatch_source_t StartKextProfilingDataPolling(io_connect_t connection) +{ + dispatch_source_t timer = dispatch_source_create(DISPATCH_SOURCE_TYPE_TIMER, 0, 0, dispatch_get_main_queue()); + dispatch_source_set_timer(timer, DISPATCH_TIME_NOW, 15 * NSEC_PER_SEC, 10 * NSEC_PER_SEC); + dispatch_source_set_event_handler(timer, ^{ + PrjFSLog_FetchAndPrintKextProfilingData(connection); + }); + dispatch_resume(timer); + return timer; +} + From 3c86170f8f4fab252af852d12aa5538964a5ec2f Mon Sep 17 00:00:00 2001 From: Phil Dennis-Jordan Date: Fri, 5 Oct 2018 12:11:54 +0200 Subject: [PATCH 247/272] Mac kext: Incorporates code review style feedback In the review, it was pointed out we should be using nullptr instead of 0 for default function argument values which were copied from the declaration of the function being overridden in the log user client. The same is true for the provider user client, so this fixes it there too. --- ProjFS.Mac/PrjFSKext/PrjFSKext/PrjFSProviderUserClient.hpp | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/ProjFS.Mac/PrjFSKext/PrjFSKext/PrjFSProviderUserClient.hpp b/ProjFS.Mac/PrjFSKext/PrjFSKext/PrjFSProviderUserClient.hpp index de487108c..650b5dc0f 100644 --- a/ProjFS.Mac/PrjFSKext/PrjFSKext/PrjFSProviderUserClient.hpp +++ b/ProjFS.Mac/PrjFSKext/PrjFSKext/PrjFSProviderUserClient.hpp @@ -30,9 +30,9 @@ class PrjFSProviderUserClient : public IOUserClient virtual IOReturn externalMethod( uint32_t selector, IOExternalMethodArguments* arguments, - IOExternalMethodDispatch* dispatch = 0, - OSObject* target = 0, - void* reference = 0) override; + IOExternalMethodDispatch* dispatch = nullptr, + OSObject* target = nullptr, + void* reference = nullptr) override; virtual IOReturn clientMemoryForType( UInt32 type, IOOptionBits* options, From 5f7cc9721b8fff584c8da2991ddf6301bc86aa40 Mon Sep 17 00:00:00 2001 From: Phil Dennis-Jordan Date: Sat, 29 Sep 2018 11:41:51 +0200 Subject: [PATCH 248/272] Mac kext: Placement of profiling probes throughout the vnode listener Using the previously created profiling mechanism, this adds various measurement points to the vnode scope listener handler, and the relevant strings to the user space extraction tool. --- .../PrjFSKext/PrjFSKext/KauthHandler.cpp | 29 ++++++++++++++++++- .../PrjFSKext/VirtualizationRoots.cpp | 5 ++++ ProjFS.Mac/PrjFSKext/public/PrjFSCommon.h | 18 ++++++++++++ .../PrjFSLib/prjfs-log/kext-perf-tracing.cpp | 17 +++++++++++ 4 files changed, 68 insertions(+), 1 deletion(-) diff --git a/ProjFS.Mac/PrjFSKext/PrjFSKext/KauthHandler.cpp b/ProjFS.Mac/PrjFSKext/PrjFSKext/KauthHandler.cpp index 111c9355d..50250b631 100644 --- a/ProjFS.Mac/PrjFSKext/PrjFSKext/KauthHandler.cpp +++ b/ProjFS.Mac/PrjFSKext/PrjFSKext/KauthHandler.cpp @@ -13,6 +13,7 @@ #include "Message.h" #include "Locks.hpp" #include "PrjFSProviderUserClient.hpp" +#include "PerformanceTracing.hpp" // Function prototypes static int HandleVnodeOperation( @@ -61,6 +62,7 @@ static bool ShouldHandleVnodeOpEvent( vfs_context_t context, const vnode_t vnode, kauth_action_t action, + ProfileSample& operationSample, // Out params: VirtualizationRootHandle* root, @@ -252,6 +254,8 @@ static int HandleVnodeOperation( uintptr_t arg3) { atomic_fetch_add(&s_numActiveKauthEvents, 1); + + ProfileSample functionSample(Probe_VnodeOp); vfs_context_t context = reinterpret_cast(arg0); vnode_t currentVnode = reinterpret_cast(arg1); @@ -296,6 +300,7 @@ static int HandleVnodeOperation( context, currentVnode, action, + functionSample, &root, &vnodeType, ¤tVnodeFileFlags, @@ -343,6 +348,8 @@ static int HandleVnodeOperation( // Recursively expand directory on delete to ensure child placeholders are created before rename operations if (isDeleteAction || FileFlagsBitIsSet(currentVnodeFileFlags, FileFlags_IsEmpty)) { + functionSample.SetProbe(Probe_VnodeOp_PopulatePlaceholderDirectory); + if (!TrySendRequestAndWaitForResponse( root, isDeleteAction ? @@ -376,6 +383,8 @@ static int HandleVnodeOperation( { if (FileFlagsBitIsSet(currentVnodeFileFlags, FileFlags_IsEmpty)) { + functionSample.SetProbe(Probe_VnodeOp_HydratePlaceholderFile); + if (!TrySendRequestAndWaitForResponse( root, MessageType_KtoU_HydrateFile, @@ -416,6 +425,8 @@ static int HandleFileOpOperation( { atomic_fetch_add(&s_numActiveKauthEvents, 1); + ProfileSample functionSample(Probe_FileOp); + vfs_context_t context = vfs_context_create(NULL); vnode_t currentVnodeFromPath = NULLVP; @@ -567,6 +578,7 @@ static bool ShouldHandleVnodeOpEvent( vfs_context_t context, const vnode_t vnode, kauth_action_t action, + ProfileSample& operationSample, // Out params: VirtualizationRootHandle* root, @@ -582,6 +594,7 @@ static bool ShouldHandleVnodeOpEvent( if (!VirtualizationRoot_VnodeIsOnAllowedFilesystem(vnode)) { + operationSample.SetProbe(Probe_Op_EarlyOut); *kauthResult = KAUTH_RESULT_DEFER; return false; } @@ -589,16 +602,22 @@ static bool ShouldHandleVnodeOpEvent( *vnodeType = vnode_vtype(vnode); if (ShouldIgnoreVnodeType(*vnodeType, vnode)) { + operationSample.SetProbe(Probe_Op_EarlyOut); *kauthResult = KAUTH_RESULT_DEFER; return false; } - *vnodeFileFlags = ReadVNodeFileFlags(vnode, context); + { + ProfileSample readflags(Probe_ReadFileFlags); + *vnodeFileFlags = ReadVNodeFileFlags(vnode, context); + } + if (!FileFlagsBitIsSet(*vnodeFileFlags, FileFlags_IsInVirtualizationRoot)) { // This vnode is not part of ANY virtualization root, so exit now before doing any more work. // This gives us a cheap way to avoid adding overhead to IO outside of a virtualization root. + operationSample.SetProbe(Probe_Op_NoVirtualizationRootFlag); *kauthResult = KAUTH_RESULT_DEFER; return false; } @@ -618,14 +637,21 @@ static bool ShouldHandleVnodeOpEvent( // get called again, so we lose the opportunity to hydrate the file/directory and it will appear as though // it is missing its contents. + operationSample.SetProbe(Probe_Op_DenyCrawler); *kauthResult = KAUTH_RESULT_DENY; return false; } + + operationSample.SetProbe(Probe_Op_EmptyFlag); } + operationSample.TakeSplitSample(Probe_Op_IdentifySplit); + *vnodeFsidInode = Vnode_GetFsidAndInode(vnode, context); *root = VirtualizationRoot_FindForVnode(vnode, *vnodeFsidInode); + operationSample.TakeSplitSample(Probe_Op_VirtualizationRootFindSplit); + if (RootHandle_ProviderTemporaryDirectory == *root) { *kauthResult = KAUTH_RESULT_DEFER; @@ -650,6 +676,7 @@ static bool ShouldHandleVnodeOpEvent( // If the calling process is the provider, we must exit right away to avoid deadlocks if (VirtualizationRoot_PIDMatchesProvider(*root, *pid)) { + operationSample.SetProbe(Probe_Op_Provider); *kauthResult = KAUTH_RESULT_DEFER; return false; } diff --git a/ProjFS.Mac/PrjFSKext/PrjFSKext/VirtualizationRoots.cpp b/ProjFS.Mac/PrjFSKext/PrjFSKext/VirtualizationRoots.cpp index 88b0a97db..4949e357b 100644 --- a/ProjFS.Mac/PrjFSKext/PrjFSKext/VirtualizationRoots.cpp +++ b/ProjFS.Mac/PrjFSKext/PrjFSKext/VirtualizationRoots.cpp @@ -11,6 +11,7 @@ #include "PrjFSProviderUserClient.hpp" #include "kernel-header-wrappers/mount.h" #include "VnodeUtilities.hpp" +#include "PerformanceTracing.hpp" struct VirtualizationRoot @@ -133,12 +134,16 @@ kern_return_t VirtualizationRoots_Cleanup() VirtualizationRootHandle VirtualizationRoot_FindForVnode(vnode_t vnode, const FsidInode& vnodeFsidInode) { + ProfileSample functionSample(Probe_VirtualizationRoot_Find); + VirtualizationRootHandle rootHandle = RootHandle_None; vnode_get(vnode); // Search up the tree until we hit a known virtualization root or THE root of the file system while (RootHandle_None == rootHandle && NULLVP != vnode && !vnode_isvroot(vnode)) { + ProfileSample iterationSample(Probe_VirtualizationRoot_FindIteration); + rootHandle = FindOrDetectRootAtVnode(vnode, nullptr /* vfs context */, vnodeFsidInode); // Note: if FindOrDetectRootAtVnode returns a "special" handle other // than RootHandle_None, we want to stop the search and return that. diff --git a/ProjFS.Mac/PrjFSKext/public/PrjFSCommon.h b/ProjFS.Mac/PrjFSKext/public/PrjFSCommon.h index 35b386b44..fe44364fb 100644 --- a/ProjFS.Mac/PrjFSKext/public/PrjFSCommon.h +++ b/ProjFS.Mac/PrjFSKext/public/PrjFSCommon.h @@ -37,6 +37,24 @@ enum PrjFS_PerfCounter : int32_t { // Note: ensure that any changes to this list are reflected in the PerfCounterNames array of strings + Probe_VnodeOp = 0, + Probe_FileOp, + Probe_Op_EarlyOut, + Probe_Op_NoVirtualizationRootFlag, + Probe_Op_EmptyFlag, + Probe_Op_DenyCrawler, + Probe_Op_Offline, + Probe_Op_Provider, + Probe_VnodeOp_PopulatePlaceholderDirectory, + Probe_VnodeOp_HydratePlaceholderFile, + + Probe_Op_IdentifySplit, + Probe_Op_VirtualizationRootFindSplit, + + Probe_ReadFileFlags, + + Probe_VirtualizationRoot_Find, + Probe_VirtualizationRoot_FindIteration, Probe_Count, diff --git a/ProjFS.Mac/PrjFSLib/prjfs-log/kext-perf-tracing.cpp b/ProjFS.Mac/PrjFSLib/prjfs-log/kext-perf-tracing.cpp index 30d46eda4..45a519a19 100644 --- a/ProjFS.Mac/PrjFSLib/prjfs-log/kext-perf-tracing.cpp +++ b/ProjFS.Mac/PrjFSLib/prjfs-log/kext-perf-tracing.cpp @@ -16,6 +16,23 @@ static uint64_t nanosecondsFromAbsoluteTime(uint64_t machAbsoluteTime) static const char* const PerfCounterNames[Probe_Count] = { + [Probe_VnodeOp] = "VnodeOp", + [Probe_FileOp] = "FileOp", + [Probe_Op_EarlyOut] = "Op_EarlyOut", + [Probe_Op_NoVirtualizationRootFlag] = "Op_NoVirtualizationRootFlag", + [Probe_Op_EmptyFlag] = "Op_EmptyFlag", + [Probe_Op_DenyCrawler] = "Op_DenyCrawler", + [Probe_Op_Offline] = "Op_Offline", + [Probe_Op_Provider] = "Op_Provider", + [Probe_VnodeOp_PopulatePlaceholderDirectory] = "VnodeOp_PopulatePlaceholderDirectory", + [Probe_VnodeOp_HydratePlaceholderFile] = "VnodeOp_HydratePlaceholderFile", + + [Probe_Op_IdentifySplit] = "Op_IdentifySplit", + [Probe_Op_VirtualizationRootFindSplit] = "Op_VirtualizationRootFindSplit", + + [Probe_ReadFileFlags] = "Probe_ReadFileFlags", + [Probe_VirtualizationRoot_Find] = "VirtualizationRoot_Find", + [Probe_VirtualizationRoot_FindIteration] = "VirtualizationRoot_FindIteration", }; bool PrjFSLog_FetchAndPrintKextProfilingData(io_connect_t connection) From b26859971b42772abfbe5a784d861ef8e0d3f121 Mon Sep 17 00:00:00 2001 From: Phil Dennis-Jordan Date: Fri, 28 Sep 2018 19:16:57 +0200 Subject: [PATCH 249/272] Mac kext: Conditionally enable performance tracing. --- .../PrjFSKext/PrjFSKext/PerformanceTracing.cpp | 4 ++++ .../PrjFSKext/PrjFSKext/PerformanceTracing.hpp | 18 +++++++++++++++++- 2 files changed, 21 insertions(+), 1 deletion(-) diff --git a/ProjFS.Mac/PrjFSKext/PrjFSKext/PerformanceTracing.cpp b/ProjFS.Mac/PrjFSKext/PrjFSKext/PerformanceTracing.cpp index 644cb574c..738ee5c79 100644 --- a/ProjFS.Mac/PrjFSKext/PrjFSKext/PerformanceTracing.cpp +++ b/ProjFS.Mac/PrjFSKext/PrjFSKext/PerformanceTracing.cpp @@ -20,6 +20,7 @@ void PerfTracing_ProbeInit(PerfTracingProbe* probe) IOReturn PerfTracing_ExportDataUserClient(IOExternalMethodArguments* arguments) { +#if PRJFS_PERFORMANCE_TRACING_ENABLE if (arguments->structureOutput == nullptr || arguments->structureOutputSize != sizeof(profile_probes)) { return kIOReturnBadArgument; @@ -27,6 +28,9 @@ IOReturn PerfTracing_ExportDataUserClient(IOExternalMethodArguments* arguments) memcpy(arguments->structureOutput, profile_probes, sizeof(profile_probes)); return kIOReturnSuccess; +#else + return kIOReturnUnsupported; +#endif } void PerfTracing_RecordSample(PerfTracingProbe* probe, uint64_t startTime, uint64_t endTime) diff --git a/ProjFS.Mac/PrjFSKext/PrjFSKext/PerformanceTracing.hpp b/ProjFS.Mac/PrjFSKext/PrjFSKext/PerformanceTracing.hpp index 4f0099814..664f1d5e3 100644 --- a/ProjFS.Mac/PrjFSKext/PrjFSKext/PerformanceTracing.hpp +++ b/ProjFS.Mac/PrjFSKext/PrjFSKext/PerformanceTracing.hpp @@ -94,10 +94,12 @@ class ProfileSample ProfileSample(const ProfileSample&) = delete; ProfileSample() = delete; +#if PRJFS_PERFORMANCE_TRACING_ENABLE const uint64_t startTimestamp; PrjFS_PerfCounter wholeSampleProbe; PrjFS_PerfCounter finalSplitProbe; uint64_t splitTimestamp; +#endif public: inline ProfileSample(PrjFS_PerfCounter defaultProbe); @@ -110,6 +112,7 @@ class ProfileSample ProfileSample::~ProfileSample() { +#if PRJFS_PERFORMANCE_TRACING_ENABLE uint64_t endTimestamp = mach_absolute_time(); if (this->wholeSampleProbe != Probe_None) { @@ -120,36 +123,49 @@ ProfileSample::~ProfileSample() { PerfTracing_RecordSample(&profile_probes[this->finalSplitProbe], this->splitTimestamp, endTimestamp); } +#endif }; void ProfileSample::TakeSplitSample(PrjFS_PerfCounter splitProbe) { +#if PRJFS_PERFORMANCE_TRACING_ENABLE uint64_t newSplitTimestamp = mach_absolute_time(); PerfTracing_RecordSample(&profile_probes[splitProbe], this->splitTimestamp, newSplitTimestamp); this->splitTimestamp = newSplitTimestamp; +#endif } void ProfileSample::TakeSplitSample(PrjFS_PerfCounter splitProbe, PrjFS_PerfCounter newFinalSplitProbe) { +#if PRJFS_PERFORMANCE_TRACING_ENABLE this->TakeSplitSample(splitProbe); this->finalSplitProbe = newFinalSplitProbe; +#endif } void ProfileSample::SetFinalSplitProbe(PrjFS_PerfCounter newFinalSplitProbe) { +#if PRJFS_PERFORMANCE_TRACING_ENABLE this->finalSplitProbe = newFinalSplitProbe; +#endif } -ProfileSample::ProfileSample(PrjFS_PerfCounter defaultProbe) : + +ProfileSample::ProfileSample(PrjFS_PerfCounter defaultProbe) +#if PRJFS_PERFORMANCE_TRACING_ENABLE + : startTimestamp(mach_absolute_time()), wholeSampleProbe(defaultProbe), finalSplitProbe(Probe_None), splitTimestamp(this->startTimestamp) +#endif { } void ProfileSample::SetProbe(PrjFS_PerfCounter probe) { +#if PRJFS_PERFORMANCE_TRACING_ENABLE this->wholeSampleProbe = probe; +#endif } From 905258ae720f8069071b3dfa44b9498dcbb8aa3c Mon Sep 17 00:00:00 2001 From: Phil Dennis-Jordan Date: Sat, 29 Sep 2018 14:24:42 +0200 Subject: [PATCH 250/272] Mac kext: New Profiling(Release) build configuration based on Release --- .../xcshareddata/xcschemes/PrjFS.xcscheme | 2 +- .../PrjFSKext.xcodeproj/project.pbxproj | 92 +++++++++++++++++++ 2 files changed, 93 insertions(+), 1 deletion(-) diff --git a/ProjFS.Mac/PrjFS.xcworkspace/xcshareddata/xcschemes/PrjFS.xcscheme b/ProjFS.Mac/PrjFS.xcworkspace/xcshareddata/xcschemes/PrjFS.xcscheme index 558130ad6..389d98c9a 100644 --- a/ProjFS.Mac/PrjFS.xcworkspace/xcshareddata/xcschemes/PrjFS.xcscheme +++ b/ProjFS.Mac/PrjFS.xcworkspace/xcshareddata/xcschemes/PrjFS.xcscheme @@ -92,7 +92,7 @@ Date: Sat, 6 Oct 2018 10:25:24 +0200 Subject: [PATCH 251/272] Mac kext: Adds check to only handle fileops on allowed filesystems The vnode handler was already performing this check, but the fileop handler previously skipped it. This is part 1 of a fix for KP bug report #340. --- ProjFS.Mac/PrjFSKext/PrjFSKext/KauthHandler.cpp | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/ProjFS.Mac/PrjFSKext/PrjFSKext/KauthHandler.cpp b/ProjFS.Mac/PrjFSKext/PrjFSKext/KauthHandler.cpp index 111c9355d..91e4e0041 100644 --- a/ProjFS.Mac/PrjFSKext/PrjFSKext/KauthHandler.cpp +++ b/ProjFS.Mac/PrjFSKext/PrjFSKext/KauthHandler.cpp @@ -670,6 +670,11 @@ static bool ShouldHandleFileOpEvent( { *root = RootHandle_None; + if (!VirtualizationRoot_VnodeIsOnAllowedFilesystem(vnode)) + { + return false; + } + vtype vnodeType = vnode_vtype(vnode); if (ShouldIgnoreVnodeType(vnodeType, vnode)) { From 5ee74b0b6c401d1eac84c619be043176623dbdb2 Mon Sep 17 00:00:00 2001 From: John Briggs Date: Mon, 8 Oct 2018 09:02:00 -0400 Subject: [PATCH 252/272] Make GVFS.Upgrader rely on GVFS.cs.props to fix incremental build issue --- GVFS/GVFS.Upgrader/GVFS.Upgrader.csproj | 24 +++--------------------- 1 file changed, 3 insertions(+), 21 deletions(-) diff --git a/GVFS/GVFS.Upgrader/GVFS.Upgrader.csproj b/GVFS/GVFS.Upgrader/GVFS.Upgrader.csproj index 8fb2f91ac..008a3eb24 100644 --- a/GVFS/GVFS.Upgrader/GVFS.Upgrader.csproj +++ b/GVFS/GVFS.Upgrader/GVFS.Upgrader.csproj @@ -1,6 +1,7 @@  + Debug AnyCPU @@ -11,45 +12,26 @@ v4.6.1 512 true - publish\ - true - Disk - false - Foreground - 7 - Days - false - false - true - 0 - 1.0.0.%2a - false - false - true true - ..\..\..\BuildOutput\GVFS.Upgrader\bin\x64\Debug\ DEBUG;TRACE true full x64 prompt - MinimumRecommendedRules.ruleset - true + false - ..\..\..\BuildOutput\GVFS.Upgrader\bin\x64\Release\ TRACE true true pdbonly x64 prompt - MinimumRecommendedRules.ruleset - true + false From 95b9bf68e4f867a514746821529aa17426fbb352 Mon Sep 17 00:00:00 2001 From: Derrick Stolee Date: Wed, 3 Oct 2018 07:59:04 -0400 Subject: [PATCH 253/272] GitProcess: allow GIT_TRACE to point to full path --- GVFS/GVFS.Common/Git/GitProcess.cs | 26 ++++++++++++----- .../EnlistmentPerFixture/StatusVerbTests.cs | 28 +++++++++++++++++++ .../Tools/GVFSFunctionalTestEnlistment.cs | 4 +-- .../GVFS.FunctionalTests/Tools/GVFSProcess.cs | 15 ++++++---- 4 files changed, 59 insertions(+), 14 deletions(-) create mode 100644 GVFS/GVFS.FunctionalTests/Tests/EnlistmentPerFixture/StatusVerbTests.cs diff --git a/GVFS/GVFS.Common/Git/GitProcess.cs b/GVFS/GVFS.Common/Git/GitProcess.cs index 8701605ab..89475752e 100644 --- a/GVFS/GVFS.Common/Git/GitProcess.cs +++ b/GVFS/GVFS.Common/Git/GitProcess.cs @@ -1,6 +1,5 @@ using GVFS.Common.FileSystem; using GVFS.Common.Tracing; -using Microsoft.Win32; using System; using System.Collections.Generic; using System.ComponentModel; @@ -175,7 +174,7 @@ public bool IsValidRepo() Result result = this.InvokeGitAgainstDotGitFolder("rev-parse --show-toplevel"); return !result.HasErrors; } - + public Result RevParse(string gitRef) { return this.InvokeGitAgainstDotGitFolder("rev-parse " + gitRef); @@ -441,11 +440,24 @@ public Process GetGitProcess(string command, string workingDirectory, string dot // Removing trace variables that might change git output and break parsing // List of environment variables: https://git-scm.com/book/gr/v2/Git-Internals-Environment-Variables - foreach (string key in processInfo.EnvironmentVariables.Keys.Cast() - .Where(x => x.StartsWith("GIT_TRACE", StringComparison.OrdinalIgnoreCase)) - .ToList()) + foreach (string key in processInfo.EnvironmentVariables.Keys.Cast().ToList()) { - processInfo.EnvironmentVariables.Remove(key); + // If GIT_TRACE is set to a fully-rooted path, then Git sends the trace + // output to that path instead of stdout (GIT_TRACE=1) or stderr (GIT_TRACE=2). + if (key.StartsWith("GIT_TRACE", StringComparison.OrdinalIgnoreCase)) + { + try + { + if (!Path.IsPathRooted(processInfo.EnvironmentVariables[key])) + { + processInfo.EnvironmentVariables.Remove(key); + } + } + catch (ArgumentException) + { + processInfo.EnvironmentVariables.Remove(key); + } + } } processInfo.EnvironmentVariables["GIT_TERMINAL_PROMPT"] = "0"; @@ -637,7 +649,7 @@ private Result InvokeGitAgainstDotGitFolder( parseStdOutLine: parseStdOutLine, timeoutMs: -1); } - + public class Result { public const int SuccessCode = 0; diff --git a/GVFS/GVFS.FunctionalTests/Tests/EnlistmentPerFixture/StatusVerbTests.cs b/GVFS/GVFS.FunctionalTests/Tests/EnlistmentPerFixture/StatusVerbTests.cs new file mode 100644 index 000000000..f7def18b9 --- /dev/null +++ b/GVFS/GVFS.FunctionalTests/Tests/EnlistmentPerFixture/StatusVerbTests.cs @@ -0,0 +1,28 @@ +using GVFS.FunctionalTests.FileSystemRunners; +using GVFS.Tests.Should; +using NUnit.Framework; +using System.Collections.Generic; +using System.IO; + +namespace GVFS.FunctionalTests.Tests.EnlistmentPerFixture +{ + [TestFixture] + public class StatusVerbTests : TestsWithEnlistmentPerFixture + { + [TestCase] + public void GitTrace() + { + Dictionary environmentVariables = new Dictionary(); + + this.Enlistment.Status(trace: "1"); + this.Enlistment.Status(trace: "2"); + + string logPath = Path.Combine(this.Enlistment.RepoRoot, "log-file.txt"); + this.Enlistment.Status(trace: logPath); + + FileSystemRunner fileSystem = new SystemIORunner(); + fileSystem.FileExists(logPath).ShouldBeTrue(); + string.IsNullOrWhiteSpace(fileSystem.ReadAllText(logPath)).ShouldBeFalse(); + } + } +} diff --git a/GVFS/GVFS.FunctionalTests/Tools/GVFSFunctionalTestEnlistment.cs b/GVFS/GVFS.FunctionalTests/Tools/GVFSFunctionalTestEnlistment.cs index 34ce2fd6d..bc28530b8 100644 --- a/GVFS/GVFS.FunctionalTests/Tools/GVFSFunctionalTestEnlistment.cs +++ b/GVFS/GVFS.FunctionalTests/Tools/GVFSFunctionalTestEnlistment.cs @@ -211,9 +211,9 @@ public string Diagnose() return this.gvfsProcess.Diagnose(); } - public string Status() + public string Status(string trace = null) { - return this.gvfsProcess.Status(); + return this.gvfsProcess.Status(trace); } public bool WaitForBackgroundOperations(int maxWaitMilliseconds = DefaultMaxWaitMSForStatusCheck) diff --git a/GVFS/GVFS.FunctionalTests/Tools/GVFSProcess.cs b/GVFS/GVFS.FunctionalTests/Tools/GVFSProcess.cs index d9acce87d..98939a8fb 100644 --- a/GVFS/GVFS.FunctionalTests/Tools/GVFSProcess.cs +++ b/GVFS/GVFS.FunctionalTests/Tools/GVFSProcess.cs @@ -8,14 +8,14 @@ public class GVFSProcess private readonly string pathToGVFS; private readonly string enlistmentRoot; private readonly string localCacheRoot; - + public GVFSProcess(string pathToGVFS, string enlistmentRoot, string localCacheRoot) { this.pathToGVFS = pathToGVFS; this.enlistmentRoot = enlistmentRoot; this.localCacheRoot = localCacheRoot; } - + public void Clone(string repositorySource, string branchToCheckout, bool skipPrefetch) { string args = string.Format( @@ -60,9 +60,9 @@ public string Diagnose() return this.CallGVFS("diagnose \"" + this.enlistmentRoot + "\""); } - public string Status() + public string Status(string trace = null) { - return this.CallGVFS("status " + this.enlistmentRoot); + return this.CallGVFS("status " + this.enlistmentRoot, trace: trace); } public string CacheServer(string args) @@ -90,7 +90,7 @@ public string RunServiceVerb(string argument) return this.CallGVFS("service " + argument, failOnError: true); } - private string CallGVFS(string args, bool failOnError = false) + private string CallGVFS(string args, bool failOnError = false, string trace = null) { ProcessStartInfo processInfo = null; processInfo = new ProcessStartInfo(this.pathToGVFS); @@ -100,6 +100,11 @@ private string CallGVFS(string args, bool failOnError = false) processInfo.UseShellExecute = false; processInfo.RedirectStandardOutput = true; + if (trace != null) + { + processInfo.EnvironmentVariables["GIT_TRACE"] = trace; + } + using (Process process = Process.Start(processInfo)) { string result = process.StandardOutput.ReadToEnd(); From b1a31fc6abfc3032a30c26fbea20976980184d80 Mon Sep 17 00:00:00 2001 From: William Baker Date: Mon, 8 Oct 2018 08:55:23 -0700 Subject: [PATCH 254/272] Switch from FileAttributeRecallOnOpen to FileAttributeRecallOnDataAccess --- .../Tests/EnlistmentPerFixture/BasicFileSystemTests.cs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/GVFS/GVFS.FunctionalTests/Tests/EnlistmentPerFixture/BasicFileSystemTests.cs b/GVFS/GVFS.FunctionalTests/Tests/EnlistmentPerFixture/BasicFileSystemTests.cs index db86eb26e..982e02dec 100644 --- a/GVFS/GVFS.FunctionalTests/Tests/EnlistmentPerFixture/BasicFileSystemTests.cs +++ b/GVFS/GVFS.FunctionalTests/Tests/EnlistmentPerFixture/BasicFileSystemTests.cs @@ -17,7 +17,7 @@ public class BasicFileSystemTests : TestsWithEnlistmentPerFixture { private const int FileAttributeSparseFile = 0x00000200; private const int FileAttributeReparsePoint = 0x00000400; - private const int FileAttributeRecallOnOpen = 0x00040000; + private const int FileAttributeRecallOnDataAccess = 0x00400000; [TestCaseSource(typeof(FileRunnersAndFolders), FileRunnersAndFolders.TestRunners)] public void ShrinkFileContents(FileSystemRunner fileSystem, string parentFolder) @@ -163,7 +163,7 @@ public void ExpandedFileAttributesAreUpdated() while (attributes != FileAttributes.Hidden && retryCount < maxRetries) { // ProjFS attributes are remoted asynchronously when files are converted to full - FileAttributes attributesLessProjFS = attributes & (FileAttributes)~(FileAttributeSparseFile | FileAttributeRecallOnOpen | FileAttributeReparsePoint); + FileAttributes attributesLessProjFS = attributes & (FileAttributes)~(FileAttributeSparseFile | FileAttributeReparsePoint | FileAttributeRecallOnDataAccess); attributesLessProjFS.ShouldEqual( FileAttributes.Hidden, From c3ea175df4d4aac0de2322ef71c02d278b2657b9 Mon Sep 17 00:00:00 2001 From: Al Ameen Shah Date: Mon, 8 Oct 2018 11:19:30 -0400 Subject: [PATCH 255/272] - Added ability to set custom key comparers in FileBasedDictionary. Consumers can set case-insensitive comparers for case-insensitive dictionary lookups. - Added platform specific installer extensions. GVFS.Hooks.OSPlatform would read the extensions from GVFSHooksPlatform.OSPlatform classes. - Updated ProductUpgrader to do case-insensitive HashSet lookups while searching for downloaded installers. --- GVFS/GVFS.Common/FileBasedDictionary.cs | 20 ++++++++++++++++--- GVFS/GVFS.Common/LocalGVFSConfig.cs | 13 +++++------- GVFS/GVFS.Common/ProductUpgrader.Shared.cs | 16 +++++++-------- .../HooksPlatform/GVFSHooksPlatform.Mac.cs | 5 +++++ .../GVFSHooksPlatform.Windows.cs | 5 +++++ GVFS/GVFS.Hooks/Program.cs | 2 +- GVFS/GVFS.Platform.Mac/MacPlatform.Shared.cs | 2 ++ .../WindowsPlatform.Shared.cs | 2 ++ 8 files changed, 44 insertions(+), 21 deletions(-) diff --git a/GVFS/GVFS.Common/FileBasedDictionary.cs b/GVFS/GVFS.Common/FileBasedDictionary.cs index ace9e2804..e02086a37 100644 --- a/GVFS/GVFS.Common/FileBasedDictionary.cs +++ b/GVFS/GVFS.Common/FileBasedDictionary.cs @@ -11,14 +11,28 @@ public class FileBasedDictionary : FileBasedCollection { private ConcurrentDictionary data = new ConcurrentDictionary(); - private FileBasedDictionary(ITracer tracer, PhysicalFileSystem fileSystem, string dataFilePath) + private FileBasedDictionary( + ITracer tracer, + PhysicalFileSystem fileSystem, + string dataFilePath, + IEqualityComparer keyComparer = null) : base(tracer, fileSystem, dataFilePath, collectionAppendsDirectlyToFile: false) { + if (keyComparer != null) + { + this.data = new ConcurrentDictionary(keyComparer); + } } - public static bool TryCreate(ITracer tracer, string dictionaryPath, PhysicalFileSystem fileSystem, out FileBasedDictionary output, out string error) + public static bool TryCreate( + ITracer tracer, + string dictionaryPath, + PhysicalFileSystem fileSystem, + out FileBasedDictionary output, + out string error, + IEqualityComparer keyComparer = null) { - output = new FileBasedDictionary(tracer, fileSystem, dictionaryPath); + output = new FileBasedDictionary(tracer, fileSystem, dictionaryPath, keyComparer); if (!output.TryLoadFromDisk( output.TryParseAddLine, output.TryParseRemoveLine, diff --git a/GVFS/GVFS.Common/LocalGVFSConfig.cs b/GVFS/GVFS.Common/LocalGVFSConfig.cs index ac65b4589..6094122fc 100644 --- a/GVFS/GVFS.Common/LocalGVFSConfig.cs +++ b/GVFS/GVFS.Common/LocalGVFSConfig.cs @@ -1,5 +1,6 @@ using GVFS.Common.FileSystem; using GVFS.Common.Tracing; +using System; using System.IO; namespace GVFS.Common @@ -35,7 +36,7 @@ public bool TryGetConfig( try { - this.allSettings.TryGetValue(this.KeyFromConfigName(name), out value); + this.allSettings.TryGetValue(name, out value); error = null; return true; } @@ -67,7 +68,7 @@ public bool TrySetConfig( try { - this.allSettings.SetValueAndFlush(this.KeyFromConfigName(name), value); + this.allSettings.SetValueAndFlush(name, value); error = null; return true; } @@ -85,11 +86,6 @@ public bool TrySetConfig( } } - private string KeyFromConfigName(string configName) - { - return configName.ToUpperInvariant(); - } - private bool TryLoadSettings(ITracer tracer, out string error) { if (this.allSettings == null) @@ -100,7 +96,8 @@ private bool TryLoadSettings(ITracer tracer, out string error) dictionaryPath: this.configFile, fileSystem: this.fileSystem, output: out config, - error: out error)) + error: out error, + keyComparer: StringComparer.OrdinalIgnoreCase)) { this.allSettings = config; return true; diff --git a/GVFS/GVFS.Common/ProductUpgrader.Shared.cs b/GVFS/GVFS.Common/ProductUpgrader.Shared.cs index 51125717e..d26994bcc 100644 --- a/GVFS/GVFS.Common/ProductUpgrader.Shared.cs +++ b/GVFS/GVFS.Common/ProductUpgrader.Shared.cs @@ -1,3 +1,4 @@ +using System; using System.Collections.Generic; using System.IO; @@ -13,24 +14,21 @@ public partial class ProductUpgrader private const string GVFSInstallerFileNamePrefix = "SetupGVFS"; private const string VFSForGitInstallerFileNamePrefix = "VFSForGit"; - public static bool IsLocalUpgradeAvailable() + public static bool IsLocalUpgradeAvailable(string[] installerExtensions) { string downloadDirectory = GetAssetDownloadsPath(); if (Directory.Exists(downloadDirectory)) { - // This method is used Only by Git hooks. Git hooks does not have access - // to GVFSPlatform to read platform specific file extensions. That is the - // reason possible installer file extensions are defined here. - HashSet extensions = new HashSet() { "EXE", "DMG", "RPM", "DEB" }; - HashSet installerNames = new HashSet() + HashSet extensions = new HashSet(installerExtensions, StringComparer.OrdinalIgnoreCase); + HashSet installerNames = new HashSet(StringComparer.OrdinalIgnoreCase) { - GVFSInstallerFileNamePrefix.ToUpperInvariant(), - VFSForGitInstallerFileNamePrefix.ToUpperInvariant() + GVFSInstallerFileNamePrefix, + VFSForGitInstallerFileNamePrefix }; foreach (string file in Directory.EnumerateFiles(downloadDirectory, "*", SearchOption.TopDirectoryOnly)) { - string[] components = Path.GetFileName(file).ToUpperInvariant().Split('.'); + string[] components = Path.GetFileName(file).Split('.'); int length = components.Length; if (length >= 2 && installerNames.Contains(components[0]) && extensions.Contains(components[length - 1])) { diff --git a/GVFS/GVFS.Hooks/HooksPlatform/GVFSHooksPlatform.Mac.cs b/GVFS/GVFS.Hooks/HooksPlatform/GVFSHooksPlatform.Mac.cs index f6188b573..3d4065878 100644 --- a/GVFS/GVFS.Hooks/HooksPlatform/GVFSHooksPlatform.Mac.cs +++ b/GVFS/GVFS.Hooks/HooksPlatform/GVFSHooksPlatform.Mac.cs @@ -4,6 +4,11 @@ namespace GVFS.Hooks.HooksPlatform { public static class GVFSHooksPlatform { + public static string[] InstallerExtensions() + { + return MacPlatform.InstallerExtensions; + } + public static bool IsElevated() { return MacPlatform.IsElevatedImplementation(); diff --git a/GVFS/GVFS.Hooks/HooksPlatform/GVFSHooksPlatform.Windows.cs b/GVFS/GVFS.Hooks/HooksPlatform/GVFSHooksPlatform.Windows.cs index 56835e112..9e23d1178 100644 --- a/GVFS/GVFS.Hooks/HooksPlatform/GVFSHooksPlatform.Windows.cs +++ b/GVFS/GVFS.Hooks/HooksPlatform/GVFSHooksPlatform.Windows.cs @@ -4,6 +4,11 @@ namespace GVFS.Hooks.HooksPlatform { public static class GVFSHooksPlatform { + public static string[] InstallerExtensions() + { + return WindowsPlatform.InstallerExtensions; + } + public static bool IsElevated() { return WindowsPlatform.IsElevatedImplementation(); diff --git a/GVFS/GVFS.Hooks/Program.cs b/GVFS/GVFS.Hooks/Program.cs index 94df57679..7178a38fd 100644 --- a/GVFS/GVFS.Hooks/Program.cs +++ b/GVFS/GVFS.Hooks/Program.cs @@ -109,7 +109,7 @@ private static void RemindUpgradeAvailable() int reminderFrequency = 10; int randomValue = random.Next(0, 100); - if (randomValue <= reminderFrequency && ProductUpgrader.IsLocalUpgradeAvailable()) + if (randomValue <= reminderFrequency && ProductUpgrader.IsLocalUpgradeAvailable(GVFSHooksPlatform.InstallerExtensions())) { Console.WriteLine(Environment.NewLine + GVFSConstants.UpgradeVerbMessages.ReminderNotification); } diff --git a/GVFS/GVFS.Platform.Mac/MacPlatform.Shared.cs b/GVFS/GVFS.Platform.Mac/MacPlatform.Shared.cs index 03568e5dc..0434aae5e 100644 --- a/GVFS/GVFS.Platform.Mac/MacPlatform.Shared.cs +++ b/GVFS/GVFS.Platform.Mac/MacPlatform.Shared.cs @@ -5,6 +5,8 @@ namespace GVFS.Platform.Mac { public partial class MacPlatform { + public static readonly string[] InstallerExtensions = { "dmg" }; + public static bool IsElevatedImplementation() { // TODO(Mac): Implement proper check diff --git a/GVFS/GVFS.Platform.Windows/WindowsPlatform.Shared.cs b/GVFS/GVFS.Platform.Windows/WindowsPlatform.Shared.cs index ca9aac5cc..1010b21c9 100644 --- a/GVFS/GVFS.Platform.Windows/WindowsPlatform.Shared.cs +++ b/GVFS/GVFS.Platform.Windows/WindowsPlatform.Shared.cs @@ -8,6 +8,8 @@ namespace GVFS.Platform.Windows { public partial class WindowsPlatform { + public static readonly string[] InstallerExtensions = { "exe" }; + private const int StillActive = 259; /* from Win32 STILL_ACTIVE */ private enum StdHandle From 5a22202b97e10d39643f5ee47287ab27efa382de Mon Sep 17 00:00:00 2001 From: Al Ameen Shah Date: Mon, 8 Oct 2018 14:33:03 -0400 Subject: [PATCH 256/272] Avoiding possible redundant initialisation of ConcurrentDictionary in FileBasedDictionary class. --- GVFS/GVFS.Common/FileBasedDictionary.cs | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/GVFS/GVFS.Common/FileBasedDictionary.cs b/GVFS/GVFS.Common/FileBasedDictionary.cs index e02086a37..cccc72ceb 100644 --- a/GVFS/GVFS.Common/FileBasedDictionary.cs +++ b/GVFS/GVFS.Common/FileBasedDictionary.cs @@ -9,7 +9,7 @@ namespace GVFS.Common { public class FileBasedDictionary : FileBasedCollection { - private ConcurrentDictionary data = new ConcurrentDictionary(); + private ConcurrentDictionary data; private FileBasedDictionary( ITracer tracer, @@ -18,10 +18,7 @@ private FileBasedDictionary( IEqualityComparer keyComparer = null) : base(tracer, fileSystem, dataFilePath, collectionAppendsDirectlyToFile: false) { - if (keyComparer != null) - { - this.data = new ConcurrentDictionary(keyComparer); - } + this.data = new ConcurrentDictionary(keyComparer ?? EqualityComparer.Default); } public static bool TryCreate( From 6fe4f037791bb645199caee840ec7aa8da4b49af Mon Sep 17 00:00:00 2001 From: Al Ameen Shah Date: Tue, 9 Oct 2018 11:56:11 -0400 Subject: [PATCH 257/272] Cleanup - Use Enlistment.GitBinPath when available, rather than invoking Platform. Updated GVFSHooksPlatforms to return string for InstallerExtensions, bcoz neither Mac nor Windows release ship with multiple installer extensions. --- GVFS/GVFS.Common/FileBasedDictionary.cs | 11 ++++++++--- GVFS/GVFS.Common/ProductUpgrader.Shared.cs | 7 ++++--- .../GVFS.Hooks/HooksPlatform/GVFSHooksPlatform.Mac.cs | 4 ++-- .../HooksPlatform/GVFSHooksPlatform.Windows.cs | 4 ++-- GVFS/GVFS.Hooks/Program.cs | 2 +- GVFS/GVFS.Platform.Mac/MacPlatform.Shared.cs | 2 +- GVFS/GVFS.Platform.Windows/WindowsPlatform.Shared.cs | 2 +- GVFS/GVFS/CommandLine/DiagnoseVerb.cs | 5 ++--- GVFS/GVFS/CommandLine/GVFSVerb.cs | 3 +-- 9 files changed, 22 insertions(+), 18 deletions(-) diff --git a/GVFS/GVFS.Common/FileBasedDictionary.cs b/GVFS/GVFS.Common/FileBasedDictionary.cs index cccc72ceb..393de7cb0 100644 --- a/GVFS/GVFS.Common/FileBasedDictionary.cs +++ b/GVFS/GVFS.Common/FileBasedDictionary.cs @@ -15,10 +15,10 @@ private FileBasedDictionary( ITracer tracer, PhysicalFileSystem fileSystem, string dataFilePath, - IEqualityComparer keyComparer = null) + IEqualityComparer keyComparer) : base(tracer, fileSystem, dataFilePath, collectionAppendsDirectlyToFile: false) { - this.data = new ConcurrentDictionary(keyComparer ?? EqualityComparer.Default); + this.data = new ConcurrentDictionary(keyComparer); } public static bool TryCreate( @@ -29,7 +29,12 @@ public static bool TryCreate( out string error, IEqualityComparer keyComparer = null) { - output = new FileBasedDictionary(tracer, fileSystem, dictionaryPath, keyComparer); + output = new FileBasedDictionary( + tracer, + fileSystem, + dictionaryPath, + keyComparer ?? EqualityComparer.Default); + if (!output.TryLoadFromDisk( output.TryParseAddLine, output.TryParseRemoveLine, diff --git a/GVFS/GVFS.Common/ProductUpgrader.Shared.cs b/GVFS/GVFS.Common/ProductUpgrader.Shared.cs index d26994bcc..7e621a249 100644 --- a/GVFS/GVFS.Common/ProductUpgrader.Shared.cs +++ b/GVFS/GVFS.Common/ProductUpgrader.Shared.cs @@ -14,12 +14,11 @@ public partial class ProductUpgrader private const string GVFSInstallerFileNamePrefix = "SetupGVFS"; private const string VFSForGitInstallerFileNamePrefix = "VFSForGit"; - public static bool IsLocalUpgradeAvailable(string[] installerExtensions) + public static bool IsLocalUpgradeAvailable(string installerExtension) { string downloadDirectory = GetAssetDownloadsPath(); if (Directory.Exists(downloadDirectory)) { - HashSet extensions = new HashSet(installerExtensions, StringComparer.OrdinalIgnoreCase); HashSet installerNames = new HashSet(StringComparer.OrdinalIgnoreCase) { GVFSInstallerFileNamePrefix, @@ -30,7 +29,9 @@ public static bool IsLocalUpgradeAvailable(string[] installerExtensions) { string[] components = Path.GetFileName(file).Split('.'); int length = components.Length; - if (length >= 2 && installerNames.Contains(components[0]) && extensions.Contains(components[length - 1])) + if (length >= 2 && + installerNames.Contains(components[0]) && + installerExtension.Equals(components[length - 1], StringComparison.OrdinalIgnoreCase)) { return true; } diff --git a/GVFS/GVFS.Hooks/HooksPlatform/GVFSHooksPlatform.Mac.cs b/GVFS/GVFS.Hooks/HooksPlatform/GVFSHooksPlatform.Mac.cs index 3d4065878..e9e8b8873 100644 --- a/GVFS/GVFS.Hooks/HooksPlatform/GVFSHooksPlatform.Mac.cs +++ b/GVFS/GVFS.Hooks/HooksPlatform/GVFSHooksPlatform.Mac.cs @@ -4,9 +4,9 @@ namespace GVFS.Hooks.HooksPlatform { public static class GVFSHooksPlatform { - public static string[] InstallerExtensions() + public static string InstallerExtension() { - return MacPlatform.InstallerExtensions; + return MacPlatform.InstallerExtension; } public static bool IsElevated() diff --git a/GVFS/GVFS.Hooks/HooksPlatform/GVFSHooksPlatform.Windows.cs b/GVFS/GVFS.Hooks/HooksPlatform/GVFSHooksPlatform.Windows.cs index 9e23d1178..3df93424c 100644 --- a/GVFS/GVFS.Hooks/HooksPlatform/GVFSHooksPlatform.Windows.cs +++ b/GVFS/GVFS.Hooks/HooksPlatform/GVFSHooksPlatform.Windows.cs @@ -4,9 +4,9 @@ namespace GVFS.Hooks.HooksPlatform { public static class GVFSHooksPlatform { - public static string[] InstallerExtensions() + public static string InstallerExtension() { - return WindowsPlatform.InstallerExtensions; + return WindowsPlatform.InstallerExtension; } public static bool IsElevated() diff --git a/GVFS/GVFS.Hooks/Program.cs b/GVFS/GVFS.Hooks/Program.cs index 7178a38fd..c3646dcf5 100644 --- a/GVFS/GVFS.Hooks/Program.cs +++ b/GVFS/GVFS.Hooks/Program.cs @@ -109,7 +109,7 @@ private static void RemindUpgradeAvailable() int reminderFrequency = 10; int randomValue = random.Next(0, 100); - if (randomValue <= reminderFrequency && ProductUpgrader.IsLocalUpgradeAvailable(GVFSHooksPlatform.InstallerExtensions())) + if (randomValue <= reminderFrequency && ProductUpgrader.IsLocalUpgradeAvailable(GVFSHooksPlatform.InstallerExtension())) { Console.WriteLine(Environment.NewLine + GVFSConstants.UpgradeVerbMessages.ReminderNotification); } diff --git a/GVFS/GVFS.Platform.Mac/MacPlatform.Shared.cs b/GVFS/GVFS.Platform.Mac/MacPlatform.Shared.cs index 0434aae5e..dd123b464 100644 --- a/GVFS/GVFS.Platform.Mac/MacPlatform.Shared.cs +++ b/GVFS/GVFS.Platform.Mac/MacPlatform.Shared.cs @@ -5,7 +5,7 @@ namespace GVFS.Platform.Mac { public partial class MacPlatform { - public static readonly string[] InstallerExtensions = { "dmg" }; + public static readonly string InstallerExtension = "dmg"; public static bool IsElevatedImplementation() { diff --git a/GVFS/GVFS.Platform.Windows/WindowsPlatform.Shared.cs b/GVFS/GVFS.Platform.Windows/WindowsPlatform.Shared.cs index 1010b21c9..b0e6487ae 100644 --- a/GVFS/GVFS.Platform.Windows/WindowsPlatform.Shared.cs +++ b/GVFS/GVFS.Platform.Windows/WindowsPlatform.Shared.cs @@ -8,7 +8,7 @@ namespace GVFS.Platform.Windows { public partial class WindowsPlatform { - public static readonly string[] InstallerExtensions = { "exe" }; + public static readonly string InstallerExtension = "exe"; private const int StillActive = 259; /* from Win32 STILL_ACTIVE */ diff --git a/GVFS/GVFS/CommandLine/DiagnoseVerb.cs b/GVFS/GVFS/CommandLine/DiagnoseVerb.cs index c8a0c0a52..e752bc625 100644 --- a/GVFS/GVFS/CommandLine/DiagnoseVerb.cs +++ b/GVFS/GVFS/CommandLine/DiagnoseVerb.cs @@ -52,8 +52,7 @@ protected override void Execute(GVFSEnlistment enlistment) GitVersion gitVersion = null; string error = null; - string gitPath = GVFSPlatform.Instance.GitInstallation.GetInstalledGitBinPath(); - if (!string.IsNullOrEmpty(gitPath) && GitProcess.TryGetVersion(gitPath, out gitVersion, out error)) + if (!string.IsNullOrEmpty(enlistment.GitBinPath) && GitProcess.TryGetVersion(enlistment.GitBinPath, out gitVersion, out error)) { this.WriteMessage("git version " + gitVersion.ToString()); } @@ -62,7 +61,7 @@ protected override void Execute(GVFSEnlistment enlistment) this.WriteMessage("Could not determine git version. " + error); } - this.WriteMessage(GVFSPlatform.Instance.GitInstallation.GetInstalledGitBinPath()); + this.WriteMessage(enlistment.GitBinPath); this.WriteMessage(string.Empty); this.WriteMessage("Enlistment root: " + enlistment.EnlistmentRoot); this.WriteMessage("Cache Server: " + CacheServerResolver.GetCacheServerFromConfig(enlistment)); diff --git a/GVFS/GVFS/CommandLine/GVFSVerb.cs b/GVFS/GVFS/CommandLine/GVFSVerb.cs index 94bd33752..4f630ab31 100644 --- a/GVFS/GVFS/CommandLine/GVFSVerb.cs +++ b/GVFS/GVFS/CommandLine/GVFSVerb.cs @@ -619,9 +619,8 @@ private void CheckFileSystemSupportsRequiredFeatures(ITracer tracer, Enlistment private void CheckGitVersion(ITracer tracer, GVFSEnlistment enlistment, out string version) { - string gitPath = GVFSPlatform.Instance.GitInstallation.GetInstalledGitBinPath(); GitVersion gitVersion = null; - if (string.IsNullOrEmpty(gitPath) || !GitProcess.TryGetVersion(gitPath, out gitVersion, out string _)) + if (string.IsNullOrEmpty(enlistment.GitBinPath) || !GitProcess.TryGetVersion(enlistment.GitBinPath, out gitVersion, out string _)) { this.ReportErrorAndExit(tracer, "Error: Unable to retrieve the git version"); } From 5b8f17a9e18cd39fb697ce9887de6446dbfe8b8f Mon Sep 17 00:00:00 2001 From: Al Ameen Shah Date: Tue, 9 Oct 2018 12:59:01 -0400 Subject: [PATCH 258/272] Cleanups - made InstallerExtension strings const. Renamed InstallerExtension accessor to GetInstallerExtension. --- GVFS/GVFS.Hooks/HooksPlatform/GVFSHooksPlatform.Mac.cs | 2 +- GVFS/GVFS.Hooks/HooksPlatform/GVFSHooksPlatform.Windows.cs | 2 +- GVFS/GVFS.Hooks/Program.cs | 2 +- GVFS/GVFS.Platform.Mac/MacPlatform.Shared.cs | 2 +- GVFS/GVFS.Platform.Windows/WindowsPlatform.Shared.cs | 2 +- 5 files changed, 5 insertions(+), 5 deletions(-) diff --git a/GVFS/GVFS.Hooks/HooksPlatform/GVFSHooksPlatform.Mac.cs b/GVFS/GVFS.Hooks/HooksPlatform/GVFSHooksPlatform.Mac.cs index e9e8b8873..2a459d964 100644 --- a/GVFS/GVFS.Hooks/HooksPlatform/GVFSHooksPlatform.Mac.cs +++ b/GVFS/GVFS.Hooks/HooksPlatform/GVFSHooksPlatform.Mac.cs @@ -4,7 +4,7 @@ namespace GVFS.Hooks.HooksPlatform { public static class GVFSHooksPlatform { - public static string InstallerExtension() + public static string GetInstallerExtension() { return MacPlatform.InstallerExtension; } diff --git a/GVFS/GVFS.Hooks/HooksPlatform/GVFSHooksPlatform.Windows.cs b/GVFS/GVFS.Hooks/HooksPlatform/GVFSHooksPlatform.Windows.cs index 3df93424c..ba5ce9b71 100644 --- a/GVFS/GVFS.Hooks/HooksPlatform/GVFSHooksPlatform.Windows.cs +++ b/GVFS/GVFS.Hooks/HooksPlatform/GVFSHooksPlatform.Windows.cs @@ -4,7 +4,7 @@ namespace GVFS.Hooks.HooksPlatform { public static class GVFSHooksPlatform { - public static string InstallerExtension() + public static string GetInstallerExtension() { return WindowsPlatform.InstallerExtension; } diff --git a/GVFS/GVFS.Hooks/Program.cs b/GVFS/GVFS.Hooks/Program.cs index c3646dcf5..eab2ec0ae 100644 --- a/GVFS/GVFS.Hooks/Program.cs +++ b/GVFS/GVFS.Hooks/Program.cs @@ -109,7 +109,7 @@ private static void RemindUpgradeAvailable() int reminderFrequency = 10; int randomValue = random.Next(0, 100); - if (randomValue <= reminderFrequency && ProductUpgrader.IsLocalUpgradeAvailable(GVFSHooksPlatform.InstallerExtension())) + if (randomValue <= reminderFrequency && ProductUpgrader.IsLocalUpgradeAvailable(GVFSHooksPlatform.GetInstallerExtension())) { Console.WriteLine(Environment.NewLine + GVFSConstants.UpgradeVerbMessages.ReminderNotification); } diff --git a/GVFS/GVFS.Platform.Mac/MacPlatform.Shared.cs b/GVFS/GVFS.Platform.Mac/MacPlatform.Shared.cs index dd123b464..2cd456ee4 100644 --- a/GVFS/GVFS.Platform.Mac/MacPlatform.Shared.cs +++ b/GVFS/GVFS.Platform.Mac/MacPlatform.Shared.cs @@ -5,7 +5,7 @@ namespace GVFS.Platform.Mac { public partial class MacPlatform { - public static readonly string InstallerExtension = "dmg"; + public const string InstallerExtension = "dmg"; public static bool IsElevatedImplementation() { diff --git a/GVFS/GVFS.Platform.Windows/WindowsPlatform.Shared.cs b/GVFS/GVFS.Platform.Windows/WindowsPlatform.Shared.cs index b0e6487ae..6a87b81be 100644 --- a/GVFS/GVFS.Platform.Windows/WindowsPlatform.Shared.cs +++ b/GVFS/GVFS.Platform.Windows/WindowsPlatform.Shared.cs @@ -8,7 +8,7 @@ namespace GVFS.Platform.Windows { public partial class WindowsPlatform { - public static readonly string InstallerExtension = "exe"; + public const string InstallerExtension = "exe"; private const int StillActive = 259; /* from Win32 STILL_ACTIVE */ From 740bd76476368f9f11a97a053cb078d7d2994659 Mon Sep 17 00:00:00 2001 From: Nick Graczyk Date: Tue, 9 Oct 2018 18:00:52 -0700 Subject: [PATCH 259/272] Update build scripts to handle new Profiling configuration --- ProjFS.Mac/Scripts/Build.sh | 5 +++++ Scripts/Mac/BuildGVFSForMac.sh | 5 +++++ 2 files changed, 10 insertions(+) diff --git a/ProjFS.Mac/Scripts/Build.sh b/ProjFS.Mac/Scripts/Build.sh index 333883c37..9a1060ab9 100755 --- a/ProjFS.Mac/Scripts/Build.sh +++ b/ProjFS.Mac/Scripts/Build.sh @@ -14,5 +14,10 @@ PROJFS=$SRCDIR/ProjFS.Mac xcodebuild -configuration $CONFIGURATION -workspace $PROJFS/PrjFS.xcworkspace build -scheme PrjFS -derivedDataPath $ROOTDIR/BuildOutput/ProjFS.Mac/Native || exit 1 +# If we're building the Profiling(Release) configuration, remove Profiling() for building .NET code +if [ "$CONFIGURATION" == "Profiling(Release)" ]; then + CONFIGURATION=Release +fi + dotnet restore $PROJFS/PrjFSLib.Mac.Managed/PrjFSLib.Mac.Managed.csproj /p:Configuration=$CONFIGURATION /p:Platform=x64 --packages $PACKAGES || exit 1 dotnet build $PROJFS/PrjFSLib.Mac.Managed/PrjFSLib.Mac.Managed.csproj /p:Configuration=$CONFIGURATION /p:Platform=x64 || exit 1 diff --git a/Scripts/Mac/BuildGVFSForMac.sh b/Scripts/Mac/BuildGVFSForMac.sh index 678bb89aa..d6d07e4ad 100755 --- a/Scripts/Mac/BuildGVFSForMac.sh +++ b/Scripts/Mac/BuildGVFSForMac.sh @@ -38,6 +38,11 @@ GITPATH="$(find $PACKAGES/gitformac.gvfs.installer/$GITVERSION -type f -name *.d # Now that we have a path containing the version number, generate GVFSConstants.GitVersion.cs $SCRIPTDIR/GenerateGitVersionConstants.sh "$GITPATH" $BUILDDIR || exit 1 +# If we're building the Profiling(Release) configuration, remove Profiling() for building .NET code +if [ "$CONFIGURATION" == "Profiling(Release)" ]; then + CONFIGURATION=Release +fi + DOTNETCONFIGURATION=$CONFIGURATION.Mac dotnet restore $SRCDIR/GVFS.sln /p:Configuration=$DOTNETCONFIGURATION --packages $PACKAGES || exit 1 dotnet build $SRCDIR/GVFS.sln --runtime osx-x64 --framework netcoreapp2.1 --configuration $DOTNETCONFIGURATION /maxcpucount:1 || exit 1 From 1af261720a9440f5a5ee0e100e9b7072a8a35e17 Mon Sep 17 00:00:00 2001 From: William Baker Date: Tue, 9 Oct 2018 15:18:07 -0700 Subject: [PATCH 260/272] Mac: Handle projection changes where git deletes folders that are still in the projection --- .../Tests/GitCommands/CheckoutTests.cs | 1 - .../Projection/GitIndexProjection.cs | 57 +++++++++++++++---- ProjFS.Mac/PrjFSLib/PrjFSLib.cpp | 25 ++++++-- 3 files changed, 66 insertions(+), 17 deletions(-) diff --git a/GVFS/GVFS.FunctionalTests/Tests/GitCommands/CheckoutTests.cs b/GVFS/GVFS.FunctionalTests/Tests/GitCommands/CheckoutTests.cs index 8457720bb..351e6a7ef 100644 --- a/GVFS/GVFS.FunctionalTests/Tests/GitCommands/CheckoutTests.cs +++ b/GVFS/GVFS.FunctionalTests/Tests/GitCommands/CheckoutTests.cs @@ -892,7 +892,6 @@ public void CheckoutBranchDirectoryWithOneFileRead() } [TestCase] - [Category(Categories.MacTODO.M3)] public void CheckoutBranchDirectoryWithOneFileWrite() { this.RunFileDirectoryWriteTest("checkout", commandBranch: GitRepoTests.DirectoryWithDifferentFileAfterBranch); diff --git a/GVFS/GVFS.Virtualization/Projection/GitIndexProjection.cs b/GVFS/GVFS.Virtualization/Projection/GitIndexProjection.cs index c99808b74..49384e3bc 100644 --- a/GVFS/GVFS.Virtualization/Projection/GitIndexProjection.cs +++ b/GVFS/GVFS.Virtualization/Projection/GitIndexProjection.cs @@ -1158,8 +1158,11 @@ private void UpdatePlaceholders() new HashSet(placeholderFoldersListCopy.Select(x => x.Path), StringComparer.OrdinalIgnoreCase) : null; - // Order the folders in decscending order so that we walk the tree from bottom up (ensuring child folders are deleted before - // their parents) + // Order the folders in decscending order so that we walk the tree from bottom up. + // Traversing the folders in this order: + // 1. Ensures child folders are deleted before their parents + // 2. Ensures that folders that have been deleted by git (but are still in the projection) are found before their + // parent folder is re-expanded (only applies on platforms where EnumerationExpandsDirectories is true) foreach (PlaceholderListDatabase.PlaceholderData folderPlaceholder in placeholderFoldersListCopy.OrderByDescending(x => x.Path)) { // Remove folder placeholders before re-expansion to ensure that projection changes that convert a folder to a file work @@ -1388,15 +1391,31 @@ private void ReExpandFolder( childRelativePath = relativeFolderPath + Path.DirectorySeparatorChar + childEntry.Name.GetString(); } - // TODO(Mac): Issue #245, handle failures of WritePlaceholderDirectory and WritePlaceholderFile if (childEntry.IsFolder) { if (!existingFolderPlaceholders.Contains(childRelativePath)) { - this.fileSystemVirtualizer.WritePlaceholderDirectory(childRelativePath); - updatedPlaceholderList.TryAdd( - childRelativePath, - new PlaceholderListDatabase.PlaceholderData(childRelativePath, PlaceholderListDatabase.PartialFolderValue)); + FileSystemResult result = this.fileSystemVirtualizer.WritePlaceholderDirectory(childRelativePath); + switch (result.Result) + { + case FSResult.Ok: + updatedPlaceholderList.TryAdd( + childRelativePath, + new PlaceholderListDatabase.PlaceholderData(childRelativePath, PlaceholderListDatabase.PartialFolderValue)); + break; + + case FSResult.FileOrPathNotFound: + // Git command must have removed the folder being re-expanded (relativeFolderPath) + // Remove the folder from existingFolderPlaceholders so that its parent will create + // it again (when it's re-expanded) + existingFolderPlaceholders.Remove(relativeFolderPath); + return; + + default: + // TODO(Mac): Issue #245, handle failures of WritePlaceholderDirectory and WritePlaceholderFile + break; + } + } } else @@ -1406,10 +1425,26 @@ private void ReExpandFolder( FileData childFileData = childEntry as FileData; string sha = childFileData.Sha.ToString(); - this.fileSystemVirtualizer.WritePlaceholderFile(childRelativePath, childFileData.Size, sha); - updatedPlaceholderList.TryAdd( - childRelativePath, - new PlaceholderListDatabase.PlaceholderData(childRelativePath, sha)); + FileSystemResult result = this.fileSystemVirtualizer.WritePlaceholderFile(childRelativePath, childFileData.Size, sha); + switch (result.Result) + { + case FSResult.Ok: + updatedPlaceholderList.TryAdd( + childRelativePath, + new PlaceholderListDatabase.PlaceholderData(childRelativePath, sha)); + break; + + case FSResult.FileOrPathNotFound: + // Git command must have removed the folder being re-expanded (relativeFolderPath) + // Remove the folder from existingFolderPlaceholders so that its parent will create + // it again (when it's re-expanded) + existingFolderPlaceholders.Remove(relativeFolderPath); + return; + + default: + // TODO(Mac): Issue #245, handle failures of WritePlaceholderDirectory and WritePlaceholderFile + break; + } } } } diff --git a/ProjFS.Mac/PrjFSLib/PrjFSLib.cpp b/ProjFS.Mac/PrjFSLib/PrjFSLib.cpp index 132ccd383..2bd6636d0 100644 --- a/ProjFS.Mac/PrjFSLib/PrjFSLib.cpp +++ b/ProjFS.Mac/PrjFSLib/PrjFSLib.cpp @@ -323,23 +323,36 @@ PrjFS_Result PrjFS_WritePlaceholderFile( return PrjFS_Result_EInvalidArgs; } + PrjFS_Result result = PrjFS_Result_Invalid; PrjFSFileXAttrData fileXattrData = {}; char fullPath[PrjFSMaxPath]; CombinePaths(s_virtualizationRootFullPath.c_str(), relativePath, fullPath); - // Mode "wbx" means - // - Create an empty file if none exists - // - Fail if a file already exists at this path - FILE* file = fopen(fullPath, "wbx"); + // Mode "wx" means: + // - "w": Open for writing. The stream is positioned at the beginning of the file. Create the file if it does not exist. + // - "x": If the file already exists, fopen() fails, and sets errno to EEXIST. + FILE* file = fopen(fullPath, "wx"); if (nullptr == file) { + switch(errno) + { + case ENOENT: // A directory component in fullPath does not exist or is a dangling symbolic link. + result = PrjFS_Result_EPathNotFound; + break; + case EEXIST: // The file already exists + default: + result = PrjFS_Result_EIOError; + break; + } + goto CleanupAndFail; } // Expand the file to the desired size if (ftruncate(fileno(file), fileSize)) { + result = PrjFS_Result_EIOError; goto CleanupAndFail; } @@ -354,12 +367,14 @@ PrjFS_Result PrjFS_WritePlaceholderFile( &fileXattrData, PrjFSFileXAttrName)) { + result = PrjFS_Result_EIOError; goto CleanupAndFail; } // TODO(Mac): Only call chmod if fileMode is different than the default file mode if (chmod(fullPath, fileMode)) { + result = PrjFS_Result_EIOError; goto CleanupAndFail; } @@ -375,7 +390,7 @@ PrjFS_Result PrjFS_WritePlaceholderFile( file = nullptr; } - return PrjFS_Result_EIOError; + return result; } PrjFS_Result PrjFS_WriteSymLink( From 8572e725024e0cc0ff95814b6793167093d0574b Mon Sep 17 00:00:00 2001 From: Kevin Willford Date: Wed, 10 Oct 2018 11:07:33 -0600 Subject: [PATCH 261/272] Add check-ignore and check-mailmap to commands that do not lock --- GVFS/GVFS.Hooks/Program.cs | 2 ++ 1 file changed, 2 insertions(+) diff --git a/GVFS/GVFS.Hooks/Program.cs b/GVFS/GVFS.Hooks/Program.cs index eab2ec0ae..64d40eceb 100644 --- a/GVFS/GVFS.Hooks/Program.cs +++ b/GVFS/GVFS.Hooks/Program.cs @@ -411,6 +411,8 @@ private static bool ShouldLock(string[] args) case "branch": case "cat-file": case "check-attr": + case "check-ignore": + case "check-mailmap": case "commit-graph": case "config": case "credential": From 5d62ef31e56f985029515b48de9676337f136eab Mon Sep 17 00:00:00 2001 From: William Baker Date: Wed, 10 Oct 2018 10:16:59 -0700 Subject: [PATCH 262/272] Fix StyleCop issue --- GVFS/GVFS.Virtualization/Projection/GitIndexProjection.cs | 1 - 1 file changed, 1 deletion(-) diff --git a/GVFS/GVFS.Virtualization/Projection/GitIndexProjection.cs b/GVFS/GVFS.Virtualization/Projection/GitIndexProjection.cs index 49384e3bc..a1a690986 100644 --- a/GVFS/GVFS.Virtualization/Projection/GitIndexProjection.cs +++ b/GVFS/GVFS.Virtualization/Projection/GitIndexProjection.cs @@ -1415,7 +1415,6 @@ private void ReExpandFolder( // TODO(Mac): Issue #245, handle failures of WritePlaceholderDirectory and WritePlaceholderFile break; } - } } else From d57e79157b62025d0315f95848f3c65ca00c3b4e Mon Sep 17 00:00:00 2001 From: William Baker Date: Wed, 10 Oct 2018 15:07:58 -0700 Subject: [PATCH 263/272] Add additional functional test and update PrjFS_WritePlaceholderDirectory to return a more specific error message --- .../Tests/GitCommands/CheckoutTests.cs | 47 +++++++++++++++++++ .../Tests/GitCommands/GitRepoTests.cs | 2 + ProjFS.Mac/PrjFSLib/PrjFSLib.cpp | 16 ++++++- 3 files changed, 64 insertions(+), 1 deletion(-) diff --git a/GVFS/GVFS.FunctionalTests/Tests/GitCommands/CheckoutTests.cs b/GVFS/GVFS.FunctionalTests/Tests/GitCommands/CheckoutTests.cs index 351e6a7ef..ac861a982 100644 --- a/GVFS/GVFS.FunctionalTests/Tests/GitCommands/CheckoutTests.cs +++ b/GVFS/GVFS.FunctionalTests/Tests/GitCommands/CheckoutTests.cs @@ -897,6 +897,53 @@ public void CheckoutBranchDirectoryWithOneFileWrite() this.RunFileDirectoryWriteTest("checkout", commandBranch: GitRepoTests.DirectoryWithDifferentFileAfterBranch); } + [TestCase] + public void CheckoutBranchDirectoryWithOneDeepFileWrite() + { + this.ControlGitRepo.Fetch(DeepDirectoryWithOneFile); + this.ControlGitRepo.Fetch(DeepDirectoryWithOneDifferentFile); + this.ValidateGitCommand($"checkout {DeepDirectoryWithOneFile}"); + this.FileShouldHaveContents( + "TestFile1\n", + "GitCommandsTests", + "CheckoutBranchDirectoryWithOneDeepFile", + "FolderDepth1", + "FolderDepth2", + "FolderDepth3", + "File1.txt"); + + // Edit the file and commit the change so that git will + // delete the file (and its parent directories) when + // changing branches + this.EditFile( + "Change file", + "GitCommandsTests", + "CheckoutBranchDirectoryWithOneDeepFile", + "FolderDepth1", + "FolderDepth2", + "FolderDepth3", + "File1.txt"); + this.ValidateGitCommand("add --all"); + this.RunGitCommand("commit -m \"Some change\""); + + this.ValidateGitCommand($"checkout {DeepDirectoryWithOneDifferentFile}"); + this.FileShouldHaveContents( + "TestFile2\n", + "GitCommandsTests", + "CheckoutBranchDirectoryWithOneDeepFile", + "FolderDepth1", + "FolderDepth2", + "FolderDepth3", + "File2.txt"); + this.ShouldNotExistOnDisk( + "GitCommandsTests", + "CheckoutBranchDirectoryWithOneDeepFile", + "FolderDepth1", + "FolderDepth2", + "FolderDepth3", + "File1.txt"); + } + private static void CopyIndexAndRename(string indexPath) { string tempIndexPath = indexPath + ".lock"; diff --git a/GVFS/GVFS.FunctionalTests/Tests/GitCommands/GitRepoTests.cs b/GVFS/GVFS.FunctionalTests/Tests/GitCommands/GitRepoTests.cs index 9200be7c5..d68e38095 100644 --- a/GVFS/GVFS.FunctionalTests/Tests/GitCommands/GitRepoTests.cs +++ b/GVFS/GVFS.FunctionalTests/Tests/GitCommands/GitRepoTests.cs @@ -19,6 +19,8 @@ public abstract class GitRepoTests protected const string DirectoryWithFileBeforeBranch = "FunctionalTests/20171025_DirectoryWithFileBefore"; protected const string DirectoryWithFileAfterBranch = "FunctionalTests/20171025_DirectoryWithFileAfter"; protected const string DirectoryWithDifferentFileAfterBranch = "FunctionalTests/20171025_DirectoryWithDifferentFile"; + protected const string DeepDirectoryWithOneFile = "FunctionalTests/20181010_DeepFolderOneFile"; + protected const string DeepDirectoryWithOneDifferentFile = "FunctionalTests/20181010_DeepFolderOneDifferentFile"; private bool enlistmentPerTest; diff --git a/ProjFS.Mac/PrjFSLib/PrjFSLib.cpp b/ProjFS.Mac/PrjFSLib/PrjFSLib.cpp index 2bd6636d0..cddef78bb 100644 --- a/ProjFS.Mac/PrjFSLib/PrjFSLib.cpp +++ b/ProjFS.Mac/PrjFSLib/PrjFSLib.cpp @@ -281,16 +281,29 @@ PrjFS_Result PrjFS_WritePlaceholderDirectory( return PrjFS_Result_EInvalidArgs; } + PrjFS_Result result = PrjFS_Result_Invalid; char fullPath[PrjFSMaxPath]; CombinePaths(s_virtualizationRootFullPath.c_str(), relativePath, fullPath); if (mkdir(fullPath, 0777)) { + switch(errno) + { + // TODO(Mac): Return more specific error codes for other failure scenarios + case ENOENT: // A component of the path prefix does not exist or path is an empty string + result = PrjFS_Result_EPathNotFound; + break; + default: + result = PrjFS_Result_EIOError; + break; + } + goto CleanupAndFail; } if (!InitializeEmptyPlaceholder(fullPath)) { + result = PrjFS_Result_EIOError; goto CleanupAndFail; } @@ -298,7 +311,7 @@ PrjFS_Result PrjFS_WritePlaceholderDirectory( CleanupAndFail: // TODO: cleanup the directory on disk if needed - return PrjFS_Result_EIOError; + return result; } PrjFS_Result PrjFS_WritePlaceholderFile( @@ -337,6 +350,7 @@ PrjFS_Result PrjFS_WritePlaceholderFile( { switch(errno) { + // TODO(Mac): Return more specific error codes for other failure scenarios case ENOENT: // A directory component in fullPath does not exist or is a dangling symbolic link. result = PrjFS_Result_EPathNotFound; break; From 74743c0198d6bee78df8a1a2f8467072449af524 Mon Sep 17 00:00:00 2001 From: William Baker Date: Wed, 10 Oct 2018 15:35:54 -0700 Subject: [PATCH 264/272] Code cleanup --- .../Tests/GitCommands/CheckoutTests.cs | 8 +- .../Projection/GitIndexProjection.cs | 75 +++++++------------ 2 files changed, 33 insertions(+), 50 deletions(-) diff --git a/GVFS/GVFS.FunctionalTests/Tests/GitCommands/CheckoutTests.cs b/GVFS/GVFS.FunctionalTests/Tests/GitCommands/CheckoutTests.cs index ac861a982..eb0491296 100644 --- a/GVFS/GVFS.FunctionalTests/Tests/GitCommands/CheckoutTests.cs +++ b/GVFS/GVFS.FunctionalTests/Tests/GitCommands/CheckoutTests.cs @@ -900,9 +900,9 @@ public void CheckoutBranchDirectoryWithOneFileWrite() [TestCase] public void CheckoutBranchDirectoryWithOneDeepFileWrite() { - this.ControlGitRepo.Fetch(DeepDirectoryWithOneFile); - this.ControlGitRepo.Fetch(DeepDirectoryWithOneDifferentFile); - this.ValidateGitCommand($"checkout {DeepDirectoryWithOneFile}"); + this.ControlGitRepo.Fetch(GitRepoTests.DeepDirectoryWithOneFile); + this.ControlGitRepo.Fetch(GitRepoTests.DeepDirectoryWithOneDifferentFile); + this.ValidateGitCommand($"checkout {GitRepoTests.DeepDirectoryWithOneFile}"); this.FileShouldHaveContents( "TestFile1\n", "GitCommandsTests", @@ -926,7 +926,7 @@ public void CheckoutBranchDirectoryWithOneDeepFileWrite() this.ValidateGitCommand("add --all"); this.RunGitCommand("commit -m \"Some change\""); - this.ValidateGitCommand($"checkout {DeepDirectoryWithOneDifferentFile}"); + this.ValidateGitCommand($"checkout {GitRepoTests.DeepDirectoryWithOneDifferentFile}"); this.FileShouldHaveContents( "TestFile2\n", "GitCommandsTests", diff --git a/GVFS/GVFS.Virtualization/Projection/GitIndexProjection.cs b/GVFS/GVFS.Virtualization/Projection/GitIndexProjection.cs index a1a690986..fb8edba06 100644 --- a/GVFS/GVFS.Virtualization/Projection/GitIndexProjection.cs +++ b/GVFS/GVFS.Virtualization/Projection/GitIndexProjection.cs @@ -1391,59 +1391,42 @@ private void ReExpandFolder( childRelativePath = relativeFolderPath + Path.DirectorySeparatorChar + childEntry.Name.GetString(); } - if (childEntry.IsFolder) + bool newChild = childEntry.IsFolder ? !existingFolderPlaceholders.Contains(childRelativePath) : !updatedPlaceholderList.ContainsKey(childRelativePath); + + if (newChild) { - if (!existingFolderPlaceholders.Contains(childRelativePath)) + FileSystemResult result; + string fileShaOrFolderValue; + if (childEntry.IsFolder) { - FileSystemResult result = this.fileSystemVirtualizer.WritePlaceholderDirectory(childRelativePath); - switch (result.Result) - { - case FSResult.Ok: - updatedPlaceholderList.TryAdd( - childRelativePath, - new PlaceholderListDatabase.PlaceholderData(childRelativePath, PlaceholderListDatabase.PartialFolderValue)); - break; - - case FSResult.FileOrPathNotFound: - // Git command must have removed the folder being re-expanded (relativeFolderPath) - // Remove the folder from existingFolderPlaceholders so that its parent will create - // it again (when it's re-expanded) - existingFolderPlaceholders.Remove(relativeFolderPath); - return; - - default: - // TODO(Mac): Issue #245, handle failures of WritePlaceholderDirectory and WritePlaceholderFile - break; - } + fileShaOrFolderValue = PlaceholderListDatabase.PartialFolderValue; + result = this.fileSystemVirtualizer.WritePlaceholderDirectory(childRelativePath); } - } - else - { - if (!updatedPlaceholderList.ContainsKey(childRelativePath)) + else { FileData childFileData = childEntry as FileData; - string sha = childFileData.Sha.ToString(); + fileShaOrFolderValue = childFileData.Sha.ToString(); + result = this.fileSystemVirtualizer.WritePlaceholderFile(childRelativePath, childFileData.Size, fileShaOrFolderValue); + } - FileSystemResult result = this.fileSystemVirtualizer.WritePlaceholderFile(childRelativePath, childFileData.Size, sha); - switch (result.Result) - { - case FSResult.Ok: - updatedPlaceholderList.TryAdd( - childRelativePath, - new PlaceholderListDatabase.PlaceholderData(childRelativePath, sha)); - break; - - case FSResult.FileOrPathNotFound: - // Git command must have removed the folder being re-expanded (relativeFolderPath) - // Remove the folder from existingFolderPlaceholders so that its parent will create - // it again (when it's re-expanded) - existingFolderPlaceholders.Remove(relativeFolderPath); - return; + switch (result.Result) + { + case FSResult.Ok: + updatedPlaceholderList.TryAdd( + childRelativePath, + new PlaceholderListDatabase.PlaceholderData(childRelativePath, fileShaOrFolderValue)); + break; - default: - // TODO(Mac): Issue #245, handle failures of WritePlaceholderDirectory and WritePlaceholderFile - break; - } + case FSResult.FileOrPathNotFound: + // Git command must have removed the folder being re-expanded (relativeFolderPath) + // Remove the folder from existingFolderPlaceholders so that its parent will create + // it again (when it's re-expanded) + existingFolderPlaceholders.Remove(relativeFolderPath); + return; + + default: + // TODO(Mac): Issue #245, handle failures of WritePlaceholderDirectory and WritePlaceholderFile + break; } } } From c3f03f800fbed0890ceabe54fcd86458b12c44c5 Mon Sep 17 00:00:00 2001 From: Derrick Stolee Date: Thu, 11 Oct 2018 10:02:56 -0400 Subject: [PATCH 265/272] Update GitForWindows to include tracing --- GVFS/GVFS.Build/GVFS.props | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/GVFS/GVFS.Build/GVFS.props b/GVFS/GVFS.Build/GVFS.props index 2c39f7500..ef965a8c4 100644 --- a/GVFS/GVFS.Build/GVFS.props +++ b/GVFS/GVFS.Build/GVFS.props @@ -3,7 +3,7 @@ 0.2.173.2 - 2.20180814.4 + 2.20181011.3 From 34a54f91bdfe11547269139e37ed49323247f58a Mon Sep 17 00:00:00 2001 From: Kevin Willford Date: Wed, 10 Oct 2018 11:08:35 -0600 Subject: [PATCH 266/272] Add tests for creating placeholders --- .../GitCommands/CreatePlaceholderTests.cs | 73 +++++++++++++++++++ 1 file changed, 73 insertions(+) create mode 100644 GVFS/GVFS.FunctionalTests/Tests/GitCommands/CreatePlaceholderTests.cs diff --git a/GVFS/GVFS.FunctionalTests/Tests/GitCommands/CreatePlaceholderTests.cs b/GVFS/GVFS.FunctionalTests/Tests/GitCommands/CreatePlaceholderTests.cs new file mode 100644 index 000000000..ca0f37162 --- /dev/null +++ b/GVFS/GVFS.FunctionalTests/Tests/GitCommands/CreatePlaceholderTests.cs @@ -0,0 +1,73 @@ +using GVFS.FunctionalTests.Should; +using GVFS.FunctionalTests.Tools; +using NUnit.Framework; +using System.IO; +using System.Runtime.InteropServices; +using System.Threading; + +namespace GVFS.FunctionalTests.Tests.GitCommands +{ + [TestFixture] + [Category(Categories.GitCommands)] + public class CreatePlaceholderTests : GitRepoTests + { + private static readonly string FileToRead = Path.Combine("GVFS", "GVFS", "Program.cs"); + + public CreatePlaceholderTests() : base(enlistmentPerTest: true) + { + } + + [TestCase("check-attr")] + [TestCase("check-ignore")] + [TestCase("check-mailmap")] + [TestCase("diff-tree")] + [TestCase("fetch-pack")] + [TestCase("hash-object")] + [TestCase("index-pack")] + [TestCase("name-rev")] + [TestCase("notes")] + [TestCase("send-pack")] + [TestCase("rev-list")] + [TestCase("update-ref")] + public void AllowsPlaceholderCreationWhileGitCommandIsRunning(string commandToRun) + { + this.CheckPlaceholderCreation(commandToRun, shouldAllow: true); + } + + [TestCase("checkout-index")] + [TestCase("reset")] + [TestCase("update-index")] + public void BlocksPlaceholderCreationWhileGitCommandIsRunning(string commandToRun) + { + this.CheckPlaceholderCreation(commandToRun, shouldAllow: false); + } + + private void CheckPlaceholderCreation(string command, bool shouldAllow) + { + string eofCharacter = "\x04"; + if (RuntimeInformation.IsOSPlatform(OSPlatform.Windows)) + { + eofCharacter = "\x1A"; + } + + this.EditFile($"Some new content for {command}.", "Protocol.md"); + ManualResetEventSlim resetEvent = GitHelpers.RunGitCommandWithWaitAndStdIn(this.Enlistment, resetTimeout: 3000, command: $"{command} --stdin", stdinToQuit: eofCharacter, processId: out _); + + if (shouldAllow) + { + this.FileContentsShouldMatch(FileToRead); + } + else + { + string virtualPath = Path.Combine(this.Enlistment.RepoRoot, FileToRead); + string controlPath = Path.Combine(this.ControlGitRepo.RootPath, FileToRead); + virtualPath.ShouldNotExistOnDisk(this.FileSystem); + controlPath.ShouldBeAFile(this.FileSystem); + } + + this.ValidateGitCommand("--no-optional-locks status"); + resetEvent.Wait(); + this.RunGitCommand("reset --hard"); + } + } +} From 7ff203c8bc8dd69ab0bb0fb7d15c31bce7821841 Mon Sep 17 00:00:00 2001 From: Kevin Willford Date: Tue, 9 Oct 2018 10:45:06 -0600 Subject: [PATCH 267/272] Add test that adds untracked file in new folder 2 levels deep --- .../Tests/EnlistmentPerFixture/GitFilesTests.cs | 1 + .../Tests/GitCommands/GitCommandsTests.cs | 5 +++++ GVFS/GVFS.FunctionalTests/Tools/GVFSHelpers.cs | 2 +- 3 files changed, 7 insertions(+), 1 deletion(-) diff --git a/GVFS/GVFS.FunctionalTests/Tests/EnlistmentPerFixture/GitFilesTests.cs b/GVFS/GVFS.FunctionalTests/Tests/EnlistmentPerFixture/GitFilesTests.cs index 3313fb8f6..b90943d12 100644 --- a/GVFS/GVFS.FunctionalTests/Tests/EnlistmentPerFixture/GitFilesTests.cs +++ b/GVFS/GVFS.FunctionalTests/Tests/EnlistmentPerFixture/GitFilesTests.cs @@ -155,6 +155,7 @@ public void CaseOnlyRenameOfNewFolderKeepsModifiedPathsEntries() this.Enlistment.WaitForBackgroundOperations().ShouldEqual(true, "Background operations failed to complete."); GVFSHelpers.ModifiedPathsShouldContain(this.fileSystem, this.Enlistment.DotGVFSRoot, "Folder/"); + GVFSHelpers.ModifiedPathsShouldNotContain(this.fileSystem, this.Enlistment.DotGVFSRoot, "Folder/testfile"); this.fileSystem.RenameDirectory(this.Enlistment.RepoRoot, "Folder", "folder"); this.Enlistment.WaitForBackgroundOperations().ShouldEqual(true, "Background operations failed to complete."); diff --git a/GVFS/GVFS.FunctionalTests/Tests/GitCommands/GitCommandsTests.cs b/GVFS/GVFS.FunctionalTests/Tests/GitCommands/GitCommandsTests.cs index 31ac5cb08..28ac30802 100644 --- a/GVFS/GVFS.FunctionalTests/Tests/GitCommands/GitCommandsTests.cs +++ b/GVFS/GVFS.FunctionalTests/Tests/GitCommands/GitCommandsTests.cs @@ -16,6 +16,8 @@ public class GitCommandsTests : GitRepoTests private const string EncodingFilename = "ريلٌأكتوبرûمارسأغسطسºٰٰۂْٗ۵ريلٌأك.txt"; private const string ContentWhenEditingFile = "// Adding a comment to the file"; private const string UnknownTestName = "Unknown"; + private const string TopLevelFolderToCreate = "level1"; + private const string SubFolderToCreate = "level2"; private static readonly string EditFilePath = Path.Combine("GVFS", "GVFS.Common", "GVFSContext.cs"); private static readonly string DeleteFilePath = Path.Combine("GVFS", "GVFS", "Program.cs"); @@ -1088,6 +1090,9 @@ private void CommitChangesSwitchBranchSwitchBack(Action fileSystemAction, [Calle private void CreateFile() { this.CreateFile("Some content here", Path.GetRandomFileName() + "tempFile.txt"); + this.CreateFolder(TopLevelFolderToCreate); + this.CreateFolder(Path.Combine(TopLevelFolderToCreate, SubFolderToCreate)); + this.CreateFile("File in new folder", Path.Combine(TopLevelFolderToCreate, SubFolderToCreate, Path.GetRandomFileName() + "folderFile.txt")); } private void EditFile() diff --git a/GVFS/GVFS.FunctionalTests/Tools/GVFSHelpers.cs b/GVFS/GVFS.FunctionalTests/Tools/GVFSHelpers.cs index 0de0d127c..5dbc77982 100644 --- a/GVFS/GVFS.FunctionalTests/Tools/GVFSHelpers.cs +++ b/GVFS/GVFS.FunctionalTests/Tools/GVFSHelpers.cs @@ -117,7 +117,7 @@ public static void ModifiedPathsShouldContain(FileSystemRunner fileSystem, strin string[] modifedPathLines = modifedPathsContents.Split(new[] { ModifiedPathsNewLine }, StringSplitOptions.None); foreach (string gitPath in gitPaths) { - modifedPathLines.ShouldContain(path => path.Equals(ModifedPathsLineAddPrefix + gitPath)); + modifedPathLines.ShouldContain(path => path.Equals(ModifedPathsLineAddPrefix + gitPath, StringComparison.OrdinalIgnoreCase)); } } From 03fba043f131d376af455f058478c29289544554 Mon Sep 17 00:00:00 2001 From: Kevin Willford Date: Tue, 9 Oct 2018 14:07:26 -0600 Subject: [PATCH 268/272] Update git for windows version --- GVFS/GVFS.Build/GVFS.props | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/GVFS/GVFS.Build/GVFS.props b/GVFS/GVFS.Build/GVFS.props index 2c39f7500..ef965a8c4 100644 --- a/GVFS/GVFS.Build/GVFS.props +++ b/GVFS/GVFS.Build/GVFS.props @@ -3,7 +3,7 @@ 0.2.173.2 - 2.20180814.4 + 2.20181011.3 From 33f8566e0f89097be8b1f31b497caebdc6343f95 Mon Sep 17 00:00:00 2001 From: Kevin Willford Date: Thu, 11 Oct 2018 13:09:34 -0600 Subject: [PATCH 269/272] Add try/catch/finally for the task running git command with stdin --- GVFS/GVFS.FunctionalTests/Tools/GitHelpers.cs | 42 ++++++++++++------- 1 file changed, 26 insertions(+), 16 deletions(-) diff --git a/GVFS/GVFS.FunctionalTests/Tools/GitHelpers.cs b/GVFS/GVFS.FunctionalTests/Tools/GitHelpers.cs index 333499ba0..bc1d8dd54 100644 --- a/GVFS/GVFS.FunctionalTests/Tools/GitHelpers.cs +++ b/GVFS/GVFS.FunctionalTests/Tools/GitHelpers.cs @@ -1,5 +1,6 @@ using GVFS.FunctionalTests.Properties; using GVFS.Tests.Should; +using NUnit.Framework; using System; using System.Collections.Generic; using System.Diagnostics; @@ -174,28 +175,37 @@ private static ManualResetEventSlim RunCommandWithWaitAndStdIn( { resetEvent.Wait(resetTimeout); - // Make sure to let the holding process end. - if (stdin != null) + try { - stdin.WriteLine(stdinToQuit); - stdin.Close(); - } - - if (holdingProcess != null) - { - bool holdingProcessHasExited = holdingProcess.WaitForExit(10000); - - if (!holdingProcess.HasExited) + // Make sure to let the holding process end. + if (stdin != null) { - holdingProcess.Kill(); + stdin.WriteLine(stdinToQuit); + stdin.Close(); } - holdingProcess.Dispose(); + if (holdingProcess != null) + { + bool holdingProcessHasExited = holdingProcess.WaitForExit(10000); - holdingProcessHasExited.ShouldBeTrue("Locking process did not exit in time."); - } + if (!holdingProcess.HasExited) + { + holdingProcess.Kill(); + } + + holdingProcess.Dispose(); - resetEvent.Set(); + holdingProcessHasExited.ShouldBeTrue("Locking process did not exit in time."); + } + } + catch (Exception ex) + { + Assert.Fail($"{nameof(RunCommandWithWaitAndStdIn)} exception closing stdin {ex.ToString()}"); + } + finally + { + resetEvent.Set(); + } }); return resetEvent; From a5c027e8425d77c3d374dce3c02f6c094620ba3d Mon Sep 17 00:00:00 2001 From: Nick Graczyk Date: Thu, 11 Oct 2018 14:35:31 -0700 Subject: [PATCH 270/272] Address PR feedback and update MirrorProvider script too --- MirrorProvider/Scripts/Mac/Build.sh | 5 +++++ Scripts/Mac/BuildGVFSForMac.sh | 7 +++---- 2 files changed, 8 insertions(+), 4 deletions(-) diff --git a/MirrorProvider/Scripts/Mac/Build.sh b/MirrorProvider/Scripts/Mac/Build.sh index a4b404697..94617466d 100755 --- a/MirrorProvider/Scripts/Mac/Build.sh +++ b/MirrorProvider/Scripts/Mac/Build.sh @@ -14,6 +14,11 @@ SLN=$SRCDIR/MirrorProvider/MirrorProvider.sln # Build the ProjFS kext and libraries $SRCDIR/ProjFS.Mac/Scripts/Build.sh $CONFIGURATION +# If we're building the Profiling(Release) configuration, remove Profiling() for building .NET code +if [ "$CONFIGURATION" == "Profiling(Release)" ]; then + CONFIGURATION=Release +fi + # Build the MirrorProvider dotnet restore $SLN /p:Configuration="$CONFIGURATION.Mac" --packages $ROOTDIR/packages dotnet build $SLN --configuration $CONFIGURATION.Mac diff --git a/Scripts/Mac/BuildGVFSForMac.sh b/Scripts/Mac/BuildGVFSForMac.sh index d6d07e4ad..907854e4a 100755 --- a/Scripts/Mac/BuildGVFSForMac.sh +++ b/Scripts/Mac/BuildGVFSForMac.sh @@ -43,10 +43,9 @@ if [ "$CONFIGURATION" == "Profiling(Release)" ]; then CONFIGURATION=Release fi -DOTNETCONFIGURATION=$CONFIGURATION.Mac -dotnet restore $SRCDIR/GVFS.sln /p:Configuration=$DOTNETCONFIGURATION --packages $PACKAGES || exit 1 -dotnet build $SRCDIR/GVFS.sln --runtime osx-x64 --framework netcoreapp2.1 --configuration $DOTNETCONFIGURATION /maxcpucount:1 || exit 1 -dotnet publish $SRCDIR/GVFS.sln /p:Configuration=$DOTNETCONFIGURATION /p:Platform=x64 --runtime osx-x64 --framework netcoreapp2.1 --self-contained --output $PUBLISHDIR /maxcpucount:1 || exit 1 +dotnet restore $SRCDIR/GVFS.sln /p:Configuration=$CONFIGURATION.Mac --packages $PACKAGES || exit 1 +dotnet build $SRCDIR/GVFS.sln --runtime osx-x64 --framework netcoreapp2.1 --configuration $CONFIGURATION.Mac /maxcpucount:1 || exit 1 +dotnet publish $SRCDIR/GVFS.sln /p:Configuration=$CONFIGURATION.Mac /p:Platform=x64 --runtime osx-x64 --framework netcoreapp2.1 --self-contained --output $PUBLISHDIR /maxcpucount:1 || exit 1 NATIVEDIR=$SRCDIR/GVFS/GVFS.Native.Mac xcodebuild -configuration $CONFIGURATION -workspace $NATIVEDIR/GVFS.Native.Mac.xcworkspace build -scheme GVFS.Native.Mac -derivedDataPath $ROOTDIR/BuildOutput/GVFS.Native.Mac || exit 1 From 23d457d7ad34251d935f416bbbf91e79db256293 Mon Sep 17 00:00:00 2001 From: Kevin Willford Date: Thu, 11 Oct 2018 15:42:46 -0600 Subject: [PATCH 271/272] Fix commands that need additional parameters with stdin --- .../GitCommands/CreatePlaceholderTests.cs | 33 ++++++++++--------- 1 file changed, 17 insertions(+), 16 deletions(-) diff --git a/GVFS/GVFS.FunctionalTests/Tests/GitCommands/CreatePlaceholderTests.cs b/GVFS/GVFS.FunctionalTests/Tests/GitCommands/CreatePlaceholderTests.cs index ca0f37162..3704e5e28 100644 --- a/GVFS/GVFS.FunctionalTests/Tests/GitCommands/CreatePlaceholderTests.cs +++ b/GVFS/GVFS.FunctionalTests/Tests/GitCommands/CreatePlaceholderTests.cs @@ -17,26 +17,27 @@ public CreatePlaceholderTests() : base(enlistmentPerTest: true) { } - [TestCase("check-attr")] - [TestCase("check-ignore")] - [TestCase("check-mailmap")] - [TestCase("diff-tree")] - [TestCase("fetch-pack")] - [TestCase("hash-object")] - [TestCase("index-pack")] - [TestCase("name-rev")] - [TestCase("notes")] - [TestCase("send-pack")] - [TestCase("rev-list")] - [TestCase("update-ref")] + [TestCase("check-attr --stdin --all")] + [TestCase("check-ignore --stdin")] + [TestCase("check-mailmap --stdin")] + [TestCase("diff-tree --stdin")] + [TestCase("hash-object --stdin")] + [TestCase("index-pack --stdin")] + [TestCase("name-rev --stdin")] + [TestCase("rev-list --stdin --quiet --all")] + [TestCase("update-ref --stdin")] public void AllowsPlaceholderCreationWhileGitCommandIsRunning(string commandToRun) { this.CheckPlaceholderCreation(commandToRun, shouldAllow: true); } - [TestCase("checkout-index")] - [TestCase("reset")] - [TestCase("update-index")] + [TestCase("checkout-index --stdin")] + [TestCase("fetch-pack --stdin URL")] + [TestCase("notes copy --stdin")] + [TestCase("reset --stdin")] + [TestCase("send-pack --stdin URL")] + [TestCase("update-index --stdin")] + [Category(Categories.WindowsOnly)] // Mac never blocks placeholder creation public void BlocksPlaceholderCreationWhileGitCommandIsRunning(string commandToRun) { this.CheckPlaceholderCreation(commandToRun, shouldAllow: false); @@ -51,7 +52,7 @@ private void CheckPlaceholderCreation(string command, bool shouldAllow) } this.EditFile($"Some new content for {command}.", "Protocol.md"); - ManualResetEventSlim resetEvent = GitHelpers.RunGitCommandWithWaitAndStdIn(this.Enlistment, resetTimeout: 3000, command: $"{command} --stdin", stdinToQuit: eofCharacter, processId: out _); + ManualResetEventSlim resetEvent = GitHelpers.RunGitCommandWithWaitAndStdIn(this.Enlistment, resetTimeout: 3000, command: $"{command}", stdinToQuit: eofCharacter, processId: out _); if (shouldAllow) { From ed46b0eb18c09b11237dcec0815cb49fa7b71b63 Mon Sep 17 00:00:00 2001 From: Derrick Stolee Date: Fri, 12 Oct 2018 00:37:09 +0000 Subject: [PATCH 272/272] GVFS.props: use release build with rebase fix --- GVFS/GVFS.Build/GVFS.props | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/GVFS/GVFS.Build/GVFS.props b/GVFS/GVFS.Build/GVFS.props index ef965a8c4..48a82f3ff 100644 --- a/GVFS/GVFS.Build/GVFS.props +++ b/GVFS/GVFS.Build/GVFS.props @@ -3,7 +3,7 @@ 0.2.173.2 - 2.20181011.3 + 2.20181012.4