Skip to content

Add filesystem audit events for Panorama Public symlinks #361

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
vagisha merged 3 commits into from
Jul 13, 2023
Merged

Add filesystem audit events for Panorama Public symlinks #361

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
5b035c5
- Added filesystem audit events when symlinks are created / updated, …
vagisha Jul 6, 2023
34968e5
Merge remote-tracking branch 'remotes/origin/release23.7-SNAPSHOT' in…
vagisha Jul 12, 2023
3dd0499
Use FileContentService.getContainersForFilePath(path) after bugfix
vagisha Jul 13, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
11 changes: 9 additions & 2 deletions panoramapublic/src/org/labkey/panoramapublic/PanoramaPublicFileImporter.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -85,8 +85,15 @@ public void process(@Nullable PipelineJob job, FolderImportContext ctx, VirtualF
Files.createDirectories(targetFiles.toPath());
}

log.info("Moving files and creating sym links in folder " + ctx.getContainer().getPath());
PanoramaPublicSymlinkManager.get().moveAndSymLinkDirectory(expJob, sourceFiles, targetFiles, false, log);
if (expJob.isMoveAndSymlink())
{
log.info("Moving files to folder " + expJob.getContainer().getPath() + " and creating symlinks");
}
else
{
log.info("Copying files to folder " + expJob.getContainer().getPath());
}
PanoramaPublicSymlinkManager.get().moveAndSymLinkDirectory(expJob, sourceFiles, targetFiles, log);

alignDataFileUrls(expJob.getUser(), ctx.getContainer(), log);
}
Expand Down
2 changes: 1 addition & 1 deletion panoramapublic/src/org/labkey/panoramapublic/PanoramaPublicFileListener.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -33,7 +33,7 @@ public void fileCreated(@NotNull File created, @Nullable User user, @Nullable Co
public int fileMoved(@NotNull File src, @NotNull File dest, @Nullable User user, @Nullable Container container)
{
// Update any symlinks targeting the file
PanoramaPublicSymlinkManager.get().fireSymlinkUpdate(src.toPath(), dest.toPath(), container);
PanoramaPublicSymlinkManager.get().fireSymlinkUpdate(src.toPath(), dest.toPath(), container, user);

ExpData data = ExperimentService.get().getExpDataByURL(src, null);
if (null != data)
Expand Down
6 changes: 3 additions & 3 deletions panoramapublic/src/org/labkey/panoramapublic/PanoramaPublicListener.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -75,13 +75,13 @@ public void containerDeleted(Container c, User user)
{
JournalManager.deleteProjectJournal(c, user);

PanoramaPublicSymlinkManager.get().beforeContainerDeleted(c);
PanoramaPublicSymlinkManager.get().beforeContainerDeleted(c, user);
}

@Override
public void containerMoved(Container c, Container oldParent, User user)
{
PanoramaPublicSymlinkManager.get().fireSymlinkUpdateContainer(oldParent, c);
PanoramaPublicSymlinkManager.get().fireSymlinkUpdateContainer(oldParent, c, user);
}

@Override
Expand Down Expand Up @@ -109,7 +109,7 @@ public void propertyChange(PropertyChangeEvent evt)
// ce.getOldValue() and ce.getNewValue() are just the names of the old and new containers. We need the full path.
Path oldPath = parentPath.resolve((String) ce.getOldValue());
Path newPath = parentPath.resolve((String) ce.getNewValue());
PanoramaPublicSymlinkManager.get().fireSymlinkUpdateContainer(oldPath.toString(), newPath.toString(), c);
PanoramaPublicSymlinkManager.get().fireSymlinkUpdateContainer(oldPath.toString(), newPath.toString(), c, ce.user);
}
}
}
Expand Down
5 changes: 4 additions & 1 deletion panoramapublic/src/org/labkey/panoramapublic/PanoramaPublicSymlinkHandler.java
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,9 +1,12 @@
package org.labkey.panoramapublic;

import org.labkey.api.data.Container;
import org.labkey.api.security.User;

import java.io.IOException;
import java.nio.file.Path;

public interface PanoramaPublicSymlinkHandler
{
void handleSymlink(Path link, Path target) throws IOException;
void handleSymlink(Path link, Path target, Container container, User user) throws IOException;
}
117 changes: 87 additions & 30 deletions panoramapublic/src/org/labkey/panoramapublic/PanoramaPublicSymlinkManager.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -4,11 +4,14 @@
import org.apache.commons.lang3.SystemUtils;
import org.apache.logging.log4j.Logger;
import org.jetbrains.annotations.Nullable;
import org.labkey.api.audit.AuditLogService;
import org.labkey.api.audit.provider.FileSystemAuditProvider;
import org.labkey.api.data.Container;
import org.labkey.api.data.ContainerManager;
import org.labkey.api.data.ContainerService;
import org.labkey.api.files.FileContentService;
import org.labkey.api.module.ModuleLoader;
import org.labkey.api.security.User;
import org.labkey.api.util.FileUtil;
import org.labkey.api.util.logging.LogHelper;
import org.labkey.panoramapublic.model.ExperimentAnnotations;
Expand Down Expand Up @@ -54,13 +57,13 @@ public static PanoramaPublicSymlinkManager get()
}


private void handleContainerSymlinks(File source, PanoramaPublicSymlinkHandler handler)
private void handleContainerSymlinks(File source, PanoramaPublicSymlinkHandler handler, Container container, User user)
{
for (File file : Objects.requireNonNull(source.listFiles()))
{
if (file.isDirectory())
{
handleContainerSymlinks(file, handler);
handleContainerSymlinks(file, handler, container, user);
}
else
{
Expand All @@ -69,7 +72,7 @@ private void handleContainerSymlinks(File source, PanoramaPublicSymlinkHandler h
{
try {
Path target = Files.readSymbolicLink(filePath);
handler.handleSymlink(filePath, target);
handler.handleSymlink(filePath, target, container, user);
} catch (IOException x) {
_log.error("Unable to resolve symlink target for symlink at " + filePath);
}
Expand All @@ -78,27 +81,27 @@ private void handleContainerSymlinks(File source, PanoramaPublicSymlinkHandler h
}
}

public void handleContainerSymlinks(Container container, PanoramaPublicSymlinkHandler handler)
public void handleContainerSymlinks(Container container, User user, PanoramaPublicSymlinkHandler handler)
{
FileContentService fcs = FileContentService.get();
if (null != fcs)
{
File root = fcs.getFileRoot(container);
if (null != root)
{
handleContainerSymlinks(root, handler);
handleContainerSymlinks(root, handler, container, user);
}
}
}

private void handleAllSymlinks(Set<Container> containers, PanoramaPublicSymlinkHandler handler)
private void handleAllSymlinks(Set<Container> containers, User user, PanoramaPublicSymlinkHandler handler)
{
for (Container container : containers)
{
Set<Container> tree = ContainerManager.getAllChildren(container);
for (Container node : tree)
{
handleContainerSymlinks(node, handler);
handleContainerSymlinks(node, user, handler);
}
}
}
Expand All @@ -112,7 +115,7 @@ private String normalizeContainerPath(String path)
return File.separator + path + File.separator;
}

public void beforeContainerDeleted(Container container)
public void beforeContainerDeleted(Container container, User user)
{
if (PanoramaPublicManager.canBeSymlinkTarget(container)) // Fire the event only if the container being deleted is in the Panorama Public project.
{
Expand All @@ -121,7 +124,7 @@ public void beforeContainerDeleted(Container container)
ExperimentAnnotations expAnnot = ExperimentAnnotationsManager.getExperimentIncludesContainer(container);
if (null != expAnnot)
{
fireSymlinkCopiedExperimentDelete(expAnnot, container);
fireSymlinkCopiedExperimentDelete(expAnnot, container, user);
}
}
}
Expand All @@ -132,7 +135,7 @@ public void beforeContainerDeleted(Container container)
* @param container container being deleted. This could be a subfolder of the experiment container if the experiment
* is configured to include subfolders.
*/
private void fireSymlinkCopiedExperimentDelete(ExperimentAnnotations expAnnot, Container container)
private void fireSymlinkCopiedExperimentDelete(ExperimentAnnotations expAnnot, Container container, User user)
{
if (expAnnot.getDataVersion() != null && !ExperimentAnnotationsManager.isCurrentVersion(expAnnot))
{
Expand All @@ -153,17 +156,19 @@ private void fireSymlinkCopiedExperimentDelete(ExperimentAnnotations expAnnot, C
if (nextHighestVersion != null)
{
Container versionContainer = nextHighestVersion.getContainer();
handleContainerSymlinks(versionContainer, (link, target) -> {
handleContainerSymlinks(versionContainer, user, (link, target, c, u) -> {
if (!target.startsWith(deletedContainerPath))
{
return;
}
Files.move(target, link, REPLACE_EXISTING); // Move the files back to the next highest version of the experiment
addReplaceSymlinkWithTargetAuditEvent(link, target, c, u);

_log.info("File moved from " + target + " to " + link);

// This should update the symlinks in the submitted folder as well as
// symlinks in versions older than this one to point to the files in the next highest version.
fireSymlinkUpdate(target, link, container);
fireSymlinkUpdate(target, link, container, user);
});
}
}
Expand All @@ -180,12 +185,14 @@ private void fireSymlinkCopiedExperimentDelete(ExperimentAnnotations expAnnot, C
}
if (null != sourceContainer)
{
handleContainerSymlinks(sourceContainer, (link, target) -> {
handleContainerSymlinks(sourceContainer, user, (link, target, c, u) -> {
if (!target.startsWith(deletedContainerPath))
{
return;
}
Files.move(target, link, REPLACE_EXISTING);
addReplaceSymlinkWithTargetAuditEvent(link, target, c, u);

_log.info("File moved from " + target + " to " + link);

// Symlinks in the source container point to -> current version container on Panorama Public
Expand All @@ -197,35 +204,36 @@ private void fireSymlinkCopiedExperimentDelete(ExperimentAnnotations expAnnot, C
}
}

public void fireSymlinkUpdateContainer(Container oldContainer, Container newContainer)
public void fireSymlinkUpdateContainer(Container oldContainer, Container newContainer, User user)
{
// Update symlinks to new target
FileContentService fcs = FileContentService.get();
if (fcs != null)
{
if (fcs.getFileRoot(oldContainer) != null && fcs.getFileRoot(newContainer) != null)
{
fireSymlinkUpdateContainer(fcs.getFileRoot(oldContainer).getPath(), fcs.getFileRoot(newContainer).getPath(), oldContainer);
fireSymlinkUpdateContainer(fcs.getFileRoot(oldContainer).getPath(), fcs.getFileRoot(newContainer).getPath(), oldContainer, user);
}
}
}

public void fireSymlinkUpdateContainer(String oldContainer, String newContainer, Container container)
public void fireSymlinkUpdateContainer(String oldContainer, String newContainer, Container container, User user)
{
if (PanoramaPublicManager.canBeSymlinkTarget(container))
{
String oldContainerPath = normalizeContainerPath(oldContainer);
String newContainerPath = normalizeContainerPath(newContainer);

Set<Container> containers = getSymlinkContainers(container);
handleAllSymlinks(containers, (link, target) -> {
handleAllSymlinks(containers, user, (link, target, c, u) -> {
if (String.valueOf(target).contains(oldContainerPath))
{
Path newTarget = Path.of(target.toString().replace(oldContainerPath, newContainerPath));
try
{
Files.delete(link);
Files.createSymbolicLink(link, newTarget);
addLinkUpdatedAuditEvent(link, newTarget, c, u);
}
catch (IOException e)
{
Expand All @@ -236,20 +244,53 @@ public void fireSymlinkUpdateContainer(String oldContainer, String newContainer,
}
}

public void fireSymlinkUpdate(Path oldTarget, Path newTarget, Container container)
private void addLinkUpdatedAuditEvent(Path link, Path newTarget, Container linkContainer, User user)
{
addFileAuditEvent(link, linkContainer, user, "Updated symlink target to " + newTarget);
}

private void addReplaceSymlinkWithTargetAuditEvent(Path link, Path target, Container linkContainer, User user)
{
addFileAuditEvent(link, linkContainer, user, "Replaced symlink with target file " + target);
}

private void addReplaceTargetWithSymlinkAuditEvent(Path link, Path target, Container container, User user)
{
addFileAuditEvent(link, container, user, "Replaced target file with symlink to " + target);
}

private void addFileAuditEvent(Path link, Container container, User user, String comment)
{
FileSystemAuditProvider.FileSystemAuditEvent event = new FileSystemAuditProvider.FileSystemAuditEvent(
container != null ? container.getId() : null, comment);
event.setFile(link.toString());
AuditLogService.get().addEvent(user, event);
}

public void fireSymlinkUpdate(Path oldTarget, Path newTarget, Container container, User user)
{
fireSymlinkUpdate(oldTarget, newTarget, container, user, null);
}

public void fireSymlinkUpdate(Path oldTarget, Path newTarget, Container container, User user, @Nullable Logger log)
{
if (PanoramaPublicManager.canBeSymlinkTarget(container)) // Only files in the Panorama Public project can be symlink targets.
{
Set<Container> containers = getSymlinkContainers(container);

handleAllSymlinks(containers, (link, target) -> {
handleAllSymlinks(containers, user, (link, target, c, u) -> {
if (!target.equals(oldTarget))
return;

try
{
Files.delete(link);
Files.createSymbolicLink(link, newTarget);
addLinkUpdatedAuditEvent(link, newTarget, c, u);
if (log != null)
{
log.info("Target for symlink " + link + " updated to " + newTarget);
}
}
catch (IOException e)
{
Expand All @@ -259,7 +300,7 @@ public void fireSymlinkUpdate(Path oldTarget, Path newTarget, Container containe
}
}

public void moveAndSymLinkDirectory(CopyExperimentPipelineJob job, File source, File target, boolean createSourceSymLinks, @Nullable Logger log) throws IOException
public void moveAndSymLinkDirectory(CopyExperimentPipelineJob job, File source, File target, @Nullable Logger log) throws IOException
{
if (null == log)
{
Expand All @@ -283,7 +324,7 @@ public void moveAndSymLinkDirectory(CopyExperimentPipelineJob job, File source,
log.debug("Directory created: " + targetPath);
}

moveAndSymLinkDirectory(job, file, targetPath.toFile(), createSourceSymLinks, log);
moveAndSymLinkDirectory(job, file, targetPath.toFile(), log);
}
else
{
Expand Down Expand Up @@ -313,6 +354,7 @@ public void moveAndSymLinkDirectory(CopyExperimentPipelineJob job, File source,

// Copy the file to panorama public
Files.copy(filePath, targetPath, REPLACE_EXISTING);
log.debug("Copied file " + filePath + " to " + targetPath);
fcs.fireFileCreateEvent(targetPath, job.getUser(), job.getContainer());

continue;
Expand All @@ -321,35 +363,50 @@ public void moveAndSymLinkDirectory(CopyExperimentPipelineJob job, File source,
// Symbolic link should move the target file over. This would be for a re-copy to public.
if (Files.isSymbolicLink(filePath))
{
log.debug("Source file is a symlink: " + filePath);

Path oldPath = Files.readSymbolicLink(filePath);
Files.move(oldPath, targetPath, REPLACE_EXISTING);
log.debug("Moved symlink target " + oldPath + " to " + targetPath);
fcs.fireFileCreateEvent(targetPath, job.getUser(), job.getContainer());

fireSymlinkUpdate(oldPath, targetPath, job.getContainer()); // job container is the target container on Panorama Public
log.debug("File moved from " + oldPath + " to " + targetPath);
fireSymlinkUpdate(oldPath, targetPath, job.getContainer(), // job container is the target container on Panorama Public
job.getUser(), log);

Path symlink = Files.createSymbolicLink(oldPath, targetPath);
log.debug("Symlink created: " + symlink);
Container oldTargetContainer = getContainerForFilePath(oldPath);
if (oldTargetContainer != null)
{
// The target of the symlink has been moved from a previous version on Panorama Public to the
// new version on Panorama Public, and a symink has been created in the previous version container.
// Add an audit event in the previous version container.
addReplaceTargetWithSymlinkAuditEvent(symlink, targetPath, oldTargetContainer, job.getUser());
}
log.debug("Replaced old target with symlink: " + symlink);
}
else
{
Files.move(filePath, targetPath, REPLACE_EXISTING);
log.debug("Moved file " + filePath + " to " + targetPath);
fcs.fireFileCreateEvent(targetPath, job.getUser(), job.getContainer());

Files.createSymbolicLink(filePath, targetPath);
log.debug("Created symlink " + filePath + " targeting " + targetPath);
// We don't need to update any symlinks here since the source container should not have any symlink targets.
}

if (createSourceSymLinks)
{
Path symlink = Files.createSymbolicLink(filePath, targetPath);
log.debug("Symlink created: " + symlink);
}
}
}
}
}

// Get the container for the given file path.
// Example: if the file path is C:\Users\vsharma\WORK\LabKey\build\deploy\files\Panorama Public\TestProject V.1\@files\Study9S_Site52_v1.sky.zip
// this will return the container for "/Panorama Public/TestProject V.1"
private Container getContainerForFilePath(Path path)
{
return FileContentService.get().getContainersForFilePath(path).stream().findFirst().orElse(null);
}

private void verifyFileTreeSymlinks(File source, Map<String, String> linkInvalidTarget, Map<String, String> linkWithSymlinkTarget) throws IOException
{
for (File file : Objects.requireNonNull(source.listFiles()))
Expand Down
Loading