From dbab2c413fc753e6ec4db55f9e8a3d1a1cec3552 Mon Sep 17 00:00:00 2001 From: toddn Date: Sun, 20 Dec 2020 16:24:25 -0600 Subject: [PATCH 01/11] adding a bulk delete files method --- app/api/Files.scala | 43 ++++++++++++++++++++----------- app/controllers/Application.scala | 17 +++++------- conf/routes | 1 + 3 files changed, 35 insertions(+), 26 deletions(-) diff --git a/app/api/Files.scala b/app/api/Files.scala index 3f432805a..098ce93fa 100644 --- a/app/api/Files.scala +++ b/app/api/Files.scala @@ -1,14 +1,8 @@ package api -import scala.annotation.tailrec -import java.io.FileInputStream -import java.net.{URL, URLEncoder} - -import javax.inject.Inject -import javax.mail.internet.MimeUtility -import _root_.util.{FileUtils, JSONLD, Parsers, RequestUtils, SearchUtils} +import _root_.util._ import com.mongodb.casbah.Imports._ -import controllers.Previewers +import controllers.{Previewers, Utils} import jsonutils.JsonUtil import models._ import play.api.Logger @@ -18,16 +12,15 @@ import play.api.libs.concurrent.Execution.Implicits._ import play.api.libs.iteratee.Enumerator import play.api.libs.json.Json._ import play.api.libs.json._ -import play.api.mvc.{Action, ResponseHeader, Result, SimpleResult} +import play.api.mvc.{ResponseHeader, SimpleResult} import services._ +import services.s3.S3ByteStorageService -import scala.collection.mutable.ListBuffer -import scala.util.parsing.json.JSONArray -import java.text.SimpleDateFormat +import java.io.FileInputStream +import java.net.URL import java.util.Date - -import controllers.Utils -import services.s3.S3ByteStorageService +import javax.inject.Inject +import scala.annotation.tailrec /** * Json API for files. @@ -1623,6 +1616,26 @@ class Files @Inject()( } } + def bulkDeleteFiles() = PrivateServerAction (parse.json) {implicit request=> + request.user match { + case Some(user) => { + val fileIds = request.body.\("fileIds").asOpt[List[String]].getOrElse(List.empty[String]) + if (fileIds.isEmpty){ + BadRequest("No file ids supplied") + } else { + for (fileId <- fileIds){ + if (UUID.isValid(fileId)){ + files.removeFile(UUID(fileId),Utils.baseUrl(request), request.apiKey, request.user) + } + } + } + Ok(toJson("Done")) + } + case None => { + BadRequest("No user supplied") + } + } + } def removeFile(id: UUID) = PermissionAction(Permission.DeleteFile, Some(ResourceRef(ResourceRef.file, id))) { implicit request => files.get(id) match { diff --git a/app/controllers/Application.scala b/app/controllers/Application.scala index 1952006e4..138f572f8 100644 --- a/app/controllers/Application.scala +++ b/app/controllers/Application.scala @@ -1,20 +1,14 @@ package controllers -import java.net.URL - -import javax.inject.{Inject, Singleton} -import api.Permission -import api.Permission._ -import play.api.{Logger, Play, Routes} +import models.{Event, UUID, UserStatus} +import play.api.Play.current import play.api.mvc.Action +import play.api.{Logger, Play, Routes} import services._ -import models.{Event, UUID, User, UserStatus} -import org.owasp.html.Sanitizers -import play.api.Logger -import play.api.libs.concurrent.Execution.Implicits._ -import play.api.Play.current import util.Formatters.sanitizeHTML +import java.net.URL +import javax.inject.{Inject, Singleton} import scala.collection.immutable.List import scala.collection.mutable.ListBuffer @@ -368,6 +362,7 @@ class Application @Inject() (files: FileService, collections: CollectionService, api.routes.javascript.Files.updateFileName, api.routes.javascript.Files.updateDescription, api.routes.javascript.Files.extract, + api.routes.javascript.Files.bulkDeleteFiles, api.routes.javascript.Files.removeFile, api.routes.javascript.Files.follow, api.routes.javascript.Files.unfollow, diff --git a/conf/routes b/conf/routes index 3bc41d686..2363795a7 100644 --- a/conf/routes +++ b/conf/routes @@ -387,6 +387,7 @@ GET /api/files/:id/blob # deprecrated use DELETE POST /api/files/:id/remove @api.Files.removeFile(id: UUID) +POST /api/files/bulkRemove @api.Files.bulkDeleteFiles() GET /api/files/:id/metadata @api.Files.get(id: UUID) POST /api/files/:id/metadata @api.Files.addMetadata(id: UUID) GET /api/files/:id/metadataDefinitions @api.Files.getMetadataDefinitions(id: UUID, space: Option[String] ?= None) From e633903fbe423261e8be49ae92e707e3713bd3b8 Mon Sep 17 00:00:00 2001 From: toddn Date: Mon, 4 Jan 2021 14:24:12 -0600 Subject: [PATCH 02/11] changelog entry --- CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 8659b7f4c..fab43c569 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -12,7 +12,7 @@ and this project adheres to [Semantic Versioning](http://semver.org/). - UpdateUserId.js to scripts/updates. This code adds user_id to each document in extractions collection in mongodb. user_id is taken from author id in uploads.files if exists, else it taken from author id in datasets collection. - Ability to submit multiple selected files within a dataset to an extractor. - +- api route for bulk delete files ### Fixed - GeospatialViewer preview tab should no longer show if it does not contain any rendered data. - Editor can now delete tags on files, datasets and sections. From 73e712c5e14ba7a6e60a5b376d8dfc41e504d2a5 Mon Sep 17 00:00:00 2001 From: tcnichol Date: Wed, 3 Mar 2021 14:59:15 -0600 Subject: [PATCH 03/11] preparing to check permissions in FileService method --- app/services/mongodb/MongoDBFileService.scala | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/app/services/mongodb/MongoDBFileService.scala b/app/services/mongodb/MongoDBFileService.scala index f2928e99a..7ef6fbc5a 100644 --- a/app/services/mongodb/MongoDBFileService.scala +++ b/app/services/mongodb/MongoDBFileService.scala @@ -39,6 +39,7 @@ import play.api.Play._ import com.mongodb.casbah.Imports._ import models.FileStatus.FileStatus import org.bson.types.ObjectId +import api.Permission /** @@ -817,6 +818,21 @@ class MongoDBFileService @Inject() ( if(!file.isIntermediate){ val fileDatasets = datasets.findByFileIdDirectlyContain(file.id) for(fileDataset <- fileDatasets){ + val datasetSpaces : List[UUID] = fileDataset.spaces + var canDeleteFile = false + if (file.author.id == user.get.id ) { + canDeleteFile = true + } + for (currentSpace <- datasetSpaces) { + userService.getUserRoleInSpace(user.get.id, currentSpace) match { + case Some(role) => { + if (role.permissions.contains(Permission.DeleteFile)){ + canDeleteFile = true + } + } + case None => + } + } datasets.removeFile(fileDataset.id, id) if(!file.xmlMetadata.isEmpty){ datasets.index(fileDataset.id) From 6547e768049f317f8085a1165d38ac1d7c054886 Mon Sep 17 00:00:00 2001 From: tcnichol Date: Fri, 5 Mar 2021 18:05:57 -0600 Subject: [PATCH 04/11] adding check to see if user can delete file needs to be improved, moved to separate method --- app/services/mongodb/MongoDBFileService.scala | 25 +++++++++++-------- 1 file changed, 14 insertions(+), 11 deletions(-) diff --git a/app/services/mongodb/MongoDBFileService.scala b/app/services/mongodb/MongoDBFileService.scala index 7ef6fbc5a..e3ba4ae08 100644 --- a/app/services/mongodb/MongoDBFileService.scala +++ b/app/services/mongodb/MongoDBFileService.scala @@ -815,24 +815,27 @@ class MongoDBFileService @Inject() ( def removeFile(id: UUID, host: String, apiKey: Option[String], user: Option[User]){ get(id) match{ case Some(file) => { - if(!file.isIntermediate){ - val fileDatasets = datasets.findByFileIdDirectlyContain(file.id) - for(fileDataset <- fileDatasets){ - val datasetSpaces : List[UUID] = fileDataset.spaces - var canDeleteFile = false - if (file.author.id == user.get.id ) { - canDeleteFile = true - } - for (currentSpace <- datasetSpaces) { - userService.getUserRoleInSpace(user.get.id, currentSpace) match { + var canDeleteFile = false + if (user.get.id == file.author.id) { + canDeleteFile = true + } else { + var fileDatasets = datasets.findByFileIdDirectlyContain(file.id) + for (fileDataset <- fileDatasets) { + var datasetSpaces = fileDataset.spaces + for (datasetSpace <- datasetSpaces) { + userService.getUserRoleInSpace(user.get.id, datasetSpace) match { case Some(role) => { if (role.permissions.contains(Permission.DeleteFile)){ canDeleteFile = true } } - case None => } } + } + } + if(!file.isIntermediate){ + val fileDatasets = datasets.findByFileIdDirectlyContain(file.id) + for(fileDataset <- fileDatasets){ datasets.removeFile(fileDataset.id, id) if(!file.xmlMetadata.isEmpty){ datasets.index(fileDataset.id) From 116c296ad3056de45927451743f10dc212551e34 Mon Sep 17 00:00:00 2001 From: tcnichol Date: Sat, 6 Mar 2021 15:09:53 -0600 Subject: [PATCH 05/11] checks if user can delete file this might not be the best approach --- app/services/mongodb/MongoDBFileService.scala | 143 +++++++++--------- 1 file changed, 72 insertions(+), 71 deletions(-) diff --git a/app/services/mongodb/MongoDBFileService.scala b/app/services/mongodb/MongoDBFileService.scala index e3ba4ae08..8ef2fcf96 100644 --- a/app/services/mongodb/MongoDBFileService.scala +++ b/app/services/mongodb/MongoDBFileService.scala @@ -4,23 +4,23 @@ import play.api.mvc.Request import services._ import models._ import com.mongodb.casbah.commons.{Imports, MongoDBObject} -import java.text.SimpleDateFormat +import java.text.SimpleDateFormat import _root_.util.{License, Parsers, SearchUtils} import scala.collection.mutable.ListBuffer import Transformation.LidoToCidocConvertion + import java.util.{ArrayList, Calendar} import java.io._ - import org.apache.commons.io.FileUtils import org.json.JSONObject import play.api.libs.json.{JsValue, Json} import com.mongodb.util.JSON + import java.nio.file.{FileSystems, Files} import java.nio.file.attribute.BasicFileAttributes import java.time.LocalDateTime - import collection.JavaConverters._ import scala.collection.JavaConversions._ import javax.inject.{Inject, Singleton} @@ -31,8 +31,8 @@ import scala.util.parsing.json.JSONArray import play.api.libs.json.JsArray import models.File import play.api.libs.json.JsObject -import java.util.Date +import java.util.Date import com.novus.salat.dao.{ModelCompanion, SalatDAO} import MongoContext.context import play.api.Play._ @@ -40,6 +40,7 @@ import com.mongodb.casbah.Imports._ import models.FileStatus.FileStatus import org.bson.types.ObjectId import api.Permission +import api.Permission.users /** @@ -815,85 +816,73 @@ class MongoDBFileService @Inject() ( def removeFile(id: UUID, host: String, apiKey: Option[String], user: Option[User]){ get(id) match{ case Some(file) => { - var canDeleteFile = false - if (user.get.id == file.author.id) { - canDeleteFile = true - } else { - var fileDatasets = datasets.findByFileIdDirectlyContain(file.id) - for (fileDataset <- fileDatasets) { - var datasetSpaces = fileDataset.spaces - for (datasetSpace <- datasetSpaces) { - userService.getUserRoleInSpace(user.get.id, datasetSpace) match { - case Some(role) => { - if (role.permissions.contains(Permission.DeleteFile)){ - canDeleteFile = true - } + val canUserDeleteFile = canDeleteFile(file, user) + + if (canUserDeleteFile) { + if(!file.isIntermediate){ + val fileDatasets = datasets.findByFileIdDirectlyContain(file.id) + for(fileDataset <- fileDatasets){ + datasets.removeFile(fileDataset.id, id) + if(!file.xmlMetadata.isEmpty){ + datasets.index(fileDataset.id) + } + + if(!file.thumbnail_id.isEmpty && !fileDataset.thumbnail_id.isEmpty){ + if(file.thumbnail_id.get.equals(fileDataset.thumbnail_id.get)){ + datasets.newThumbnail(fileDataset.id) + collections.get(fileDataset.collections).found.foreach(collection => { + if(!collection.thumbnail_id.isEmpty){ + if(collection.thumbnail_id.get.equals(fileDataset.thumbnail_id.get)){ + collections.createThumbnail(collection.id) + } + } + }) } } - } - } - } - if(!file.isIntermediate){ - val fileDatasets = datasets.findByFileIdDirectlyContain(file.id) - for(fileDataset <- fileDatasets){ - datasets.removeFile(fileDataset.id, id) - if(!file.xmlMetadata.isEmpty){ - datasets.index(fileDataset.id) + } - if(!file.thumbnail_id.isEmpty && !fileDataset.thumbnail_id.isEmpty){ - if(file.thumbnail_id.get.equals(fileDataset.thumbnail_id.get)){ - datasets.newThumbnail(fileDataset.id) - collections.get(fileDataset.collections).found.foreach(collection => { - if(!collection.thumbnail_id.isEmpty){ - if(collection.thumbnail_id.get.equals(fileDataset.thumbnail_id.get)){ - collections.createThumbnail(collection.id) - } - } - }) - } + val fileFolders = folders.findByFileId(file.id) + for(fileFolder <- fileFolders) { + folders.removeFile(fileFolder.id, file.id) + } + for(section <- sections.findByFileId(file.id)){ + sections.removeSection(section) + } + for(comment <- comments.findCommentsByFileId(id)){ + comments.removeComment(comment) + } + for(texture <- threeD.findTexturesByFileId(file.id)){ + ThreeDTextureDAO.removeById(new ObjectId(texture.id.stringify)) + } + for (follower <- file.followers) { + userService.unfollowFile(follower, id) } - } - val fileFolders = folders.findByFileId(file.id) - for(fileFolder <- fileFolders) { - folders.removeFile(fileFolder.id, file.id) - } - for(section <- sections.findByFileId(file.id)){ - sections.removeSection(section) - } - for(comment <- comments.findCommentsByFileId(id)){ - comments.removeComment(comment) - } - for(texture <- threeD.findTexturesByFileId(file.id)){ - ThreeDTextureDAO.removeById(new ObjectId(texture.id.stringify)) - } - for (follower <- file.followers) { - userService.unfollowFile(follower, id) + // delete the actual file + if(isLastPointingToLoader(file.loader, file.loader_id)) { + for(preview <- previews.findByFileId(file.id)){ + previews.removePreview(preview) + } + if(!file.thumbnail_id.isEmpty) + thumbnails.remove(UUID(file.thumbnail_id.get)) + ByteStorageService.delete(file.loader, file.loader_id, FileDAO.COLLECTION) } - } - // delete the actual file - if(isLastPointingToLoader(file.loader, file.loader_id)) { - for(preview <- previews.findByFileId(file.id)){ - previews.removePreview(preview) + import UUIDConversions._ + FileDAO.removeById(file.id) + appConfig.incrementCount('files, -1) + appConfig.incrementCount('bytes, -file.length) + current.plugin[ElasticsearchPlugin].foreach { + _.delete(id.stringify) } - if(!file.thumbnail_id.isEmpty) - thumbnails.remove(UUID(file.thumbnail_id.get)) - ByteStorageService.delete(file.loader, file.loader_id, FileDAO.COLLECTION) - } - import UUIDConversions._ - FileDAO.removeById(file.id) - appConfig.incrementCount('files, -1) - appConfig.incrementCount('bytes, -file.length) - current.plugin[ElasticsearchPlugin].foreach { - _.delete(id.stringify) + // finally remove metadata - if done before file is deleted, document metadataCounts won't match + metadatas.removeMetadataByAttachTo(ResourceRef(ResourceRef.file, id), host, apiKey, user) + } else { + Logger.info("User cannot delete file : " + file.id.stringify) } - - // finally remove metadata - if done before file is deleted, document metadataCounts won't match - metadatas.removeMetadataByAttachTo(ResourceRef(ResourceRef.file, id), host, apiKey, user) } case None => Logger.debug("File not found") } @@ -1247,6 +1236,18 @@ class MongoDBFileService @Inject() ( until.foreach(t => query = query ++ ("uploadDate" $lte Parsers.fromISO8601(t))) FileDAO.find(query) } + + private def canDeleteFile(file: File, user: Option[User]): Boolean = { + datasets.findByFileIdDirectlyContain(file.id).foreach { dataset => + dataset.spaces.map { + spaceId => for(role <- users.getUserRoleInSpace(user.get.id, spaceId)) { + if(role.permissions.contains(Permission.DeleteFile)) + return true + } + } + } + false + } } object FileDAO extends ModelCompanion[File, ObjectId] { From 869e8762504f3bd5098633afb5cceb1d155db833 Mon Sep 17 00:00:00 2001 From: tcnichol Date: Sun, 7 Mar 2021 15:26:38 -0600 Subject: [PATCH 06/11] checks permission inside method only deletes files with proper permission --- app/services/mongodb/MongoDBFileService.scala | 17 +++-------------- 1 file changed, 3 insertions(+), 14 deletions(-) diff --git a/app/services/mongodb/MongoDBFileService.scala b/app/services/mongodb/MongoDBFileService.scala index 8ef2fcf96..52db807a9 100644 --- a/app/services/mongodb/MongoDBFileService.scala +++ b/app/services/mongodb/MongoDBFileService.scala @@ -816,9 +816,10 @@ class MongoDBFileService @Inject() ( def removeFile(id: UUID, host: String, apiKey: Option[String], user: Option[User]){ get(id) match{ case Some(file) => { - val canUserDeleteFile = canDeleteFile(file, user) + val currentResourceRef = ResourceRef(ResourceRef.file, file.id) + val hasPermission = Permission.checkPermission(user.get,Permission.DeleteFile, currentResourceRef) - if (canUserDeleteFile) { + if (hasPermission) { if(!file.isIntermediate){ val fileDatasets = datasets.findByFileIdDirectlyContain(file.id) for(fileDataset <- fileDatasets){ @@ -1236,18 +1237,6 @@ class MongoDBFileService @Inject() ( until.foreach(t => query = query ++ ("uploadDate" $lte Parsers.fromISO8601(t))) FileDAO.find(query) } - - private def canDeleteFile(file: File, user: Option[User]): Boolean = { - datasets.findByFileIdDirectlyContain(file.id).foreach { dataset => - dataset.spaces.map { - spaceId => for(role <- users.getUserRoleInSpace(user.get.id, spaceId)) { - if(role.permissions.contains(Permission.DeleteFile)) - return true - } - } - } - false - } } object FileDAO extends ModelCompanion[File, ObjectId] { From a250cb41f32872db7a4704da387f6e9c321cf142 Mon Sep 17 00:00:00 2001 From: tcnichol Date: Thu, 25 Mar 2021 16:09:48 -0500 Subject: [PATCH 07/11] using the new checkPermissions method for list of resource ref instead of changing code in the fileservice method --- app/api/Files.scala | 15 +++++++++++---- 1 file changed, 11 insertions(+), 4 deletions(-) diff --git a/app/api/Files.scala b/app/api/Files.scala index 2b5fa4775..35f1b6849 100644 --- a/app/api/Files.scala +++ b/app/api/Files.scala @@ -21,6 +21,7 @@ import java.net.URL import java.util.Date import javax.inject.Inject import scala.annotation.tailrec +import scala.collection.mutable.ListBuffer /** * Json API for files. @@ -1616,13 +1617,19 @@ class Files @Inject()( if (fileIds.isEmpty){ BadRequest("No file ids supplied") } else { - for (fileId <- fileIds){ - if (UUID.isValid(fileId)){ - files.removeFile(UUID(fileId),Utils.baseUrl(request), request.apiKey, request.user) + var resourceRefList: ListBuffer[ResourceRef] = ListBuffer.empty[ResourceRef] + for (fileId <- fileIds) { + if (UUID.isValid(fileId)) { + val current_resource_ref = ResourceRef(ResourceRef.file, UUID(fileId)) + resourceRefList += current_resource_ref } } + val filesIdsCanDelete = Permission.checkPermissions(request.user, Permission.DeleteFile, resourceRefList.toList).approved.map(_.id) + for (id <- filesIdsCanDelete) { + files.removeFile(id,Utils.baseUrl(request), request.apiKey, request.user) + } + Ok(toJson(Map("status" -> "success"))) } - Ok(toJson("Done")) } case None => { BadRequest("No user supplied") From d6beba41383a7121b48947b2d373a08b05134bc6 Mon Sep 17 00:00:00 2001 From: tcnichol Date: Wed, 31 Mar 2021 13:19:00 -0500 Subject: [PATCH 08/11] removing redundant permissions check --- app/services/mongodb/MongoDBFileService.scala | 4 ---- 1 file changed, 4 deletions(-) diff --git a/app/services/mongodb/MongoDBFileService.scala b/app/services/mongodb/MongoDBFileService.scala index c68929e04..3a77cd286 100644 --- a/app/services/mongodb/MongoDBFileService.scala +++ b/app/services/mongodb/MongoDBFileService.scala @@ -814,10 +814,6 @@ class MongoDBFileService @Inject() ( def removeFile(id: UUID, host: String, apiKey: Option[String], user: Option[User]){ get(id) match{ case Some(file) => { - val currentResourceRef = ResourceRef(ResourceRef.file, file.id) - val hasPermission = Permission.checkPermission(user.get,Permission.DeleteFile, currentResourceRef) - - if (hasPermission) { if(!file.isIntermediate){ val fileDatasets = datasets.findByFileIdDirectlyContain(file.id) for(fileDataset <- fileDatasets){ From 6198cf4858521e1db24493470c87925d4e1b14f9 Mon Sep 17 00:00:00 2001 From: tcnichol Date: Wed, 31 Mar 2021 13:22:35 -0500 Subject: [PATCH 09/11] matching develop --- app/services/mongodb/MongoDBFileService.scala | 57 ++++++++----------- 1 file changed, 23 insertions(+), 34 deletions(-) diff --git a/app/services/mongodb/MongoDBFileService.scala b/app/services/mongodb/MongoDBFileService.scala index 3a77cd286..a526c769b 100644 --- a/app/services/mongodb/MongoDBFileService.scala +++ b/app/services/mongodb/MongoDBFileService.scala @@ -11,7 +11,7 @@ import _root_.util.{License, Parsers, SearchUtils} import scala.collection.mutable.ListBuffer import Transformation.LidoToCidocConvertion -import java.util.{ArrayList, Calendar} +import java.util.{ArrayList, Calendar, Date} import java.io._ import org.apache.commons.io.FileUtils import org.json.JSONObject @@ -20,7 +20,7 @@ import com.mongodb.util.JSON import java.nio.file.{FileSystems, Files} import java.nio.file.attribute.BasicFileAttributes -import java.time.LocalDateTime +import java.time.Instant import collection.JavaConverters._ import scala.collection.JavaConversions._ import javax.inject.{Inject, Singleton} @@ -31,16 +31,15 @@ import scala.util.parsing.json.JSONArray import play.api.libs.json.JsArray import models.File import play.api.libs.json.JsObject - -import java.util.Date import com.novus.salat.dao.{ModelCompanion, SalatDAO} import MongoContext.context import play.api.Play._ import com.mongodb.casbah.Imports._ import models.FileStatus.FileStatus import org.bson.types.ObjectId -import api.Permission -import api.Permission.users + +import java.time.temporal.ChronoUnit +import scala.concurrent.duration.FiniteDuration /** @@ -203,48 +202,41 @@ class MongoDBFileService @Inject() ( * This may be expanded to support per-space configuration in the future. * * Reads the following parameters from Clowder configuration: - * - archiveAutoAfterDaysInactive - timeout after which files are considered + * - archiveAutoAfterInactiveCount - timeout after which files are considered * to be candidates for archival (see below) - * - archiveMinimumStorageSize - files below this size (in Bytes) should not be archived + * - archiveAutoAfterInactiveUnits - time unit that should be used for the timeout (see below) + * - archiveAutoAboveMinimumStorageSize - files below this size (in Bytes) should not be archived * - clowder.rabbitmq.clowderurl - the Clowder hostname to pass to the archival extractor * - commKey - the admin key to pass to the archival extractor * * Archival candidates are currently defined as follows: - * - file must be over `archiveMinimumStorageSize` Bytes in size - * - file must be over `archiveAutoAfterDaysInactive` days old + * - file's size must be greater than `archiveAutoAboveMinimumStorageSize` Bytes + * - file's age must be greater than `archiveAutoAfterInactiveCount` * `archiveAutoAfterInactiveUnits` + * (e.g. 10 days old) * - AND one of the following must be true: * - file has never been downloaded (0 downloads) * OR - * - file has not been downloaded in the past `archiveAutoAfterDaysInactive` days + * - file has not been downloaded in the past `archiveAutoAfterInactiveCount` `archiveAutoAfterInactiveUnits` * * */ def autoArchiveCandidateFiles() = { - val timeout = configuration(play.api.Play.current).getInt("archiveAutoAfterDaysInactive") + val timeout: Option[Long] = configuration(play.api.Play.current).getLong("archiveAutoAfterInactiveCount") timeout match { case None => Logger.info("No archival auto inactivity timeout set - skipping auto archival loop.") - case Some(days) => { - if (days == 0) { + case Some(inactiveTimeout) => { + if (inactiveTimeout == 0) { Logger.info("Archival auto inactivity timeout set to 0 - skipping auto archival loop.") } else { - // DEBUG ONLY: query for files that were uploaded within the past hour - val archiveDebug = configuration(play.api.Play.current).getBoolean("archiveDebug").getOrElse(false) - val oneHourAgo = LocalDateTime.now.minusHours(1).toString + "-00:00" - - // Query for files that haven't been accessed for at least this many days - val daysAgo = LocalDateTime.now.minusDays(days).toString + "-00:00" - val notDownloadedWithinTimeout = if (archiveDebug) { - ("stats.last_downloaded" $gte Parsers.fromISO8601(oneHourAgo)) ++ ("status" $eq FileStatus.PROCESSED.toString) - } else { - ("stats.last_downloaded" $lt Parsers.fromISO8601(daysAgo)) ++ ("status" $eq FileStatus.PROCESSED.toString) - } + val unit = configuration(play.api.Play.current).getString("archiveAutoAfterInactiveUnits").getOrElse("days") + val timeoutAgo = FiniteDuration(inactiveTimeout, unit) + + // Query for files that haven't been accessed for at least this many units + val since = Instant.now().minus(timeoutAgo.length.toLong, ChronoUnit.valueOf(timeoutAgo.unit.toString)).toString + "-00:00" + val notDownloadedWithinTimeout = ("stats.last_downloaded" $lt Parsers.fromISO8601(since)) ++ ("status" $eq FileStatus.PROCESSED.toString) // Include files that have never been downloaded, but make sure they are old enough - val neverDownloaded = if (archiveDebug) { - ("stats.downloads" $eq 0) ++ ("uploadDate" $gte Parsers.fromISO8601(oneHourAgo)) ++ ("status" $eq FileStatus.PROCESSED.toString) - } else { - ("stats.downloads" $eq 0) ++ ("uploadDate" $lt Parsers.fromISO8601(daysAgo)) ++ ("status" $eq FileStatus.PROCESSED.toString) - } + val neverDownloaded = ("stats.downloads" $eq 0) ++ ("uploadDate" $lt Parsers.fromISO8601(since)) ++ ("status" $eq FileStatus.PROCESSED.toString) // TODO: How to get host / apiKey / admin internally without a request? val host = configuration(play.api.Play.current).getString("clowder.rabbitmq.clowderurl").getOrElse("http://localhost:9000") @@ -259,7 +251,7 @@ class MongoDBFileService @Inject() ( Logger.info("Archival candidates found: " + matchingFiles.length) // Exclude candidates that do not exceed our minimum file size threshold - val minSize = configuration(play.api.Play.current).getLong("archiveMinimumStorageSize").getOrElse(1000000L) + val minSize = configuration(play.api.Play.current).getLong("archiveAutoAboveMinimumStorageSize").getOrElse(1000000L) // Loop all candidate files and submit each one for archival for (file <- matchingFiles) { @@ -875,9 +867,6 @@ class MongoDBFileService @Inject() ( // finally remove metadata - if done before file is deleted, document metadataCounts won't match metadatas.removeMetadataByAttachTo(ResourceRef(ResourceRef.file, id), host, apiKey, user) - } else { - Logger.info("User cannot delete file : " + file.id.stringify) - } } case None => Logger.debug("File not found") } From e7eed038d9fa619e0d3e1e5bb951c425ad733c0a Mon Sep 17 00:00:00 2001 From: tcnichol Date: Wed, 31 Mar 2021 13:34:26 -0500 Subject: [PATCH 10/11] adding swagger entry --- public/swagger.yml | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/public/swagger.yml b/public/swagger.yml index cb3004723..c621dae79 100644 --- a/public/swagger.yml +++ b/public/swagger.yml @@ -189,6 +189,24 @@ paths: '401': $ref: '#/components/responses/Disabled' + /files/bulkRemove: + post: + tags: + - files + summary: Deletes files + descriptions: Deletes a list of files by fileIds + responses: + '200': + description: Returns Status Success + content: + application/json: + schema: + type: array + items: + $ref: '#components/parameters/fileIds' + '401': + $ref: '#components/responses/Disabled' + /files/{id}: parameters: - $ref: '#/components/parameters/fileId' From 8ce9c3bad0e8a21654d6af589102c820014d3374 Mon Sep 17 00:00:00 2001 From: Luigi Marini Date: Tue, 27 Apr 2021 15:25:31 -0500 Subject: [PATCH 11/11] Updated CHANGELOG.md. --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 0bd9eb302..4a00a7ba7 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -13,6 +13,7 @@ and this project adheres to [Semantic Versioning](http://semver.org/). - Log an event each time that a user archives or unarchives a file. - Endpoint `/api/datasets/createfrombag` to ingest datasets in BagIt format. Includes basic dataset metadata, files, folders and technical metadata. Downloading datasets now includes extra Datacite and Clowder metadata. +- Endpoint /api/files/bulkRemove to delete multiple files in one call. [#12](https://github.com/clowder-framework/clowder/issues/12) ## 1.16.0 - 2021-03-31