diff --git a/.github/ISSUE_TEMPLATE/bug_report.md b/.github/ISSUE_TEMPLATE/bug_report.md index dd84ea782..f3d5c415e 100644 --- a/.github/ISSUE_TEMPLATE/bug_report.md +++ b/.github/ISSUE_TEMPLATE/bug_report.md @@ -2,7 +2,7 @@ name: Bug report about: Create a report to help us improve title: '' -labels: '' +labels: bug assignees: '' --- diff --git a/.github/ISSUE_TEMPLATE/feature_request.md b/.github/ISSUE_TEMPLATE/feature_request.md index bbcbbe7d6..11fc491ef 100644 --- a/.github/ISSUE_TEMPLATE/feature_request.md +++ b/.github/ISSUE_TEMPLATE/feature_request.md @@ -2,7 +2,7 @@ name: Feature request about: Suggest an idea for this project title: '' -labels: '' +labels: enhancement assignees: '' --- diff --git a/CHANGELOG.md b/CHANGELOG.md index 686046231..b0def3e65 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,23 @@ All notable changes to this project will be documented in this file. The format is based on [Keep a Changelog](http://keepachangelog.com/) and this project adheres to [Semantic Versioning](http://semver.org/). +## 1.19.0 - 2021-10-05 +**_Important:_** This update requires a MongoDB update schema due to the new ability of showing summary statistics at the +space level. Make sure to start the application with -DMONGOUPDATE=1. + +### Fixed +- Adding dataset to space. Space list on dataset page would be empty - fixed error when no spaces would load. [#274](https://github.com/clowder-framework/clowder/issues/274) +- Typos "success" when returning status from API and "occurred" when logging to console. +- If a dataset had multiple folders the layout would be wrong. +- Collections created using api route are now indexed upon creation. [#257](https://github.com/clowder-framework/clowder/issues/257) + +### Added +- Mime type for geojson +- "when" parameter in a few GET API endpoints to enable pagination [#266](https://github.com/clowder-framework/clowder/issues/266) +- Show space statistics (bytes, users. etc) [#119](https://github.com/clowder-framework/clowder/issues/119) +- "id" in GET metadata.jsonld endpoints [#278](https://github.com/clowder-framework/clowder/issues/278) +- 'POST /api/files/uploadToDataset' now allows folder_id for uploading file to folder. [#232](https://github.com/clowder-framework/clowder/issues/232) + ## 1.18.1 - 2021-08-16 This release fixes a critical issue where invalid zip files could result in the files not being uploaded correctly. To check to see if you are affected, please use the following query: @@ -55,7 +72,6 @@ If any files are returned, you should check to see if these files affected and a ### Changed - Updated Sphinx dependencies due to security and changes in required packages. - - Updated the three.js libraries for the FBX previewer ## 1.16.0 - 2021-03-31 diff --git a/CONTRIBUTORS.md b/CONTRIBUTORS.md index 372a6c941..dffa18f38 100644 --- a/CONTRIBUTORS.md +++ b/CONTRIBUTORS.md @@ -34,6 +34,7 @@ Following is a list of contributors in alphabetical order: - Sandeep Puthanveetil Satheesan - Smruti Padhy - Theerasit Issaranon +- Tim Yardley - Todd Nicholson - Varun Kethineedi - Ward Poelmans diff --git a/app/api/Collections.scala b/app/api/Collections.scala index 09d399014..1960b373b 100644 --- a/app/api/Collections.scala +++ b/app/api/Collections.scala @@ -1,32 +1,28 @@ package api -import java.io.{ByteArrayInputStream, InputStream, ByteArrayOutputStream} -import java.security.{DigestInputStream, MessageDigest} -import java.text.SimpleDateFormat -import java.util.zip.{ZipEntry, ZipOutputStream, Deflater} - import Iterators.RootCollectionIterator -import _root_.util.JSONLD +import util.SearchUtils import api.Permission.Permission -import org.apache.commons.codec.binary.Hex +import controllers.Utils +import models._ import play.api.Logger import play.api.Play.current -import models._ +import play.api.libs.concurrent.Execution.Implicits._ import play.api.libs.iteratee.Enumerator -import services._ -import play.api.libs.json._ -import play.api.libs.json.{JsObject, JsValue} import play.api.libs.json.Json.toJson -import javax.inject.{ Singleton, Inject} -import scala.collection.mutable.ListBuffer -import scala.concurrent.{Future, ExecutionContext} -import play.api.libs.concurrent.Execution.Implicits._ -import scala.util.parsing.json.JSONArray -import scala.util.{Try, Success, Failure} -import java.util.{Calendar, Date} -import controllers.Utils +import play.api.libs.json.{JsObject, JsValue, _} +import services._ + +import java.io.ByteArrayOutputStream +import java.security.MessageDigest +import java.util.zip.{Deflater, ZipOutputStream} +import java.util.{Calendar, Date} +import javax.inject.{Inject, Singleton} import scala.collection.immutable.List +import scala.collection.mutable.ListBuffer +import scala.concurrent.{ExecutionContext, Future} +import scala.util.{Failure, Success, Try} /** @@ -72,6 +68,13 @@ class Collections @Inject() (datasets: DatasetService, collections.addToRootSpaces(c.id, s.id) events.addSourceEvent(request.user, c.id, c.name, s.id, s.name, EventType.ADD_COLLECTION_SPACE.toString) }) + // index collection + current.plugin[ElasticsearchPlugin].foreach{ + _.index(SearchUtils.getElasticsearchObject(c)) + } + //Add to Events Table + val option_user = userService.findByIdentity(identity) + events.addObjectEvent(option_user, c.id, c.name, EventType.CREATE_COLLECTION.toString) Ok(toJson(Map("id" -> id))) } case None => Ok(toJson(Map("status" -> "error"))) @@ -237,15 +240,15 @@ class Collections @Inject() (datasets: DatasetService, } } - def list(title: Option[String], date: Option[String], limit: Int, exact: Boolean) = PrivateServerAction { implicit request => - Ok(toJson(listCollections(title, date, limit, Set[Permission](Permission.ViewCollection), false, request.user, request.user.fold(false)(_.superAdminMode), exact))) + def list(when: Option[String], title: Option[String], date: Option[String], limit: Int, exact: Boolean) = PrivateServerAction { implicit request => + Ok(toJson(listCollections(when, title, date, limit, Set[Permission](Permission.ViewCollection), false, request.user, request.user.fold(false)(_.superAdminMode), exact))) } - def listCanEdit(title: Option[String], date: Option[String], limit: Int, exact: Boolean) = PrivateServerAction { implicit request => - Ok(toJson(listCollections(title, date, limit, Set[Permission](Permission.AddResourceToCollection, Permission.EditCollection), false, request.user, request.user.fold(false)(_.superAdminMode), exact))) + def listCanEdit(when: Option[String], title: Option[String], date: Option[String], limit: Int, exact: Boolean) = PrivateServerAction { implicit request => + Ok(toJson(listCollections(when, title, date, limit, Set[Permission](Permission.AddResourceToCollection, Permission.EditCollection), false, request.user, request.user.fold(false)(_.superAdminMode), exact))) } - def addDatasetToCollectionOptions(datasetId: UUID, title: Option[String], date: Option[String], limit: Int, exact: Boolean) = PrivateServerAction { implicit request => + def addDatasetToCollectionOptions(when: Option[String], datasetId: UUID, title: Option[String], date: Option[String], limit: Int, exact: Boolean) = PrivateServerAction { implicit request => implicit val user = request.user var listAll = false var collectionList: List[Collection] = List.empty @@ -265,7 +268,7 @@ class Collections @Inject() (datasets: DatasetService, } } if(listAll) { - collectionList = listCollections(title, date, limit, Set[Permission](Permission.AddResourceToCollection, Permission.EditCollection), false, request.user, request.user.fold(false)(_.superAdminMode), exact) + collectionList = listCollections(when, title, date, limit, Set[Permission](Permission.AddResourceToCollection, Permission.EditCollection), false, request.user, request.user.fold(false)(_.superAdminMode), exact) } Ok(toJson(collectionList)) } @@ -274,10 +277,10 @@ class Collections @Inject() (datasets: DatasetService, collections.get(current_collections.map(_.child_collection_ids).flatten).found } - def listPossibleParents(currentCollectionId : String, title: Option[String], date: Option[String], limit: Int, exact: Boolean) = PrivateServerAction { implicit request => + def listPossibleParents(when: Option[String], currentCollectionId : String, title: Option[String], date: Option[String], limit: Int, exact: Boolean) = PrivateServerAction { implicit request => val selfAndAncestors = collections.getSelfAndAncestors(UUID(currentCollectionId)) val descendants = collections.getAllDescendants(UUID(currentCollectionId)).toList - val allCollections = listCollections(title, date, limit, Set[Permission](Permission.AddResourceToCollection, Permission.EditCollection), false, + val allCollections = listCollections(when, title, date, limit, Set[Permission](Permission.AddResourceToCollection, Permission.EditCollection), false, request.user, request.user.fold(false)(_.superAdminMode), exact) val possibleNewParents = allCollections.filter(c => if(play.api.Play.current.plugin[services.SpaceSharingPlugin].isDefined) { @@ -303,29 +306,55 @@ class Collections @Inject() (datasets: DatasetService, * Returns list of collections based on parameters and permissions. * TODO this needs to be cleaned up when do permissions for adding to a resource */ - private def listCollections(title: Option[String], date: Option[String], limit: Int, permission: Set[Permission], mine: Boolean, user: Option[User], superAdmin: Boolean, exact: Boolean) : List[Collection] = { + private def listCollections(when: Option[String], title: Option[String], date: Option[String], limit: Int, permission: Set[Permission], mine: Boolean, user: Option[User], superAdmin: Boolean, exact: Boolean) : List[Collection] = { if (mine && user.isEmpty) return List.empty[Collection] - (title, date) match { - case (Some(t), Some(d)) => { + (when, title, date) match { + case (Some(w), Some(t), Some(d)) => { + if (mine) + collections.listUser(d, nextPage=(w=="a"), limit, t, user, superAdmin, user.get, exact) + else + collections.listAccess(d, nextPage=(w=="a"), limit, t, permission, user, superAdmin, true,false, exact) + } + case (Some(w), Some(t), None) => { + if (mine) + collections.listUser(limit, t, user, superAdmin, user.get, exact) + else + collections.listAccess(limit, t, permission, user, superAdmin, true,false, exact) + } + case (Some(w), None, Some(d)) => { + if (mine) + collections.listUser(d, nextPage=(w=="a"), limit, user, superAdmin, user.get) + else + collections.listAccess(d, nextPage=(w=="a"), limit, permission, user, superAdmin, true,false) + } + case (Some(w), None, None) => { + if (mine) + collections.listUser(limit, user, superAdmin, user.get) + else + collections.listAccess(limit, permission, user, superAdmin, true,false) + } + + // default when to be "after" if not present in parameters. i.e. nextPage=true + case (None, Some(t), Some(d)) => { if (mine) collections.listUser(d, true, limit, t, user, superAdmin, user.get, exact) else collections.listAccess(d, true, limit, t, permission, user, superAdmin, true,false, exact) } - case (Some(t), None) => { + case (None, Some(t), None) => { if (mine) collections.listUser(limit, t, user, superAdmin, user.get, exact) else collections.listAccess(limit, t, permission, user, superAdmin, true,false, exact) } - case (None, Some(d)) => { + case (None, None, Some(d)) => { if (mine) collections.listUser(d, true, limit, user, superAdmin, user.get) else collections.listAccess(d, true, limit, permission, user, superAdmin, true,false) } - case (None, None) => { + case (None, None, None) => { if (mine) collections.listUser(limit, user, superAdmin, user.get) else @@ -572,6 +601,14 @@ class Collections @Inject() (datasets: DatasetService, events.addSourceEvent(request.user, c.id, c.name, s.id, s.name, EventType.ADD_COLLECTION_SPACE.toString) } + // index collection + current.plugin[ElasticsearchPlugin].foreach{ + _.index(SearchUtils.getElasticsearchObject(c)) + } + //Add to Events Table + val option_user = userService.findByIdentity(identity) + events.addObjectEvent(option_user, c.id, c.name, EventType.CREATE_COLLECTION.toString) + //do stuff with parent here (request.body \"parentId").asOpt[String] match { case Some(parentId) => { diff --git a/app/api/Datasets.scala b/app/api/Datasets.scala index 710fb8f3e..07c7b9a55 100644 --- a/app/api/Datasets.scala +++ b/app/api/Datasets.scala @@ -65,19 +65,19 @@ class Datasets @Inject()( } } - def list(title: Option[String], date: Option[String], limit: Int, exact: Boolean) = PrivateServerAction { implicit request => - Ok(toJson(listDatasets(title, date, limit, Set[Permission](Permission.ViewDataset), request.user, request.user.fold(false)(_.superAdminMode), exact))) + def list(when: Option[String], title: Option[String], date: Option[String], limit: Int, exact: Boolean) = PrivateServerAction { implicit request => + Ok(toJson(listDatasets(when, title, date, limit, Set[Permission](Permission.ViewDataset), request.user, request.user.fold(false)(_.superAdminMode), exact))) } - def listCanEdit(title: Option[String], date: Option[String], limit: Int, exact: Boolean) = PrivateServerAction { implicit request => - Ok(toJson(listDatasets(title, date, limit, Set[Permission](Permission.AddResourceToDataset, Permission.EditDataset), request.user, request.user.fold(false)(_.superAdminMode), exact))) + def listCanEdit(when: Option[String], title: Option[String], date: Option[String], limit: Int, exact: Boolean) = PrivateServerAction { implicit request => + Ok(toJson(listDatasets(when, title, date, limit, Set[Permission](Permission.AddResourceToDataset, Permission.EditDataset), request.user, request.user.fold(false)(_.superAdminMode), exact))) } def listMoveFileToDataset(file_id: UUID, title: Option[String], limit: Int, exact: Boolean) = PrivateServerAction { implicit request => if (play.Play.application().configuration().getBoolean("datasetFileWithinSpace")) { Ok(toJson(listDatasetsInSpace(file_id, title, limit, Set[Permission](Permission.AddResourceToDataset, Permission.EditDataset), request.user, request.user.fold(false)(_.superAdminMode), exact))) } else { - Ok(toJson(listDatasets(title, None, limit, Set[Permission](Permission.AddResourceToDataset, Permission.EditDataset), request.user, request.user.fold(false)(_.superAdminMode), exact))) + Ok(toJson(listDatasets(None, title, None, limit, Set[Permission](Permission.AddResourceToDataset, Permission.EditDataset), request.user, request.user.fold(false)(_.superAdminMode), exact))) } } @@ -152,18 +152,31 @@ class Datasets @Inject()( /** * Returns list of datasets based on parameters and permissions. */ - private def listDatasets(title: Option[String], date: Option[String], limit: Int, permission: Set[Permission], user: Option[User], superAdmin: Boolean, exact: Boolean) : List[Dataset] = { - (title, date) match { - case (Some(t), Some(d)) => { + private def listDatasets(when: Option[String], title: Option[String], date: Option[String], limit: Int, permission: Set[Permission], user: Option[User], superAdmin: Boolean, exact: Boolean) : List[Dataset] = { + (when, title, date) match { + case (Some(w), Some(t), Some(d)) => { + datasets.listAccess(d, nextPage=(w=="a"), limit, t, permission, user, superAdmin, true,false, exact) + } + case (Some(w), Some(t), None) => { + datasets.listAccess(limit, t, permission, user, superAdmin, true,false, exact) + } + case (Some(w), None, Some(d)) => { + datasets.listAccess(d, nextPage=(w=="a"), limit, permission, user, superAdmin, true,false) + } + case (Some(w), None, None) => { + datasets.listAccess(limit, permission, user, superAdmin, true,false) + } + // default when to be "after" if not present in parameters. i.e. nextPage=true + case (None, Some(t), Some(d)) => { datasets.listAccess(d, true, limit, t, permission, user, superAdmin, true,false, exact) } - case (Some(t), None) => { + case (None, Some(t), None) => { datasets.listAccess(limit, t, permission, user, superAdmin, true,false, exact) } - case (None, Some(d)) => { + case (None, None, Some(d)) => { datasets.listAccess(d, true, limit, permission, user, superAdmin, true,false) } - case (None, None) => { + case (None, None, None) => { datasets.listAccess(limit, permission, user, superAdmin, true,false) } } diff --git a/app/api/Files.scala b/app/api/Files.scala index b9bae4130..7f141c4ec 100644 --- a/app/api/Files.scala +++ b/app/api/Files.scala @@ -523,10 +523,16 @@ class Files @Inject()( /** * Upload a file to a specific dataset */ - def uploadToDataset(dataset_id: UUID, showPreviews: String = "DatasetLevel", originalZipFile: String = "", flagsFromPrevious: String = "", extract: Boolean = true) = PermissionAction(Permission.AddResourceToDataset, Some(ResourceRef(ResourceRef.dataset, dataset_id)))(parse.multipartFormData) { implicit request => + def uploadToDataset(dataset_id: UUID, showPreviews: String = "DatasetLevel", originalZipFile: String = "", flagsFromPrevious: String = "", extract: Boolean = true, folder_id: Option[String]) = PermissionAction(Permission.AddResourceToDataset, Some(ResourceRef(ResourceRef.dataset, dataset_id)))(parse.multipartFormData) { implicit request => datasets.get(dataset_id) match { case Some(dataset) => { - val uploadedFiles = FileUtils.uploadFilesMultipart(request, Some(dataset), showPreviews = showPreviews, originalZipFile = originalZipFile, flagsFromPrevious = flagsFromPrevious, runExtractors = extract, apiKey = request.apiKey) + var current_folder : Option[Folder] = None + if (folder_id != None) { + if (UUID.isValid(folder_id.get)){ + current_folder = folders.get(UUID(folder_id.get)) + } + } + val uploadedFiles = FileUtils.uploadFilesMultipart(request, Some(dataset), current_folder, showPreviews = showPreviews, originalZipFile = originalZipFile, flagsFromPrevious = flagsFromPrevious, runExtractors = extract, apiKey = request.apiKey) uploadedFiles.length match { case 0 => BadRequest("No files uploaded") case 1 => Ok(Json.obj("id" -> uploadedFiles.head.id)) @@ -1669,6 +1675,10 @@ class Files @Inject()( // notify rabbitmq datasets.findByFileIdAllContain(file.id).foreach { ds => routing.fileRemovedFromDataset(file, ds, Utils.baseUrl(request), request.apiKey) + val ds_spaces = ds.spaces + for (ds_s <- ds_spaces) { + spaces.decrementSpaceBytes(ds_s, file.length) + } } //this stmt has to be before files.removeFile diff --git a/app/api/Selected.scala b/app/api/Selected.scala index 3f5b3996e..0ef35436f 100644 --- a/app/api/Selected.scala +++ b/app/api/Selected.scala @@ -87,7 +87,7 @@ class Selected @Inject()(selections: SelectionService, selections.get(user.email.get).map(d => { selections.remove(d.id, user.email.get) }) - Ok(toJson(Map("sucess"->"true"))) + Ok(toJson(Map("success"->"true"))) } } } @@ -100,7 +100,7 @@ class Selected @Inject()(selections: SelectionService, datasets.removeDataset(d.id, Utils.baseUrl(request), request.apiKey, request.user) selections.remove(d.id, user.email.get) }) - Ok(toJson(Map("sucess"->"true"))) + Ok(toJson(Map("success"->"true"))) } } } @@ -189,7 +189,7 @@ class Selected @Inject()(selections: SelectionService, events.addObjectEvent(request.user, d.id, d.name, EventType.ADD_TAGS_DATASET.toString) datasets.index(d.id) }) - Ok(toJson(Map("sucess"->"true"))) + Ok(toJson(Map("success"->"true"))) } } } diff --git a/app/api/Sensors.scala b/app/api/Sensors.scala index 6a5b49eee..e6b5bad90 100644 --- a/app/api/Sensors.scala +++ b/app/api/Sensors.scala @@ -5,7 +5,7 @@ import play.api.Play.current import services.PostgresPlugin /** - * Metadata about sensors registered with the system. Datastreams can be associalted with sensors. + * Metadata about sensors registered with the system. Datastreams can be associated with sensors. */ object Sensors extends Controller with ApiController { diff --git a/app/api/Spaces.scala b/app/api/Spaces.scala index 21931c678..d82ba2c5e 100644 --- a/app/api/Spaces.scala +++ b/app/api/Spaces.scala @@ -41,7 +41,7 @@ class Spaces @Inject()(spaces: SpaceService, val userId = request.user.get.id val c = ProjectSpace(name = name, description = description, created = new Date(), creator = userId, homePage = List.empty, logoURL = None, bannerURL = None, collectionCount = 0, - datasetCount = 0, userCount = 0, metadata = List.empty) + datasetCount = 0, userCount = 0, spaceBytes = 0, metadata = List.empty) spaces.insert(c) match { case Some(id) => { appConfig.incrementCount('spaces, 1) @@ -86,45 +86,70 @@ class Spaces @Inject()(spaces: SpaceService, } } - def list(title: Option[String], date: Option[String], limit: Int) = UserAction(needActive=false) { implicit request => - Ok(toJson(listSpaces(title, date, limit, Set[Permission](Permission.ViewSpace), false, request.user, request.user.fold(false)(_.superAdminMode), true).map(spaceToJson))) + def list(when: Option[String], title: Option[String], date: Option[String], limit: Int) = UserAction(needActive=false) { implicit request => + Ok(toJson(listSpaces(when, title, date, limit, Set[Permission](Permission.ViewSpace), false, request.user, request.user.fold(false)(_.superAdminMode), true).map(spaceToJson))) } - def listCanEdit(title: Option[String], date: Option[String], limit: Int) = UserAction(needActive=true) { implicit request => - Ok(toJson(listSpaces(title, date, limit, Set[Permission](Permission.AddResourceToSpace, Permission.EditSpace), false, request.user, request.user.fold(false)(_.superAdminMode), true).map(spaceToJson))) + def listCanEdit(when: Option[String], title: Option[String], date: Option[String], limit: Int) = UserAction(needActive=true) { implicit request => + Ok(toJson(listSpaces(when, title, date, limit, Set[Permission](Permission.AddResourceToSpace, Permission.EditSpace), false, request.user, request.user.fold(false)(_.superAdminMode), true).map(spaceToJson))) } - def listCanEditNotAlreadyIn(collectionId : UUID, title: Option[String], date: Option[String], limit: Int) = UserAction(needActive=true ){ implicit request => - Ok(toJson(listSpaces(title, date, limit, Set[Permission](Permission.AddResourceToSpace, Permission.EditSpace), false, request.user, request.user.fold(false)(_.superAdminMode), true).map(spaceToJson))) + def listCanEditNotAlreadyIn(when: Option[String], collectionId : UUID, title: Option[String], date: Option[String], limit: Int) = UserAction(needActive=true ){ implicit request => + Ok(toJson(listSpaces(when, title, date, limit, Set[Permission](Permission.AddResourceToSpace, Permission.EditSpace), false, request.user, request.user.fold(false)(_.superAdminMode), true).map(spaceToJson))) } /** * Returns list of collections based on parameters and permissions. * TODO this needs to be cleaned up when do permissions for adding to a resource */ - private def listSpaces(title: Option[String], date: Option[String], limit: Int, permission: Set[Permission], mine: Boolean, user: Option[User], superAdmin: Boolean, showPublic: Boolean, onlyTrial: Boolean = false) : List[ProjectSpace] = { + private def listSpaces(when: Option[String], title: Option[String], date: Option[String], limit: Int, permission: Set[Permission], mine: Boolean, user: Option[User], superAdmin: Boolean, showPublic: Boolean, onlyTrial: Boolean = false) : List[ProjectSpace] = { if (mine && user.isEmpty) return List.empty[ProjectSpace] - (title, date) match { - case (Some(t), Some(d)) => { + (when, title, date) match { + case (Some(w), Some(t), Some(d)) => { + if (mine) + spaces.listUser(d, nextPage=(w=="a"), limit, t, user, superAdmin, user.get) + else + spaces.listAccess(d, nextPage=(w=="a"), limit, t, permission, user, superAdmin, showPublic, showOnlyShared = false) + } + case (Some(w), Some(t), None) => { + if (mine) + spaces.listUser(limit, t, user, superAdmin, user.get) + else + spaces.listAccess(limit, t, permission, user, superAdmin, showPublic, showOnlyShared = false) + } + case (Some(w), None, Some(d)) => { + if (mine) + spaces.listUser(d, nextPage=(w=="a"), limit, user, superAdmin, user.get) + else + spaces.listAccess(d, nextPage=(w=="a"), limit, permission, user, superAdmin, showPublic, onlyTrial, showOnlyShared = false) + } + case (Some(w), None, None) => { + if (mine) + spaces.listUser(limit, user, superAdmin, user.get) + else + spaces.listAccess(limit, permission, user, superAdmin, showPublic, onlyTrial, showOnlyShared = false) + } + // default when to be "after" if not present in parameters. i.e. nextPage=true + case (None, Some(t), Some(d)) => { if (mine) spaces.listUser(d, true, limit, t, user, superAdmin, user.get) else spaces.listAccess(d, true, limit, t, permission, user, superAdmin, showPublic, showOnlyShared = false) } - case (Some(t), None) => { + case (None, Some(t), None) => { if (mine) spaces.listUser(limit, t, user, superAdmin, user.get) else spaces.listAccess(limit, t, permission, user, superAdmin, showPublic, showOnlyShared = false) } - case (None, Some(d)) => { + case (None, None, Some(d)) => { if (mine) spaces.listUser(d, true, limit, user, superAdmin, user.get) else spaces.listAccess(d, true, limit, permission, user, superAdmin, showPublic, onlyTrial, showOnlyShared = false) } - case (None, None) => { + case (None, None, None) => { if (mine) spaces.listUser(limit, user, superAdmin, user.get) else diff --git a/app/assets/javascripts/select-bulk.js b/app/assets/javascripts/select-bulk.js index b2dd09111..30a4f68f9 100644 --- a/app/assets/javascripts/select-bulk.js +++ b/app/assets/javascripts/select-bulk.js @@ -20,7 +20,7 @@ $(function() { }); request.fail(function (jqXHR, textStatus, errorThrown){ - console.error("The following error occured: "+ textStatus, errorThrown); + console.error("The following error occurred: "+ textStatus, errorThrown); window.location = "../login"; // FIXME hardcoded }); diff --git a/app/assets/javascripts/select.js b/app/assets/javascripts/select.js index ba387aad7..137bc2be5 100644 --- a/app/assets/javascripts/select.js +++ b/app/assets/javascripts/select.js @@ -19,7 +19,7 @@ $(function() { }); request.fail(function (jqXHR, textStatus, errorThrown){ - console.error("The following error occured: "+ textStatus, errorThrown); + console.error("The following error occurred: "+ textStatus, errorThrown); window.location = "../login"; // FIXME hardcoded }); @@ -40,7 +40,7 @@ $(function() { }); request.fail(function (jqXHR, textStatus, errorThrown){ - console.error("The following error occured: "+ textStatus, errorThrown); + console.error("The following error occurred: "+ textStatus, errorThrown); window.location = "../login"; // FIXME hardcoded }); diff --git a/app/controllers/SecuredController.scala b/app/controllers/SecuredController.scala index be5035931..0f527f3db 100644 --- a/app/controllers/SecuredController.scala +++ b/app/controllers/SecuredController.scala @@ -2,14 +2,11 @@ package controllers import api.Permission.Permission import api.{Permission, UserRequest} -import models.{ClowderUser, RequestResource, ResourceRef, User, UserStatus} -import org.apache.commons.lang.StringEscapeUtils._ +import models.{ClowderUser, ResourceRef, User, UserStatus} import play.api.i18n.Messages import play.api.mvc._ import securesocial.core.{Authenticator, SecureSocial, UserService} import services._ -import securesocial.core.IdentityProvider -import securesocial.core.providers.utils.RoutesHelper import scala.concurrent.Future @@ -156,7 +153,7 @@ trait SecuredController extends Controller { val spaces: SpaceService = DI.injector.getInstance(classOf[SpaceService]) spaces.get(id) match { case None => Future.successful(BadRequest(views.html.notFound(spaceTitle + " does not exist.")(user))) - case Some(space) => Future.successful(Forbidden(views.html.spaces.space(space,List(),List(),List(),List(),"", Map(),List())(user))) + case Some(space) => Future.successful(Forbidden(views.html.spaces.space(space,List(),List(),List(),List(),"", Map(),List(),0,0)(user))) } } diff --git a/app/controllers/Spaces.scala b/app/controllers/Spaces.scala index f95d5b07b..ec0ae5d2b 100644 --- a/app/controllers/Spaces.scala +++ b/app/controllers/Spaces.scala @@ -1,31 +1,22 @@ package controllers -import java.net.URL -import java.util.{ Calendar, Date } -import javax.inject.Inject - import api.Permission import api.Permission._ import models._ -import play.api.{ Logger, Play } +import org.joda.time.DateTime import play.api.data.Forms._ -import play.api.data.{ Form, Forms } -import play.api.libs.json.JsValue -import play.api.libs.json.Json +import play.api.data.{Form, Forms} import play.api.i18n.Messages +import play.api.{Logger, Play} +import securesocial.core.providers.{Token, UsernamePasswordProvider} import services._ -import securesocial.core.providers.{ Token, UsernamePasswordProvider } -import org.joda.time.DateTime -import play.api.i18n.Messages -import play.api.libs.ws._ -import services.AppConfiguration -import util.{ Formatters, Mail, Publications } +import util.{Formatters, Mail, Publications} +import java.net.URL +import java.util.{Calendar, Date} +import javax.inject.Inject import scala.collection.immutable.List -import scala.collection.mutable.{ ArrayBuffer, ListBuffer } -import scala.concurrent.{ Future, Await } -import scala.concurrent.duration._ -import org.apache.commons.lang.StringEscapeUtils.escapeJava +import scala.collection.mutable.{ArrayBuffer, ListBuffer} /** * Spaces allow users to partition the data into realms only accessible to users with the right permissions. @@ -176,6 +167,8 @@ class Spaces @Inject() (spaces: SpaceService, users: UserService, events: EventS var creatorActual: User = null val collectionsInSpace = spaces.getCollectionsInSpace(Some(id.stringify), Some(size)) val datasetsInSpace = datasets.listSpace(size, id.toString(), user) + val spaceBytes : Long = s.spaceBytes + val spaceFiles : Integer = getFilesPerSpace(id, user.get) val publicDatasetsInSpace = datasets.listSpaceStatus(size, id.toString(), "publicAll", user) val usersInSpace = spaces.getUsersInSpace(id, None) var curationObjectsInSpace: List[CurationObject] = List() @@ -224,7 +217,7 @@ class Spaces @Inject() (spaces: SpaceService, users: UserService, events: EventS case None => List.empty } sinkService.logSpaceViewEvent(s, user) - Ok(views.html.spaces.space(Utils.decodeSpaceElements(s), collectionsInSpace, publicDatasetsInSpace, datasetsInSpace, rs, play.Play.application().configuration().getString("SEADservices.uri"), userRoleMap, userSelections)) + Ok(views.html.spaces.space(Utils.decodeSpaceElements(s), collectionsInSpace, publicDatasetsInSpace, datasetsInSpace, rs, play.Play.application().configuration().getString("SEADservices.uri"), userRoleMap, userSelections, spaceBytes, spaceFiles)) } case None => BadRequest(views.html.notFound(spaceTitle + " does not exist.")) } @@ -421,7 +414,7 @@ class Spaces @Inject() (spaces: SpaceService, users: UserService, events: EventS val newSpace = ProjectSpace(name = formData.name, description = formData.description, created = new Date, creator = userId, homePage = formData.homePage, logoURL = formData.logoURL, bannerURL = formData.bannerURL, - collectionCount = 0, datasetCount = 0, userCount = 0, metadata = List.empty, + collectionCount = 0, datasetCount = 0, userCount = 0, spaceBytes = 0, metadata = List.empty, resourceTimeToLive = formData.resourceTimeToLive * 60 * 60 * 1000L, isTimeToLiveEnabled = formData.isTimeToLiveEnabled, status = formData.access, affiliatedSpaces = formData.affSpace) @@ -648,4 +641,14 @@ class Spaces @Inject() (spaces: SpaceService, users: UserService, events: EventS } } + private def getFilesPerSpace(spaceId: UUID, user: models.User) : Integer = { + var spaceFiles: Integer = 0 + val allDatasetsInSpace = datasets.listSpace(0, spaceId.toString(), Some(user)) + for (ds <- allDatasetsInSpace) { + val files_in_ds = ds.files.length + spaceFiles += files_in_ds + } + spaceFiles + } + } diff --git a/app/models/Space.scala b/app/models/Space.scala index 0e0df7079..6f2d545e1 100644 --- a/app/models/Space.scala +++ b/app/models/Space.scala @@ -23,6 +23,7 @@ case class ProjectSpace ( collectionCount: Integer, datasetCount: Integer, userCount: Integer, + spaceBytes: Long, metadata: List[Metadata], resourceTimeToLive: Long = SpaceConfig.getTimeToLive(), isTimeToLiveEnabled: Boolean = SpaceConfig.getIsTimeToLiveEnabled(), @@ -52,7 +53,8 @@ case class UserSpace ( bannerURL: Option[URL], collectionCount: Integer, datasetCount: Integer, - userCount: Integer) + userCount: Integer, + spaceBytes: Long) case class SpaceInvite( id: UUID = UUID.generate, diff --git a/app/services/DatasetService.scala b/app/services/DatasetService.scala index d2538b4b1..071a707bf 100644 --- a/app/services/DatasetService.scala +++ b/app/services/DatasetService.scala @@ -295,6 +295,8 @@ trait DatasetService { def findMetadataChangedDatasets(): List[Dataset] + def getBytesForDataset(datasetId: UUID) : Long + /** * Check recursively whether a dataset's user-input metadata match a requested search tree. */ diff --git a/app/services/SpaceService.scala b/app/services/SpaceService.scala index a0cc4b486..3a9f1e60b 100644 --- a/app/services/SpaceService.scala +++ b/app/services/SpaceService.scala @@ -94,6 +94,10 @@ trait SpaceService { def incrementCollectionCounter(collection: UUID, space: UUID, increment: Int) + def incrementSpaceBytes(space: UUID, increment: Long) + + def decrementSpaceBytes(space: UUID, decrement: Long) + def addDataset(dataset: UUID, space: UUID) def removeDataset(dataset:UUID, space: UUID) diff --git a/app/services/mongodb/MongoDBDatasetService.scala b/app/services/mongodb/MongoDBDatasetService.scala index f2e2616c0..dac16f0c1 100644 --- a/app/services/mongodb/MongoDBDatasetService.scala +++ b/app/services/mongodb/MongoDBDatasetService.scala @@ -1068,6 +1068,17 @@ class MongoDBDatasetService @Inject() ( Dataset.find(MongoDBObject("userMetadataWasModified" -> true)).toList } + def getBytesForDataset(datasetId: UUID) : Long = { + val dataset = Dataset.findOneById(new ObjectId(datasetId.stringify)).get + val datasetFiles = dataset.files + var datasetBytes : Long = 0 + datasetFiles.foreach{ f => { + val currentFileBytes = files.get(f).get.length + datasetBytes += currentFileBytes + }} + datasetBytes + } + def removeTag(id: UUID, tagId: UUID) { Logger.debug("Removing tag " + tagId) val result = Dataset.update(MongoDBObject("_id" -> new ObjectId(id.stringify)), $pull("tags" -> MongoDBObject("_id" -> new ObjectId(tagId.stringify))), false, false, WriteConcern.Safe) diff --git a/app/services/mongodb/MongoDBSpaceService.scala b/app/services/mongodb/MongoDBSpaceService.scala index 749d46f72..6f164f8d0 100644 --- a/app/services/mongodb/MongoDBSpaceService.scala +++ b/app/services/mongodb/MongoDBSpaceService.scala @@ -378,6 +378,14 @@ class MongoDBSpaceService @Inject() ( ProjectSpaceDAO.update(MongoDBObject("_id" -> new ObjectId(space.stringify)), $inc("collectionCount" -> -1), upsert=false, multi=false, WriteConcern.Safe) } + def incrementSpaceBytes(space: UUID, increment: Long ): Unit = { + ProjectSpaceDAO.update(MongoDBObject("_id" -> new ObjectId(space.stringify)), $inc("spaceBytes" -> increment), upsert=false, multi=false, WriteConcern.Safe) + } + + def decrementSpaceBytes(space: UUID, decrement: Long): Unit = { + ProjectSpaceDAO.update(MongoDBObject("_id" -> new ObjectId(space.stringify)), $inc("spaceBytes" -> decrement), upsert=false, multi=false, WriteConcern.Safe) + } + def removeCollection(collection:UUID, space:UUID): Unit = { log.debug(s"Space Service - removing $collection from $space") collections.removeFromSpace(collection, space) @@ -390,7 +398,9 @@ class MongoDBSpaceService @Inject() ( */ def addDataset(dataset: UUID, space: UUID): Unit = { log.debug(s"Space Service - Adding $dataset to $space") + val datasetBytes = datasets.getBytesForDataset(dataset) datasets.addToSpace(dataset, space) + ProjectSpaceDAO.update(MongoDBObject("_id" -> new ObjectId(space.stringify)), $inc("spaceBytes" -> datasetBytes), upsert=false, multi=false, WriteConcern.Safe) ProjectSpaceDAO.update(MongoDBObject("_id" -> new ObjectId(space.stringify)), $inc("datasetCount" -> 1), upsert=false, multi=false, WriteConcern.Safe) } @@ -404,6 +414,8 @@ class MongoDBSpaceService @Inject() ( def removeDataset(dataset:UUID, space:UUID): Unit = { log.debug(s"Space Service - removing $dataset from $space") datasets.removeFromSpace(dataset, space) + val datasetBytes = datasets.getBytesForDataset(dataset) + ProjectSpaceDAO.update(MongoDBObject("_id" -> new ObjectId(space.stringify)), $inc("spaceBytes" -> -datasetBytes), upsert=false, multi=false, WriteConcern.Safe) ProjectSpaceDAO.update(MongoDBObject("_id" -> new ObjectId(space.stringify)), $inc("datasetCount" -> -1), upsert=false, multi=false, WriteConcern.Safe) } diff --git a/app/services/mongodb/MongoSalatPlugin.scala b/app/services/mongodb/MongoSalatPlugin.scala index a695cb602..b5ede2229 100644 --- a/app/services/mongodb/MongoSalatPlugin.scala +++ b/app/services/mongodb/MongoSalatPlugin.scala @@ -2,7 +2,6 @@ package services.mongodb import java.net.URL import java.util.{Calendar, Date} - import com.mongodb.{BasicDBObject, CommandFailureException} import com.mongodb.casbah.Imports._ import com.mongodb.casbah.commons.MongoDBObject @@ -15,10 +14,7 @@ import org.bson.BSONException import play.api.libs.json._ import play.api.{Application, Logger, Play, Plugin} import play.api.Play.current -import com.mongodb.casbah.MongoURI -import com.mongodb.casbah.MongoConnection -import com.mongodb.casbah.MongoDB -import com.mongodb.casbah.MongoCollection +import com.mongodb.casbah.{MongoCollection, MongoConnection, MongoDB, MongoURI, commons} import com.mongodb.casbah.gridfs.GridFS import com.mongodb.casbah.Imports.DBObject import org.bson.types.ObjectId @@ -453,6 +449,9 @@ class MongoSalatPlugin(app: Application) extends Plugin { // Updates extractors enabled and disabled in a space updateMongo("update-space-extractors-selection", updateSpaceExtractorsSelection) + + // Adds space bytes to space + updateMongo(updateKey = "update-space-bytes", updateSpaceBytes) } private def updateMongo(updateKey: String, block: () => Unit): Unit = { @@ -514,7 +513,7 @@ class MongoSalatPlugin(app: Application) extends Plugin { val spacename = java.net.InetAddress.getLocalHost.getHostName val newspace = new ProjectSpace(name = spacename, description = "", created = new Date(), creator = UUID("000000000000000000000000"), homePage = List.empty[URL], logoURL = None, bannerURL = None, metadata = List.empty[Metadata], - collectionCount = collections.toInt, datasetCount = datasets.toInt, userCount = users.toInt) + collectionCount = collections.toInt, datasetCount = datasets.toInt, userCount = users.toInt, spaceBytes = 0) ProjectSpaceDAO.save(newspace) val spaceId = new ObjectId(newspace.id.stringify) @@ -1690,4 +1689,22 @@ class MongoSalatPlugin(app: Application) extends Plugin { } print("DONE") } + + private def updateSpaceBytes(): Unit = { + val spaces = collection("spaces.projects").find().toList.foreach{ space => + var currentSpaceBytes: Long = 0 + val spaceId = space.get("_id") + val spaceDatasets = collection("datasets").find(MongoDBObject("spaces" -> spaceId)).toList + spaceDatasets.foreach{ spaceDataset => + val datasetFileIds = spaceDataset.getAsOrElse[MongoDBList]("files", MongoDBList.empty) + datasetFileIds.foreach{ fileId => + collection("uploads").findOne(MongoDBObject("_id" -> fileId)) match { + case Some(file) => currentSpaceBytes += file.get("length").asInstanceOf[Long] + case None => Logger.info(s"Could not find file ${fileId} in space ${spaceId}") + } + } + } + collection("spaces.projects").update(MongoDBObject("_id" -> spaceId), $set("spaceBytes" -> currentSpaceBytes)) + } + } } diff --git a/app/util/FileUtils.scala b/app/util/FileUtils.scala index 26fc7e86d..949fa6dab 100644 --- a/app/util/FileUtils.scala +++ b/app/util/FileUtils.scala @@ -1,15 +1,9 @@ package util -import java.io.{File => JFile} -import java.net.URL -import java.util.Date - -import collection.JavaConversions._ import api.UserRequest import controllers.Utils import fileutils.FilesUtils import models._ -import org.apache.commons.codec.digest.DigestUtils import play.api.Logger import play.api.Play._ import play.api.libs.Files @@ -18,13 +12,14 @@ import play.api.mvc.MultipartFormData import play.libs.Akka import services._ +import java.net.{URL, URLEncoder} +import java.util.Date +import javax.mail.internet.MimeUtility +import scala.collection.JavaConversions._ import scala.collection.mutable import scala.concurrent.{ExecutionContext, Future} import scala.util.Try -import javax.mail.internet.MimeUtility -import java.net.URLEncoder - object FileUtils { val appConfig: AppConfigurationService = DI.injector.getInstance(classOf[AppConfigurationService]) @@ -41,6 +36,7 @@ object FileUtils { lazy val thumbnails : ThumbnailService = DI.injector.getInstance(classOf[ThumbnailService]) lazy val routing : ExtractorRoutingService = DI.injector.getInstance(classOf[ExtractorRoutingService]) lazy val sinkService : EventSinkService = DI.injector.getInstance(classOf[EventSinkService]) + lazy val spaceService : SpaceService = DI.injector.getInstance(classOf[SpaceService]) def getContentType(filename: Option[String], contentType: Option[String]): String = { @@ -596,6 +592,10 @@ object FileUtils { events.addObjectEvent(Some(user), ds.id, ds.name, EventType.ADD_FILE.toString) } datasets.addFile(ds.id, file) + val datasetSpaces = dataset.get.spaces + for (s <- datasetSpaces){ + spaceService.incrementSpaceBytes(s, file.length) + } } } } diff --git a/app/util/JSONLD.scala b/app/util/JSONLD.scala index c45dca24d..3eacc7214 100644 --- a/app/util/JSONLD.scala +++ b/app/util/JSONLD.scala @@ -63,7 +63,8 @@ object JSONLD { ) //convert metadata to json using implicit writes in Metadata model - val metadataJson = resourceJson ++ toJson(metadata).asInstanceOf[JsObject] + //include metadata ID in the json + val metadataJson = JsObject(Seq("id" -> JsString(metadata.id.toString()))) ++ resourceJson ++ toJson(metadata).asInstanceOf[JsObject] //combine the two json objects and return if (contextJson.isEmpty) metadataJson else contextJson.get ++ metadataJson diff --git a/app/views/admin/customize.scala.html b/app/views/admin/customize.scala.html index f1c0c41fa..e8e00cef5 100644 --- a/app/views/admin/customize.scala.html +++ b/app/views/admin/customize.scala.html @@ -130,7 +130,7 @@
@Html(space.description.replace("\n","
"))