Release/1.19.0

.github/ISSUE_TEMPLATE/bug_report.md

@@ -2,7 +2,7 @@
 name: Bug report
 about: Create a report to help us improve
 title: ''
-labels: ''
+labels: bug
 assignees: ''
 
 ---

.github/ISSUE_TEMPLATE/feature_request.md

@@ -2,7 +2,7 @@
 name: Feature request
 about: Suggest an idea for this project
 title: ''
-labels: ''
+labels: enhancement
 assignees: ''
 
 ---

CHANGELOG.md

@@ -4,6 +4,23 @@ All notable changes to this project will be documented in this file.
 The format is based on [Keep a Changelog](http://keepachangelog.com/)
 and this project adheres to [Semantic Versioning](http://semver.org/).
 
+## 1.19.0 - 2021-10-05
+**_Important:_** This update requires a MongoDB update schema due to the new ability of showing summary statistics at the
+space level. Make sure to start the application with -DMONGOUPDATE=1.
+
+### Fixed
+- Adding dataset to space. Space list on dataset page would be empty - fixed error when no spaces would load. [#274](https://github.com/clowder-framework/clowder/issues/274)
+- Typos "success" when returning status from API and "occurred" when logging to console.
+- If a dataset had multiple folders the layout would be wrong.
+- Collections created using api route are now indexed upon creation. [#257](https://github.com/clowder-framework/clowder/issues/257)
+
+### Added
+- Mime type for geojson
+- "when" parameter in a few GET API endpoints to enable pagination [#266](https://github.com/clowder-framework/clowder/issues/266)
+- Show space statistics (bytes, users. etc) [#119](https://github.com/clowder-framework/clowder/issues/119)
+- "id" in GET metadata.jsonld endpoints [#278](https://github.com/clowder-framework/clowder/issues/278)
+- 'POST /api/files/uploadToDataset' now allows folder_id for uploading file to folder. [#232](https://github.com/clowder-framework/clowder/issues/232)
+
 ## 1.18.1 - 2021-08-16
 
 This release fixes a critical issue where invalid zip files could result in the files not being uploaded correctly. To check to see if you are affected, please use the following query:
@@ -55,7 +72,6 @@ If any files are returned, you should check to see if these files affected and a
 
 ### Changed
 - Updated Sphinx dependencies due to security and changes in required packages.
-
 - Updated the three.js libraries for the FBX previewer
 
 ## 1.16.0 - 2021-03-31

CONTRIBUTORS.md

@@ -34,6 +34,7 @@ Following is a list of contributors in alphabetical order:
 - Sandeep Puthanveetil Satheesan
 - Smruti Padhy
 - Theerasit Issaranon
+- Tim Yardley
 - Todd Nicholson
 - Varun Kethineedi
 - Ward Poelmans

app/api/Collections.scala

@@ -1,32 +1,28 @@
 package api
 
-import java.io.{ByteArrayInputStream, InputStream, ByteArrayOutputStream}
-import java.security.{DigestInputStream, MessageDigest}
-import java.text.SimpleDateFormat
-import java.util.zip.{ZipEntry, ZipOutputStream, Deflater}
-
 import Iterators.RootCollectionIterator
-import _root_.util.JSONLD
+import util.SearchUtils
 import api.Permission.Permission
-import org.apache.commons.codec.binary.Hex
+import controllers.Utils
+import models._
 import play.api.Logger
 import play.api.Play.current
-import models._
+import play.api.libs.concurrent.Execution.Implicits._
 import play.api.libs.iteratee.Enumerator
-import services._
-import play.api.libs.json._
-import play.api.libs.json.{JsObject, JsValue}
 import play.api.libs.json.Json.toJson
-import javax.inject.{ Singleton, Inject}
-import scala.collection.mutable.ListBuffer
-import scala.concurrent.{Future, ExecutionContext}
-import play.api.libs.concurrent.Execution.Implicits._
-import scala.util.parsing.json.JSONArray
-import scala.util.{Try, Success, Failure}
-import java.util.{Calendar, Date}
-import controllers.Utils
+import play.api.libs.json.{JsObject, JsValue, _}
+import services._
 
+
+import java.io.ByteArrayOutputStream
+import java.security.MessageDigest
+import java.util.zip.{Deflater, ZipOutputStream}
+import java.util.{Calendar, Date}
+import javax.inject.{Inject, Singleton}
 import scala.collection.immutable.List
+import scala.collection.mutable.ListBuffer
+import scala.concurrent.{ExecutionContext, Future}
+import scala.util.{Failure, Success, Try}
 
 
 /**
@@ -72,6 +68,13 @@ class Collections @Inject() (datasets: DatasetService,
                 collections.addToRootSpaces(c.id, s.id)
                 events.addSourceEvent(request.user, c.id, c.name, s.id, s.name, EventType.ADD_COLLECTION_SPACE.toString)
               })
+              // index collection
+              current.plugin[ElasticsearchPlugin].foreach{
+                _.index(SearchUtils.getElasticsearchObject(c))
+              }
+              //Add to Events Table
+              val option_user = userService.findByIdentity(identity)
+              events.addObjectEvent(option_user, c.id, c.name, EventType.CREATE_COLLECTION.toString)
               Ok(toJson(Map("id" -> id)))
             }
             case None => Ok(toJson(Map("status" -> "error")))
@@ -237,15 +240,15 @@ class Collections @Inject() (datasets: DatasetService,
     }
   }
 
-  def list(title: Option[String], date: Option[String], limit: Int, exact: Boolean) = PrivateServerAction { implicit request =>
-    Ok(toJson(listCollections(title, date, limit, Set[Permission](Permission.ViewCollection), false, request.user, request.user.fold(false)(_.superAdminMode), exact)))
+  def list(when: Option[String], title: Option[String], date: Option[String], limit: Int, exact: Boolean) = PrivateServerAction { implicit request =>
+    Ok(toJson(listCollections(when, title, date, limit, Set[Permission](Permission.ViewCollection), false, request.user, request.user.fold(false)(_.superAdminMode), exact)))
   }
 
-  def listCanEdit(title: Option[String], date: Option[String], limit: Int, exact: Boolean) = PrivateServerAction { implicit request =>
-    Ok(toJson(listCollections(title, date, limit, Set[Permission](Permission.AddResourceToCollection, Permission.EditCollection), false, request.user, request.user.fold(false)(_.superAdminMode), exact)))
+  def listCanEdit(when: Option[String], title: Option[String], date: Option[String], limit: Int, exact: Boolean) = PrivateServerAction { implicit request =>
+    Ok(toJson(listCollections(when, title, date, limit, Set[Permission](Permission.AddResourceToCollection, Permission.EditCollection), false, request.user, request.user.fold(false)(_.superAdminMode), exact)))
   }
 
-  def addDatasetToCollectionOptions(datasetId: UUID, title: Option[String], date: Option[String], limit: Int, exact: Boolean) = PrivateServerAction { implicit request =>
+  def addDatasetToCollectionOptions(when: Option[String], datasetId: UUID, title: Option[String], date: Option[String], limit: Int, exact: Boolean) = PrivateServerAction { implicit request =>
     implicit val user = request.user
     var listAll = false
     var collectionList: List[Collection] = List.empty
@@ -265,7 +268,7 @@ class Collections @Inject() (datasets: DatasetService,
       }
     }
     if(listAll) {
-      collectionList = listCollections(title, date, limit, Set[Permission](Permission.AddResourceToCollection, Permission.EditCollection), false, request.user, request.user.fold(false)(_.superAdminMode), exact)
+      collectionList = listCollections(when, title, date, limit, Set[Permission](Permission.AddResourceToCollection, Permission.EditCollection), false, request.user, request.user.fold(false)(_.superAdminMode), exact)
     }
     Ok(toJson(collectionList))
   }
@@ -274,10 +277,10 @@ class Collections @Inject() (datasets: DatasetService,
     collections.get(current_collections.map(_.child_collection_ids).flatten).found
   }
 
-  def listPossibleParents(currentCollectionId : String, title: Option[String], date: Option[String], limit: Int, exact: Boolean) = PrivateServerAction { implicit request =>
+  def listPossibleParents(when: Option[String], currentCollectionId : String, title: Option[String], date: Option[String], limit: Int, exact: Boolean) = PrivateServerAction { implicit request =>
     val selfAndAncestors = collections.getSelfAndAncestors(UUID(currentCollectionId))
     val descendants = collections.getAllDescendants(UUID(currentCollectionId)).toList
-    val allCollections = listCollections(title, date, limit, Set[Permission](Permission.AddResourceToCollection, Permission.EditCollection), false,
+    val allCollections = listCollections(when, title, date, limit, Set[Permission](Permission.AddResourceToCollection, Permission.EditCollection), false,
       request.user, request.user.fold(false)(_.superAdminMode), exact)
     val possibleNewParents = allCollections.filter(c =>
       if(play.api.Play.current.plugin[services.SpaceSharingPlugin].isDefined) {
@@ -303,29 +306,55 @@ class Collections @Inject() (datasets: DatasetService,
    * Returns list of collections based on parameters and permissions.
    * TODO this needs to be cleaned up when do permissions for adding to a resource
    */
-  private def listCollections(title: Option[String], date: Option[String], limit: Int, permission: Set[Permission], mine: Boolean, user: Option[User], superAdmin: Boolean, exact: Boolean) : List[Collection] = {
+  private def listCollections(when: Option[String], title: Option[String], date: Option[String], limit: Int, permission: Set[Permission], mine: Boolean, user: Option[User], superAdmin: Boolean, exact: Boolean) : List[Collection] = {
     if (mine && user.isEmpty) return List.empty[Collection]
 
-    (title, date) match {
-      case (Some(t), Some(d)) => {
+    (when, title, date) match {
+      case (Some(w), Some(t), Some(d)) => {
+        if (mine)
+          collections.listUser(d, nextPage=(w=="a"), limit, t, user, superAdmin, user.get, exact)
+        else
+          collections.listAccess(d, nextPage=(w=="a"), limit, t, permission, user, superAdmin, true,false, exact)
+      }
+      case (Some(w), Some(t), None) => {
+        if (mine)
+          collections.listUser(limit, t, user, superAdmin, user.get, exact)
+        else
+          collections.listAccess(limit, t, permission, user, superAdmin, true,false, exact)
+      }
+      case (Some(w), None, Some(d)) => {
+        if (mine)
+          collections.listUser(d, nextPage=(w=="a"), limit, user, superAdmin, user.get)
+        else
+          collections.listAccess(d, nextPage=(w=="a"), limit, permission, user, superAdmin, true,false)
+      }
+       case (Some(w), None, None) => {
+        if (mine)
+          collections.listUser(limit, user, superAdmin, user.get)
+        else
+          collections.listAccess(limit, permission, user, superAdmin, true,false)
+      }
+
+      // default when to be "after" if not present in parameters. i.e. nextPage=true
+      case (None, Some(t), Some(d)) => {
         if (mine)
           collections.listUser(d, true, limit, t, user, superAdmin, user.get, exact)
         else
           collections.listAccess(d, true, limit, t, permission, user, superAdmin, true,false, exact)
       }
-      case (Some(t), None) => {
+      case (None, Some(t), None) => {
         if (mine)
           collections.listUser(limit, t, user, superAdmin, user.get, exact)
         else
           collections.listAccess(limit, t, permission, user, superAdmin, true,false, exact)
       }
-      case (None, Some(d)) => {
+      case (None, None, Some(d)) => {
         if (mine)
           collections.listUser(d, true, limit, user, superAdmin, user.get)
         else
           collections.listAccess(d, true, limit, permission, user, superAdmin, true,false)
       }
-       case (None, None) => {
+      case (None, None, None) => {
         if (mine)
           collections.listUser(limit, user, superAdmin, user.get)
         else
@@ -572,6 +601,14 @@ class Collections @Inject() (datasets: DatasetService,
                   events.addSourceEvent(request.user, c.id, c.name, s.id, s.name, EventType.ADD_COLLECTION_SPACE.toString)
               }
 
+              // index collection
+              current.plugin[ElasticsearchPlugin].foreach{
+                _.index(SearchUtils.getElasticsearchObject(c))
+              }
+              //Add to Events Table
+              val option_user = userService.findByIdentity(identity)
+              events.addObjectEvent(option_user, c.id, c.name, EventType.CREATE_COLLECTION.toString)
+
               //do stuff with parent here
               (request.body \"parentId").asOpt[String] match {
                 case Some(parentId) => {

app/api/Datasets.scala

@@ -65,19 +65,19 @@ class  Datasets @Inject()(
     }
   }
 
-  def list(title: Option[String], date: Option[String], limit: Int, exact: Boolean) = PrivateServerAction { implicit request =>
-    Ok(toJson(listDatasets(title, date, limit, Set[Permission](Permission.ViewDataset), request.user, request.user.fold(false)(_.superAdminMode), exact)))
+  def list(when: Option[String], title: Option[String], date: Option[String], limit: Int, exact: Boolean) = PrivateServerAction { implicit request =>
+    Ok(toJson(listDatasets(when, title, date, limit, Set[Permission](Permission.ViewDataset), request.user, request.user.fold(false)(_.superAdminMode), exact)))
   }
 
-  def listCanEdit(title: Option[String], date: Option[String], limit: Int, exact: Boolean) = PrivateServerAction { implicit request =>
-      Ok(toJson(listDatasets(title, date, limit, Set[Permission](Permission.AddResourceToDataset, Permission.EditDataset), request.user, request.user.fold(false)(_.superAdminMode), exact)))
+  def listCanEdit(when: Option[String], title: Option[String], date: Option[String], limit: Int, exact: Boolean) = PrivateServerAction { implicit request =>
+      Ok(toJson(listDatasets(when, title, date, limit, Set[Permission](Permission.AddResourceToDataset, Permission.EditDataset), request.user, request.user.fold(false)(_.superAdminMode), exact)))
   }
 
   def listMoveFileToDataset(file_id: UUID, title: Option[String], limit: Int, exact: Boolean) = PrivateServerAction { implicit request =>
     if (play.Play.application().configuration().getBoolean("datasetFileWithinSpace")) {
       Ok(toJson(listDatasetsInSpace(file_id, title, limit, Set[Permission](Permission.AddResourceToDataset, Permission.EditDataset), request.user, request.user.fold(false)(_.superAdminMode), exact)))
     } else {
-      Ok(toJson(listDatasets(title, None, limit, Set[Permission](Permission.AddResourceToDataset, Permission.EditDataset), request.user, request.user.fold(false)(_.superAdminMode), exact)))
+      Ok(toJson(listDatasets(None, title, None, limit, Set[Permission](Permission.AddResourceToDataset, Permission.EditDataset), request.user, request.user.fold(false)(_.superAdminMode), exact)))
     }
   }
 
@@ -152,18 +152,31 @@ class  Datasets @Inject()(
   /**
     * Returns list of datasets based on parameters and permissions.
     */
-  private def listDatasets(title: Option[String], date: Option[String], limit: Int, permission: Set[Permission], user: Option[User], superAdmin: Boolean, exact: Boolean) : List[Dataset] = {
-    (title, date) match {
-      case (Some(t), Some(d)) => {
+  private def listDatasets(when: Option[String], title: Option[String], date: Option[String], limit: Int, permission: Set[Permission], user: Option[User], superAdmin: Boolean, exact: Boolean) : List[Dataset] = {
+    (when, title, date) match {
+      case (Some(w), Some(t), Some(d)) => {
+        datasets.listAccess(d, nextPage=(w=="a"), limit, t, permission, user, superAdmin, true,false, exact)
+      }
+      case (Some(w), Some(t), None) => {
+        datasets.listAccess(limit, t, permission, user, superAdmin, true,false, exact)
+      }
+      case (Some(w), None, Some(d)) => {
+        datasets.listAccess(d, nextPage=(w=="a"), limit, permission, user, superAdmin, true,false)
+      }
+      case (Some(w), None, None) => {
+        datasets.listAccess(limit, permission, user, superAdmin, true,false)
+      }
+      // default when to be "after" if not present in parameters. i.e. nextPage=true
+      case (None, Some(t), Some(d)) => {
         datasets.listAccess(d, true, limit, t, permission, user, superAdmin, true,false, exact)
       }
-      case (Some(t), None) => {
+      case (None, Some(t), None) => {
         datasets.listAccess(limit, t, permission, user, superAdmin, true,false, exact)
       }
-      case (None, Some(d)) => {
+      case (None, None, Some(d)) => {
         datasets.listAccess(d, true, limit, permission, user, superAdmin, true,false)
       }
-      case (None, None) => {
+      case (None, None, None) => {
         datasets.listAccess(limit, permission, user, superAdmin, true,false)
       }
     }

app/api/Files.scala

@@ -523,10 +523,16 @@ class Files @Inject()(
   /**
     * Upload a file to a specific dataset
     */
-  def uploadToDataset(dataset_id: UUID, showPreviews: String = "DatasetLevel", originalZipFile: String = "", flagsFromPrevious: String = "", extract: Boolean = true) = PermissionAction(Permission.AddResourceToDataset, Some(ResourceRef(ResourceRef.dataset, dataset_id)))(parse.multipartFormData) { implicit request =>
+  def uploadToDataset(dataset_id: UUID, showPreviews: String = "DatasetLevel", originalZipFile: String = "", flagsFromPrevious: String = "", extract: Boolean = true, folder_id: Option[String]) = PermissionAction(Permission.AddResourceToDataset, Some(ResourceRef(ResourceRef.dataset, dataset_id)))(parse.multipartFormData) { implicit request =>
     datasets.get(dataset_id) match {
       case Some(dataset) => {
-        val uploadedFiles = FileUtils.uploadFilesMultipart(request, Some(dataset), showPreviews = showPreviews, originalZipFile = originalZipFile, flagsFromPrevious = flagsFromPrevious, runExtractors = extract, apiKey = request.apiKey)
+        var current_folder : Option[Folder] = None
+        if (folder_id != None) {
+          if (UUID.isValid(folder_id.get)){
+            current_folder = folders.get(UUID(folder_id.get))
+          }
+        }
+        val uploadedFiles = FileUtils.uploadFilesMultipart(request, Some(dataset), current_folder, showPreviews = showPreviews, originalZipFile = originalZipFile, flagsFromPrevious = flagsFromPrevious, runExtractors = extract, apiKey = request.apiKey)
         uploadedFiles.length match {
           case 0 => BadRequest("No files uploaded")
           case 1 => Ok(Json.obj("id" -> uploadedFiles.head.id))
@@ -1669,6 +1675,10 @@ class Files @Inject()(
         // notify rabbitmq
         datasets.findByFileIdAllContain(file.id).foreach { ds =>
           routing.fileRemovedFromDataset(file, ds, Utils.baseUrl(request), request.apiKey)
+          val ds_spaces = ds.spaces
+          for (ds_s <- ds_spaces) {
+            spaces.decrementSpaceBytes(ds_s, file.length)
+          }
         }
 
         //this stmt has to be before files.removeFile

app/api/Selected.scala

@@ -87,7 +87,7 @@ class Selected @Inject()(selections: SelectionService,
         selections.get(user.email.get).map(d => {
           selections.remove(d.id, user.email.get)
         })
-        Ok(toJson(Map("sucess"->"true")))
+        Ok(toJson(Map("success"->"true")))
       }
     }
   }
@@ -100,7 +100,7 @@ class Selected @Inject()(selections: SelectionService,
           datasets.removeDataset(d.id, Utils.baseUrl(request), request.apiKey, request.user)
           selections.remove(d.id, user.email.get)
         })
-        Ok(toJson(Map("sucess"->"true")))
+        Ok(toJson(Map("success"->"true")))
       }
     }
   }
@@ -189,7 +189,7 @@ class Selected @Inject()(selections: SelectionService,
           events.addObjectEvent(request.user, d.id, d.name, EventType.ADD_TAGS_DATASET.toString)
           datasets.index(d.id)
         })
-        Ok(toJson(Map("sucess"->"true")))
+        Ok(toJson(Map("success"->"true")))
       }
     }
   }

app/api/Sensors.scala

@@ -5,7 +5,7 @@ import play.api.Play.current
 import services.PostgresPlugin
 
 /**
- * Metadata about sensors registered with the system. Datastreams can be associalted with sensors.
+ * Metadata about sensors registered with the system. Datastreams can be associated with sensors.
  */
 object Sensors extends Controller with ApiController {