199 show space bytes

CHANGELOG.md

@@ -14,6 +14,9 @@ and this project adheres to [Semantic Versioning](http://semver.org/).
 ### Added
 - Adding mime type for geojson
 - Add "when" parameter in a few GET API endpoints to enable pagination [#266](https://github.com/clowder-framework/clowder/issues/266)
+- Show statistics of spaces (bytes, users. etc) [#119](https://github.com/clowder-framework/clowder/issues/119)
+
+### Changed
 - Add "id" in GET metadata.jsonld endpoints [#278](https://github.com/clowder-framework/clowder/issues/278)
 
 ## 1.18.1 - 2021-08-16
@@ -71,7 +74,6 @@ If any files are returned, you should check to see if these files affected and a
 
 ### Changed
 - Updated Sphinx dependencies due to security and changes in required packages.
-
 - Updated the three.js libraries for the FBX previewer
 
 ## 1.16.0 - 2021-03-31

app/api/Files.scala

@@ -1675,6 +1675,10 @@ class Files @Inject()(
         // notify rabbitmq
         datasets.findByFileIdAllContain(file.id).foreach { ds =>
           routing.fileRemovedFromDataset(file, ds, Utils.baseUrl(request), request.apiKey)
+          val ds_spaces = ds.spaces
+          for (ds_s <- ds_spaces) {
+            spaces.decrementSpaceBytes(ds_s, file.length)
+          }
         }
 
         //this stmt has to be before files.removeFile

app/api/Spaces.scala

@@ -41,7 +41,7 @@ class Spaces @Inject()(spaces: SpaceService,
         val userId = request.user.get.id
         val c = ProjectSpace(name = name, description = description, created = new Date(), creator = userId,
           homePage = List.empty, logoURL = None, bannerURL = None, collectionCount = 0,
-          datasetCount = 0, userCount = 0, metadata = List.empty)
+          datasetCount = 0, userCount = 0, spaceBytes = 0, metadata = List.empty)
         spaces.insert(c) match {
           case Some(id) => {
             appConfig.incrementCount('spaces, 1)

app/controllers/SecuredController.scala

@@ -2,14 +2,11 @@ package controllers
 
 import api.Permission.Permission
 import api.{Permission, UserRequest}
-import models.{ClowderUser, RequestResource, ResourceRef, User, UserStatus}
-import org.apache.commons.lang.StringEscapeUtils._
+import models.{ClowderUser, ResourceRef, User, UserStatus}
 import play.api.i18n.Messages
 import play.api.mvc._
 import securesocial.core.{Authenticator, SecureSocial, UserService}
 import services._
-import securesocial.core.IdentityProvider
-import securesocial.core.providers.utils.RoutesHelper
 
 import scala.concurrent.Future
 
@@ -156,7 +153,7 @@ trait SecuredController extends Controller {
         val spaces: SpaceService = DI.injector.getInstance(classOf[SpaceService])
         spaces.get(id) match {
           case None => Future.successful(BadRequest(views.html.notFound(spaceTitle + " does not exist.")(user)))
-          case Some(space) => Future.successful(Forbidden(views.html.spaces.space(space,List(),List(),List(),List(),"", Map(),List())(user)))
+          case Some(space) => Future.successful(Forbidden(views.html.spaces.space(space,List(),List(),List(),List(),"", Map(),List(),0,0)(user)))
         }
       }
 

app/controllers/Spaces.scala

@@ -1,31 +1,22 @@
 package controllers
 
-import java.net.URL
-import java.util.{ Calendar, Date }
-import javax.inject.Inject
-
 import api.Permission
 import api.Permission._
 import models._
-import play.api.{ Logger, Play }
+import org.joda.time.DateTime
 import play.api.data.Forms._
-import play.api.data.{ Form, Forms }
-import play.api.libs.json.JsValue
-import play.api.libs.json.Json
+import play.api.data.{Form, Forms}
 import play.api.i18n.Messages
+import play.api.{Logger, Play}
+import securesocial.core.providers.{Token, UsernamePasswordProvider}
 import services._
-import securesocial.core.providers.{ Token, UsernamePasswordProvider }
-import org.joda.time.DateTime
-import play.api.i18n.Messages
-import play.api.libs.ws._
-import services.AppConfiguration
-import util.{ Formatters, Mail, Publications }
+import util.{Formatters, Mail, Publications}
 
+import java.net.URL
+import java.util.{Calendar, Date}
+import javax.inject.Inject
 import scala.collection.immutable.List
-import scala.collection.mutable.{ ArrayBuffer, ListBuffer }
-import scala.concurrent.{ Future, Await }
-import scala.concurrent.duration._
-import org.apache.commons.lang.StringEscapeUtils.escapeJava
+import scala.collection.mutable.{ArrayBuffer, ListBuffer}
 
 /**
  * Spaces allow users to partition the data into realms only accessible to users with the right permissions.
@@ -176,6 +167,8 @@ class Spaces @Inject() (spaces: SpaceService, users: UserService, events: EventS
         var creatorActual: User = null
         val collectionsInSpace = spaces.getCollectionsInSpace(Some(id.stringify), Some(size))
         val datasetsInSpace = datasets.listSpace(size, id.toString(), user)
+        val spaceBytes : Long = s.spaceBytes
+        val spaceFiles : Integer = getFilesPerSpace(id, user.get)
         val publicDatasetsInSpace = datasets.listSpaceStatus(size, id.toString(), "publicAll", user)
         val usersInSpace = spaces.getUsersInSpace(id, None)
         var curationObjectsInSpace: List[CurationObject] = List()
@@ -224,7 +217,7 @@ class Spaces @Inject() (spaces: SpaceService, users: UserService, events: EventS
           case None => List.empty
         }
         sinkService.logSpaceViewEvent(s, user)
-        Ok(views.html.spaces.space(Utils.decodeSpaceElements(s), collectionsInSpace, publicDatasetsInSpace, datasetsInSpace, rs, play.Play.application().configuration().getString("SEADservices.uri"), userRoleMap, userSelections))
+        Ok(views.html.spaces.space(Utils.decodeSpaceElements(s), collectionsInSpace, publicDatasetsInSpace, datasetsInSpace, rs, play.Play.application().configuration().getString("SEADservices.uri"), userRoleMap, userSelections, spaceBytes, spaceFiles))
       }
       case None => BadRequest(views.html.notFound(spaceTitle + " does not exist."))
     }
@@ -421,7 +414,7 @@ class Spaces @Inject() (spaces: SpaceService, users: UserService, events: EventS
                       val newSpace = ProjectSpace(name = formData.name, description = formData.description,
                         created = new Date, creator = userId, homePage = formData.homePage,
                         logoURL = formData.logoURL, bannerURL = formData.bannerURL,
-                        collectionCount = 0, datasetCount = 0, userCount = 0, metadata = List.empty,
+                        collectionCount = 0, datasetCount = 0, userCount = 0, spaceBytes = 0, metadata = List.empty,
                         resourceTimeToLive = formData.resourceTimeToLive * 60 * 60 * 1000L, isTimeToLiveEnabled = formData.isTimeToLiveEnabled,
                         status = formData.access,
                         affiliatedSpaces = formData.affSpace)
@@ -648,4 +641,14 @@ class Spaces @Inject() (spaces: SpaceService, users: UserService, events: EventS
       }
   }
 
+  private def getFilesPerSpace(spaceId: UUID, user: models.User) : Integer = {
+    var spaceFiles: Integer = 0
+    val allDatasetsInSpace = datasets.listSpace(0, spaceId.toString(), Some(user))
+    for (ds <- allDatasetsInSpace) {
+      val files_in_ds = ds.files.length
+      spaceFiles += files_in_ds
+    }
+    spaceFiles
+  }
+
 }

app/models/Space.scala

@@ -23,6 +23,7 @@ case class ProjectSpace (
   collectionCount: Integer,
   datasetCount: Integer,
   userCount: Integer,
+  spaceBytes: Long, 
   metadata: List[Metadata],
   resourceTimeToLive: Long = SpaceConfig.getTimeToLive(),
   isTimeToLiveEnabled: Boolean = SpaceConfig.getIsTimeToLiveEnabled(),
@@ -52,7 +53,8 @@ case class UserSpace (
    bannerURL: Option[URL],
    collectionCount: Integer,
    datasetCount: Integer,
-   userCount: Integer)
+   userCount: Integer,
+   spaceBytes: Long)
 
 case class SpaceInvite(
     id: UUID = UUID.generate,

app/services/DatasetService.scala

@@ -295,6 +295,8 @@ trait DatasetService {
 
   def findMetadataChangedDatasets(): List[Dataset]
 
+  def getBytesForDataset(datasetId: UUID) : Long
+
   /**
    * Check recursively whether a dataset's user-input metadata match a requested search tree.
    */

app/services/SpaceService.scala

@@ -94,6 +94,10 @@ trait SpaceService {
 
   def incrementCollectionCounter(collection: UUID, space: UUID, increment: Int)
 
+  def incrementSpaceBytes(space: UUID, increment: Long)
+
+  def decrementSpaceBytes(space: UUID, decrement: Long)
+
   def addDataset(dataset: UUID, space: UUID)
 
   def removeDataset(dataset:UUID, space: UUID)

app/services/mongodb/MongoDBDatasetService.scala

@@ -1068,6 +1068,17 @@ class MongoDBDatasetService @Inject() (
     Dataset.find(MongoDBObject("userMetadataWasModified" -> true)).toList
   }
 
+  def getBytesForDataset(datasetId: UUID) : Long = {
+    val dataset = Dataset.findOneById(new ObjectId(datasetId.stringify)).get
+    val datasetFiles = dataset.files
+    var datasetBytes : Long = 0
+    datasetFiles.foreach{ f => {
+      val currentFileBytes = files.get(f).get.length
+      datasetBytes += currentFileBytes
+    }}
+    datasetBytes
+  }
+
   def removeTag(id: UUID, tagId: UUID) {
     Logger.debug("Removing tag " + tagId)
     val result = Dataset.update(MongoDBObject("_id" -> new ObjectId(id.stringify)), $pull("tags" -> MongoDBObject("_id" -> new ObjectId(tagId.stringify))), false, false, WriteConcern.Safe)

app/services/mongodb/MongoDBSpaceService.scala

@@ -378,6 +378,14 @@ class MongoDBSpaceService @Inject() (
     ProjectSpaceDAO.update(MongoDBObject("_id" -> new ObjectId(space.stringify)), $inc("collectionCount" -> -1), upsert=false, multi=false, WriteConcern.Safe)
   }
 
+  def incrementSpaceBytes(space: UUID, increment: Long ): Unit = {
+    ProjectSpaceDAO.update(MongoDBObject("_id" -> new ObjectId(space.stringify)), $inc("spaceBytes" -> increment), upsert=false, multi=false, WriteConcern.Safe)
+  }
+
+  def decrementSpaceBytes(space: UUID, decrement: Long): Unit = {
+    ProjectSpaceDAO.update(MongoDBObject("_id" -> new ObjectId(space.stringify)), $inc("spaceBytes" -> decrement), upsert=false, multi=false, WriteConcern.Safe)
+  }
+
   def removeCollection(collection:UUID, space:UUID): Unit = {
     log.debug(s"Space Service - removing $collection from $space")
     collections.removeFromSpace(collection, space)
@@ -390,7 +398,9 @@ class MongoDBSpaceService @Inject() (
    */
   def addDataset(dataset: UUID, space: UUID): Unit = {
     log.debug(s"Space Service - Adding $dataset to $space")
+    val datasetBytes = datasets.getBytesForDataset(dataset)
     datasets.addToSpace(dataset, space)
+    ProjectSpaceDAO.update(MongoDBObject("_id" -> new ObjectId(space.stringify)), $inc("spaceBytes" -> datasetBytes), upsert=false, multi=false, WriteConcern.Safe)
     ProjectSpaceDAO.update(MongoDBObject("_id" -> new ObjectId(space.stringify)), $inc("datasetCount" -> 1), upsert=false, multi=false, WriteConcern.Safe)
 
   }
@@ -404,6 +414,8 @@ class MongoDBSpaceService @Inject() (
   def removeDataset(dataset:UUID, space:UUID): Unit = {
     log.debug(s"Space Service - removing $dataset from $space")
     datasets.removeFromSpace(dataset, space)
+    val datasetBytes = datasets.getBytesForDataset(dataset)
+    ProjectSpaceDAO.update(MongoDBObject("_id" -> new ObjectId(space.stringify)), $inc("spaceBytes" -> -datasetBytes), upsert=false, multi=false, WriteConcern.Safe)
     ProjectSpaceDAO.update(MongoDBObject("_id" -> new ObjectId(space.stringify)), $inc("datasetCount" -> -1), upsert=false, multi=false, WriteConcern.Safe)
   }
 

app/services/mongodb/MongoSalatPlugin.scala

@@ -2,7 +2,6 @@ package services.mongodb
 
 import java.net.URL
 import java.util.{Calendar, Date}
-
 import com.mongodb.{BasicDBObject, CommandFailureException}
 import com.mongodb.casbah.Imports._
 import com.mongodb.casbah.commons.MongoDBObject
@@ -15,10 +14,7 @@ import org.bson.BSONException
 import play.api.libs.json._
 import play.api.{Application, Logger, Play, Plugin}
 import play.api.Play.current
-import com.mongodb.casbah.MongoURI
-import com.mongodb.casbah.MongoConnection
-import com.mongodb.casbah.MongoDB
-import com.mongodb.casbah.MongoCollection
+import com.mongodb.casbah.{MongoCollection, MongoConnection, MongoDB, MongoURI, commons}
 import com.mongodb.casbah.gridfs.GridFS
 import com.mongodb.casbah.Imports.DBObject
 import org.bson.types.ObjectId
@@ -453,6 +449,9 @@ class MongoSalatPlugin(app: Application) extends Plugin {
 
     // Updates extractors enabled and disabled in a space
     updateMongo("update-space-extractors-selection", updateSpaceExtractorsSelection)
+
+    // Adds space bytes to space
+    updateMongo(updateKey = "update-space-bytes", updateSpaceBytes)
   }
 
   private def updateMongo(updateKey: String, block: () => Unit): Unit = {
@@ -514,7 +513,7 @@ class MongoSalatPlugin(app: Application) extends Plugin {
         val spacename = java.net.InetAddress.getLocalHost.getHostName
         val newspace = new ProjectSpace(name = spacename, description = "", created = new Date(), creator = UUID("000000000000000000000000"),
           homePage = List.empty[URL], logoURL = None, bannerURL = None, metadata = List.empty[Metadata],
-          collectionCount = collections.toInt, datasetCount = datasets.toInt, userCount = users.toInt)
+          collectionCount = collections.toInt, datasetCount = datasets.toInt, userCount = users.toInt, spaceBytes = 0)
         ProjectSpaceDAO.save(newspace)
         val spaceId = new ObjectId(newspace.id.stringify)
 
@@ -1690,4 +1689,22 @@ class MongoSalatPlugin(app: Application) extends Plugin {
     }
     print("DONE")
   }
+
+  private def updateSpaceBytes(): Unit = {
+    val spaces = collection("spaces.projects").find().toList.foreach{ space =>
+      var currentSpaceBytes: Long = 0
+      val spaceId = space.get("_id")
+      val spaceDatasets = collection("datasets").find(MongoDBObject("spaces" -> spaceId)).toList
+      spaceDatasets.foreach{ spaceDataset =>
+        val datasetFileIds = spaceDataset.getAsOrElse[MongoDBList]("files", MongoDBList.empty)
+        datasetFileIds.foreach{ fileId =>
+          collection("uploads").findOne(MongoDBObject("_id" -> fileId)) match {
+            case Some(file) => currentSpaceBytes += file.get("length").asInstanceOf[Long]
+            case None => Logger.info(s"Could not find file ${fileId} in space ${spaceId}")
+          }
+        }
+      }
+      collection("spaces.projects").update(MongoDBObject("_id" -> spaceId), $set("spaceBytes" -> currentSpaceBytes))
+    }
+  }
 }

app/util/FileUtils.scala

@@ -1,15 +1,9 @@
 package util
 
-import java.io.{File => JFile}
-import java.net.URL
-import java.util.Date
-
-import collection.JavaConversions._
 import api.UserRequest
 import controllers.Utils
 import fileutils.FilesUtils
 import models._
-import org.apache.commons.codec.digest.DigestUtils
 import play.api.Logger
 import play.api.Play._
 import play.api.libs.Files
@@ -18,13 +12,14 @@ import play.api.mvc.MultipartFormData
 import play.libs.Akka
 import services._
 
+import java.net.{URL, URLEncoder}
+import java.util.Date
+import javax.mail.internet.MimeUtility
+import scala.collection.JavaConversions._
 import scala.collection.mutable
 import scala.concurrent.{ExecutionContext, Future}
 import scala.util.Try
 
-import javax.mail.internet.MimeUtility
-import java.net.URLEncoder
-
 object FileUtils {
   val appConfig: AppConfigurationService = DI.injector.getInstance(classOf[AppConfigurationService])
 
@@ -41,6 +36,7 @@ object FileUtils {
   lazy val thumbnails : ThumbnailService = DI.injector.getInstance(classOf[ThumbnailService])
   lazy val routing : ExtractorRoutingService = DI.injector.getInstance(classOf[ExtractorRoutingService])
   lazy val sinkService : EventSinkService = DI.injector.getInstance(classOf[EventSinkService])
+  lazy val spaceService : SpaceService = DI.injector.getInstance(classOf[SpaceService])
 
 
   def getContentType(filename: Option[String], contentType: Option[String]): String = {
@@ -596,6 +592,10 @@ object FileUtils {
             events.addObjectEvent(Some(user), ds.id, ds.name, EventType.ADD_FILE.toString)
           }
           datasets.addFile(ds.id, file)
+          val datasetSpaces = dataset.get.spaces
+          for (s <- datasetSpaces){
+            spaceService.incrementSpaceBytes(s, file.length)
+          }
         }
       }
     }