diff --git a/CHANGELOG.md b/CHANGELOG.md index 4e4c2f9c6..fa3955f0c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -14,6 +14,9 @@ and this project adheres to [Semantic Versioning](http://semver.org/). ### Added - Adding mime type for geojson - Add "when" parameter in a few GET API endpoints to enable pagination [#266](https://github.com/clowder-framework/clowder/issues/266) +- Show statistics of spaces (bytes, users. etc) [#119](https://github.com/clowder-framework/clowder/issues/119) + +### Changed - Add "id" in GET metadata.jsonld endpoints [#278](https://github.com/clowder-framework/clowder/issues/278) ## 1.18.1 - 2021-08-16 @@ -71,7 +74,6 @@ If any files are returned, you should check to see if these files affected and a ### Changed - Updated Sphinx dependencies due to security and changes in required packages. - - Updated the three.js libraries for the FBX previewer ## 1.16.0 - 2021-03-31 diff --git a/app/api/Files.scala b/app/api/Files.scala index f19084dd9..7f141c4ec 100644 --- a/app/api/Files.scala +++ b/app/api/Files.scala @@ -1675,6 +1675,10 @@ class Files @Inject()( // notify rabbitmq datasets.findByFileIdAllContain(file.id).foreach { ds => routing.fileRemovedFromDataset(file, ds, Utils.baseUrl(request), request.apiKey) + val ds_spaces = ds.spaces + for (ds_s <- ds_spaces) { + spaces.decrementSpaceBytes(ds_s, file.length) + } } //this stmt has to be before files.removeFile diff --git a/app/api/Spaces.scala b/app/api/Spaces.scala index 7d762485f..d82ba2c5e 100644 --- a/app/api/Spaces.scala +++ b/app/api/Spaces.scala @@ -41,7 +41,7 @@ class Spaces @Inject()(spaces: SpaceService, val userId = request.user.get.id val c = ProjectSpace(name = name, description = description, created = new Date(), creator = userId, homePage = List.empty, logoURL = None, bannerURL = None, collectionCount = 0, - datasetCount = 0, userCount = 0, metadata = List.empty) + datasetCount = 0, userCount = 0, spaceBytes = 0, metadata = List.empty) spaces.insert(c) match { case Some(id) => { appConfig.incrementCount('spaces, 1) diff --git a/app/controllers/SecuredController.scala b/app/controllers/SecuredController.scala index be5035931..0f527f3db 100644 --- a/app/controllers/SecuredController.scala +++ b/app/controllers/SecuredController.scala @@ -2,14 +2,11 @@ package controllers import api.Permission.Permission import api.{Permission, UserRequest} -import models.{ClowderUser, RequestResource, ResourceRef, User, UserStatus} -import org.apache.commons.lang.StringEscapeUtils._ +import models.{ClowderUser, ResourceRef, User, UserStatus} import play.api.i18n.Messages import play.api.mvc._ import securesocial.core.{Authenticator, SecureSocial, UserService} import services._ -import securesocial.core.IdentityProvider -import securesocial.core.providers.utils.RoutesHelper import scala.concurrent.Future @@ -156,7 +153,7 @@ trait SecuredController extends Controller { val spaces: SpaceService = DI.injector.getInstance(classOf[SpaceService]) spaces.get(id) match { case None => Future.successful(BadRequest(views.html.notFound(spaceTitle + " does not exist.")(user))) - case Some(space) => Future.successful(Forbidden(views.html.spaces.space(space,List(),List(),List(),List(),"", Map(),List())(user))) + case Some(space) => Future.successful(Forbidden(views.html.spaces.space(space,List(),List(),List(),List(),"", Map(),List(),0,0)(user))) } } diff --git a/app/controllers/Spaces.scala b/app/controllers/Spaces.scala index f95d5b07b..ec0ae5d2b 100644 --- a/app/controllers/Spaces.scala +++ b/app/controllers/Spaces.scala @@ -1,31 +1,22 @@ package controllers -import java.net.URL -import java.util.{ Calendar, Date } -import javax.inject.Inject - import api.Permission import api.Permission._ import models._ -import play.api.{ Logger, Play } +import org.joda.time.DateTime import play.api.data.Forms._ -import play.api.data.{ Form, Forms } -import play.api.libs.json.JsValue -import play.api.libs.json.Json +import play.api.data.{Form, Forms} import play.api.i18n.Messages +import play.api.{Logger, Play} +import securesocial.core.providers.{Token, UsernamePasswordProvider} import services._ -import securesocial.core.providers.{ Token, UsernamePasswordProvider } -import org.joda.time.DateTime -import play.api.i18n.Messages -import play.api.libs.ws._ -import services.AppConfiguration -import util.{ Formatters, Mail, Publications } +import util.{Formatters, Mail, Publications} +import java.net.URL +import java.util.{Calendar, Date} +import javax.inject.Inject import scala.collection.immutable.List -import scala.collection.mutable.{ ArrayBuffer, ListBuffer } -import scala.concurrent.{ Future, Await } -import scala.concurrent.duration._ -import org.apache.commons.lang.StringEscapeUtils.escapeJava +import scala.collection.mutable.{ArrayBuffer, ListBuffer} /** * Spaces allow users to partition the data into realms only accessible to users with the right permissions. @@ -176,6 +167,8 @@ class Spaces @Inject() (spaces: SpaceService, users: UserService, events: EventS var creatorActual: User = null val collectionsInSpace = spaces.getCollectionsInSpace(Some(id.stringify), Some(size)) val datasetsInSpace = datasets.listSpace(size, id.toString(), user) + val spaceBytes : Long = s.spaceBytes + val spaceFiles : Integer = getFilesPerSpace(id, user.get) val publicDatasetsInSpace = datasets.listSpaceStatus(size, id.toString(), "publicAll", user) val usersInSpace = spaces.getUsersInSpace(id, None) var curationObjectsInSpace: List[CurationObject] = List() @@ -224,7 +217,7 @@ class Spaces @Inject() (spaces: SpaceService, users: UserService, events: EventS case None => List.empty } sinkService.logSpaceViewEvent(s, user) - Ok(views.html.spaces.space(Utils.decodeSpaceElements(s), collectionsInSpace, publicDatasetsInSpace, datasetsInSpace, rs, play.Play.application().configuration().getString("SEADservices.uri"), userRoleMap, userSelections)) + Ok(views.html.spaces.space(Utils.decodeSpaceElements(s), collectionsInSpace, publicDatasetsInSpace, datasetsInSpace, rs, play.Play.application().configuration().getString("SEADservices.uri"), userRoleMap, userSelections, spaceBytes, spaceFiles)) } case None => BadRequest(views.html.notFound(spaceTitle + " does not exist.")) } @@ -421,7 +414,7 @@ class Spaces @Inject() (spaces: SpaceService, users: UserService, events: EventS val newSpace = ProjectSpace(name = formData.name, description = formData.description, created = new Date, creator = userId, homePage = formData.homePage, logoURL = formData.logoURL, bannerURL = formData.bannerURL, - collectionCount = 0, datasetCount = 0, userCount = 0, metadata = List.empty, + collectionCount = 0, datasetCount = 0, userCount = 0, spaceBytes = 0, metadata = List.empty, resourceTimeToLive = formData.resourceTimeToLive * 60 * 60 * 1000L, isTimeToLiveEnabled = formData.isTimeToLiveEnabled, status = formData.access, affiliatedSpaces = formData.affSpace) @@ -648,4 +641,14 @@ class Spaces @Inject() (spaces: SpaceService, users: UserService, events: EventS } } + private def getFilesPerSpace(spaceId: UUID, user: models.User) : Integer = { + var spaceFiles: Integer = 0 + val allDatasetsInSpace = datasets.listSpace(0, spaceId.toString(), Some(user)) + for (ds <- allDatasetsInSpace) { + val files_in_ds = ds.files.length + spaceFiles += files_in_ds + } + spaceFiles + } + } diff --git a/app/models/Space.scala b/app/models/Space.scala index 0e0df7079..6f2d545e1 100644 --- a/app/models/Space.scala +++ b/app/models/Space.scala @@ -23,6 +23,7 @@ case class ProjectSpace ( collectionCount: Integer, datasetCount: Integer, userCount: Integer, + spaceBytes: Long, metadata: List[Metadata], resourceTimeToLive: Long = SpaceConfig.getTimeToLive(), isTimeToLiveEnabled: Boolean = SpaceConfig.getIsTimeToLiveEnabled(), @@ -52,7 +53,8 @@ case class UserSpace ( bannerURL: Option[URL], collectionCount: Integer, datasetCount: Integer, - userCount: Integer) + userCount: Integer, + spaceBytes: Long) case class SpaceInvite( id: UUID = UUID.generate, diff --git a/app/services/DatasetService.scala b/app/services/DatasetService.scala index d2538b4b1..071a707bf 100644 --- a/app/services/DatasetService.scala +++ b/app/services/DatasetService.scala @@ -295,6 +295,8 @@ trait DatasetService { def findMetadataChangedDatasets(): List[Dataset] + def getBytesForDataset(datasetId: UUID) : Long + /** * Check recursively whether a dataset's user-input metadata match a requested search tree. */ diff --git a/app/services/SpaceService.scala b/app/services/SpaceService.scala index a0cc4b486..3a9f1e60b 100644 --- a/app/services/SpaceService.scala +++ b/app/services/SpaceService.scala @@ -94,6 +94,10 @@ trait SpaceService { def incrementCollectionCounter(collection: UUID, space: UUID, increment: Int) + def incrementSpaceBytes(space: UUID, increment: Long) + + def decrementSpaceBytes(space: UUID, decrement: Long) + def addDataset(dataset: UUID, space: UUID) def removeDataset(dataset:UUID, space: UUID) diff --git a/app/services/mongodb/MongoDBDatasetService.scala b/app/services/mongodb/MongoDBDatasetService.scala index f2e2616c0..dac16f0c1 100644 --- a/app/services/mongodb/MongoDBDatasetService.scala +++ b/app/services/mongodb/MongoDBDatasetService.scala @@ -1068,6 +1068,17 @@ class MongoDBDatasetService @Inject() ( Dataset.find(MongoDBObject("userMetadataWasModified" -> true)).toList } + def getBytesForDataset(datasetId: UUID) : Long = { + val dataset = Dataset.findOneById(new ObjectId(datasetId.stringify)).get + val datasetFiles = dataset.files + var datasetBytes : Long = 0 + datasetFiles.foreach{ f => { + val currentFileBytes = files.get(f).get.length + datasetBytes += currentFileBytes + }} + datasetBytes + } + def removeTag(id: UUID, tagId: UUID) { Logger.debug("Removing tag " + tagId) val result = Dataset.update(MongoDBObject("_id" -> new ObjectId(id.stringify)), $pull("tags" -> MongoDBObject("_id" -> new ObjectId(tagId.stringify))), false, false, WriteConcern.Safe) diff --git a/app/services/mongodb/MongoDBSpaceService.scala b/app/services/mongodb/MongoDBSpaceService.scala index 749d46f72..6f164f8d0 100644 --- a/app/services/mongodb/MongoDBSpaceService.scala +++ b/app/services/mongodb/MongoDBSpaceService.scala @@ -378,6 +378,14 @@ class MongoDBSpaceService @Inject() ( ProjectSpaceDAO.update(MongoDBObject("_id" -> new ObjectId(space.stringify)), $inc("collectionCount" -> -1), upsert=false, multi=false, WriteConcern.Safe) } + def incrementSpaceBytes(space: UUID, increment: Long ): Unit = { + ProjectSpaceDAO.update(MongoDBObject("_id" -> new ObjectId(space.stringify)), $inc("spaceBytes" -> increment), upsert=false, multi=false, WriteConcern.Safe) + } + + def decrementSpaceBytes(space: UUID, decrement: Long): Unit = { + ProjectSpaceDAO.update(MongoDBObject("_id" -> new ObjectId(space.stringify)), $inc("spaceBytes" -> decrement), upsert=false, multi=false, WriteConcern.Safe) + } + def removeCollection(collection:UUID, space:UUID): Unit = { log.debug(s"Space Service - removing $collection from $space") collections.removeFromSpace(collection, space) @@ -390,7 +398,9 @@ class MongoDBSpaceService @Inject() ( */ def addDataset(dataset: UUID, space: UUID): Unit = { log.debug(s"Space Service - Adding $dataset to $space") + val datasetBytes = datasets.getBytesForDataset(dataset) datasets.addToSpace(dataset, space) + ProjectSpaceDAO.update(MongoDBObject("_id" -> new ObjectId(space.stringify)), $inc("spaceBytes" -> datasetBytes), upsert=false, multi=false, WriteConcern.Safe) ProjectSpaceDAO.update(MongoDBObject("_id" -> new ObjectId(space.stringify)), $inc("datasetCount" -> 1), upsert=false, multi=false, WriteConcern.Safe) } @@ -404,6 +414,8 @@ class MongoDBSpaceService @Inject() ( def removeDataset(dataset:UUID, space:UUID): Unit = { log.debug(s"Space Service - removing $dataset from $space") datasets.removeFromSpace(dataset, space) + val datasetBytes = datasets.getBytesForDataset(dataset) + ProjectSpaceDAO.update(MongoDBObject("_id" -> new ObjectId(space.stringify)), $inc("spaceBytes" -> -datasetBytes), upsert=false, multi=false, WriteConcern.Safe) ProjectSpaceDAO.update(MongoDBObject("_id" -> new ObjectId(space.stringify)), $inc("datasetCount" -> -1), upsert=false, multi=false, WriteConcern.Safe) } diff --git a/app/services/mongodb/MongoSalatPlugin.scala b/app/services/mongodb/MongoSalatPlugin.scala index a695cb602..b5ede2229 100644 --- a/app/services/mongodb/MongoSalatPlugin.scala +++ b/app/services/mongodb/MongoSalatPlugin.scala @@ -2,7 +2,6 @@ package services.mongodb import java.net.URL import java.util.{Calendar, Date} - import com.mongodb.{BasicDBObject, CommandFailureException} import com.mongodb.casbah.Imports._ import com.mongodb.casbah.commons.MongoDBObject @@ -15,10 +14,7 @@ import org.bson.BSONException import play.api.libs.json._ import play.api.{Application, Logger, Play, Plugin} import play.api.Play.current -import com.mongodb.casbah.MongoURI -import com.mongodb.casbah.MongoConnection -import com.mongodb.casbah.MongoDB -import com.mongodb.casbah.MongoCollection +import com.mongodb.casbah.{MongoCollection, MongoConnection, MongoDB, MongoURI, commons} import com.mongodb.casbah.gridfs.GridFS import com.mongodb.casbah.Imports.DBObject import org.bson.types.ObjectId @@ -453,6 +449,9 @@ class MongoSalatPlugin(app: Application) extends Plugin { // Updates extractors enabled and disabled in a space updateMongo("update-space-extractors-selection", updateSpaceExtractorsSelection) + + // Adds space bytes to space + updateMongo(updateKey = "update-space-bytes", updateSpaceBytes) } private def updateMongo(updateKey: String, block: () => Unit): Unit = { @@ -514,7 +513,7 @@ class MongoSalatPlugin(app: Application) extends Plugin { val spacename = java.net.InetAddress.getLocalHost.getHostName val newspace = new ProjectSpace(name = spacename, description = "", created = new Date(), creator = UUID("000000000000000000000000"), homePage = List.empty[URL], logoURL = None, bannerURL = None, metadata = List.empty[Metadata], - collectionCount = collections.toInt, datasetCount = datasets.toInt, userCount = users.toInt) + collectionCount = collections.toInt, datasetCount = datasets.toInt, userCount = users.toInt, spaceBytes = 0) ProjectSpaceDAO.save(newspace) val spaceId = new ObjectId(newspace.id.stringify) @@ -1690,4 +1689,22 @@ class MongoSalatPlugin(app: Application) extends Plugin { } print("DONE") } + + private def updateSpaceBytes(): Unit = { + val spaces = collection("spaces.projects").find().toList.foreach{ space => + var currentSpaceBytes: Long = 0 + val spaceId = space.get("_id") + val spaceDatasets = collection("datasets").find(MongoDBObject("spaces" -> spaceId)).toList + spaceDatasets.foreach{ spaceDataset => + val datasetFileIds = spaceDataset.getAsOrElse[MongoDBList]("files", MongoDBList.empty) + datasetFileIds.foreach{ fileId => + collection("uploads").findOne(MongoDBObject("_id" -> fileId)) match { + case Some(file) => currentSpaceBytes += file.get("length").asInstanceOf[Long] + case None => Logger.info(s"Could not find file ${fileId} in space ${spaceId}") + } + } + } + collection("spaces.projects").update(MongoDBObject("_id" -> spaceId), $set("spaceBytes" -> currentSpaceBytes)) + } + } } diff --git a/app/util/FileUtils.scala b/app/util/FileUtils.scala index 26fc7e86d..949fa6dab 100644 --- a/app/util/FileUtils.scala +++ b/app/util/FileUtils.scala @@ -1,15 +1,9 @@ package util -import java.io.{File => JFile} -import java.net.URL -import java.util.Date - -import collection.JavaConversions._ import api.UserRequest import controllers.Utils import fileutils.FilesUtils import models._ -import org.apache.commons.codec.digest.DigestUtils import play.api.Logger import play.api.Play._ import play.api.libs.Files @@ -18,13 +12,14 @@ import play.api.mvc.MultipartFormData import play.libs.Akka import services._ +import java.net.{URL, URLEncoder} +import java.util.Date +import javax.mail.internet.MimeUtility +import scala.collection.JavaConversions._ import scala.collection.mutable import scala.concurrent.{ExecutionContext, Future} import scala.util.Try -import javax.mail.internet.MimeUtility -import java.net.URLEncoder - object FileUtils { val appConfig: AppConfigurationService = DI.injector.getInstance(classOf[AppConfigurationService]) @@ -41,6 +36,7 @@ object FileUtils { lazy val thumbnails : ThumbnailService = DI.injector.getInstance(classOf[ThumbnailService]) lazy val routing : ExtractorRoutingService = DI.injector.getInstance(classOf[ExtractorRoutingService]) lazy val sinkService : EventSinkService = DI.injector.getInstance(classOf[EventSinkService]) + lazy val spaceService : SpaceService = DI.injector.getInstance(classOf[SpaceService]) def getContentType(filename: Option[String], contentType: Option[String]): String = { @@ -596,6 +592,10 @@ object FileUtils { events.addObjectEvent(Some(user), ds.id, ds.name, EventType.ADD_FILE.toString) } datasets.addFile(ds.id, file) + val datasetSpaces = dataset.get.spaces + for (s <- datasetSpaces){ + spaceService.incrementSpaceBytes(s, file.length) + } } } } diff --git a/app/views/spaces/otherActions.scala.html b/app/views/spaces/otherActions.scala.html index 3a79be035..47f4d0c37 100644 --- a/app/views/spaces/otherActions.scala.html +++ b/app/views/spaces/otherActions.scala.html @@ -1,38 +1,61 @@ @(space: ProjectSpace)(implicit user: Option[models.User]) @import api.Permission @import play.api.i18n.Messages -
@Html(space.description.replace("\n","
"))