From dc13227068002fab29b63d0a70213dfd239c6865 Mon Sep 17 00:00:00 2001
From: Tim Yardley
Date: Wed, 18 Dec 2019 16:18:49 -0600
Subject: [PATCH 1/9] random things
---
app/views/JSConsole.scala.html | 4 +++-
app/views/ss/provider.scala.html | 2 +-
2 files changed, 4 insertions(+), 2 deletions(-)
diff --git a/app/views/JSConsole.scala.html b/app/views/JSConsole.scala.html
index 0835269f7..2ba480af1 100644
--- a/app/views/JSConsole.scala.html
+++ b/app/views/JSConsole.scala.html
@@ -1,9 +1,11 @@
@play.api.Play.maybeApplication.map(_.mode) match {
case Some(play.api.Mode.Prod) => {
}
case _ => {}
diff --git a/app/views/ss/provider.scala.html b/app/views/ss/provider.scala.html
index 303a046a8..475121e6f 100644
--- a/app/views/ss/provider.scala.html
+++ b/app/views/ss/provider.scala.html
@@ -29,7 +29,7 @@
}
@if( provider.authMethod == UserPassword ) {
-
\n" +
"Request: " + re + "
\n" +
"Error: " + ex + "
")
+ // FIXME: hardcoded email
val recipient: String = "opensource+clowder@ncsa.illinois.edu"
Mail.sendEmail(subject, request.user, recipient, body)
Ok(toJson("Send Email success"))
diff --git a/app/services/mongodb/MongoSalatPlugin.scala b/app/services/mongodb/MongoSalatPlugin.scala
index 2e994e691..b94922266 100644
--- a/app/services/mongodb/MongoSalatPlugin.scala
+++ b/app/services/mongodb/MongoSalatPlugin.scala
@@ -643,6 +643,7 @@ class MongoSalatPlugin(app: Application) extends Plugin {
if (umd.keySet().size() > 0) {
val userMD = Json.parse(com.mongodb.util.JSON.serialize(umd))
val user = User.anonymous
+ // FIXME: hardcoded URL
val userURI = "https://clowder.ncsa.illinois.edu/clowder/api/users/" + user.id
val creatorUser = UserAgent(user.id, "cat:user", MiniUser(user.id, user.fullName, user.avatarUrl.getOrElse(""), user.email), Some(new URL(userURI)))
val metadataUser = models.Metadata(UUID.generate(), attachedTo.get, contextID, contextURL, createdAt, creatorUser, userMD, version)
@@ -656,6 +657,7 @@ class MongoSalatPlugin(app: Application) extends Plugin {
case Some(tmd) => {
if (tmd.keySet().size() > 0) {
val techMD = Json.parse(com.mongodb.util.JSON.serialize(tmd))
+ // FIXME: hardcoded URL
val creatorExtractor = ExtractorAgent(id = UUID.generate(), extractorId = Some(new URL("http://clowder.ncsa.illinois.edu/extractors/migration")))
val metadataTech = models.Metadata(UUID.generate(), attachedTo.get, contextID, contextURL, createdAt, creatorExtractor, techMD, version)
metadataService.addMetadata(metadataTech)
@@ -682,6 +684,7 @@ class MongoSalatPlugin(app: Application) extends Plugin {
if (umd.keySet().size() > 0) {
val userMD = Json.parse(com.mongodb.util.JSON.serialize(umd))
val user = User.anonymous
+ // FIXME: hardcoded URL
val userURI = "https://clowder.ncsa.illinois.edu/clowder/api/users/" + user.id
val creatorUser = UserAgent(user.id, "cat:user", MiniUser(user.id, user.fullName, user.avatarUrl.getOrElse(""), user.email), Some(new URL(userURI)))
val metadataUser = models.Metadata(UUID.generate(), attachedTo.get, contextID, contextURL, createdAt, creatorUser, userMD, version)
@@ -697,12 +700,14 @@ class MongoSalatPlugin(app: Application) extends Plugin {
val tmdlist = tmd.asInstanceOf[BasicDBList]
tmdlist.foreach { x =>
val techMD = Json.parse(com.mongodb.util.JSON.serialize(x))
+ // FIXME: hardcoded URL
val creatorExtractor = ExtractorAgent(id = UUID.generate(), extractorId = Some(new URL("http://clowder.ncsa.illinois.edu/extractors/migration")))
val metadataTech = models.Metadata(UUID.generate(), attachedTo.get, contextID, contextURL, createdAt, creatorExtractor, techMD, version)
metadataService.addMetadata(metadataTech)
}
} else {
val techMD = Json.parse(com.mongodb.util.JSON.serialize(tmd))
+ // FIXME: hardcoded URL
val creatorExtractor = ExtractorAgent(id = UUID.generate(), extractorId = Some(new URL("http://clowder.ncsa.illinois.edu/extractors/migration")))
val metadataTech = models.Metadata(UUID.generate(), attachedTo.get, contextID, contextURL, createdAt, creatorExtractor, techMD, version)
metadataService.addMetadata(metadataTech)
diff --git a/app/util/Mail.scala b/app/util/Mail.scala
index 2f8dce106..7bdd9cef7 100644
--- a/app/util/Mail.scala
+++ b/app/util/Mail.scala
@@ -96,6 +96,7 @@ object Mail {
}
private def emailAddress(user: Option[User]): String = {
+ // FIXME: hardcoded email
val from = current.configuration.getString("smtp.from").getOrElse("devnull@ncsa.illinois.edu")
val name = current.configuration.getString("smtp.fromName").getOrElse("Clowder")
user match {
diff --git a/app/views/bookmarklet.scala.html b/app/views/bookmarklet.scala.html
index 8cb728b40..b6da8190a 100644
--- a/app/views/bookmarklet.scala.html
+++ b/app/views/bookmarklet.scala.html
@@ -507,6 +507,7 @@
.attr('width', '100');
var link = jQuery('')
+ // FIXME: hardcoded URL
.attr('href', 'http://browndog.ncsa.illinois.edu')
.attr('id', 'poweredby')
.css('position', 'fixed')
diff --git a/app/views/emails/footer.scala.html b/app/views/emails/footer.scala.html
index 9b1662cc9..2c0088c3c 100644
--- a/app/views/emails/footer.scala.html
+++ b/app/views/emails/footer.scala.html
@@ -7,7 +7,7 @@
@if(sender!="") {This message was sent on behalf of @sender.
}
This message was sent by @AppConfiguration.getDisplayName an
- instance of Clowder running
+ instance of Clowder running
@sys.props.getOrElse("build.version", default = "0.0.0")#@sys.props.getOrElse("build.bamboo", default = "development")
If you have any questions about this email please contact the server admins.
diff --git a/app/views/main.scala.html b/app/views/main.scala.html
index aed1830b5..7af5f5d11 100644
--- a/app/views/main.scala.html
+++ b/app/views/main.scala.html
@@ -291,7 +291,7 @@
-
Powered by Clowder
+
Powered by Clowder
(@sys.props.getOrElse("build.version", default = "0.0.0")#@sys.props.getOrElse("build.bamboo", default = "development")
branch:@sys.props.getOrElse("build.branch", default = "unknown") sha1:@sys.props.getOrElse("build.gitsha1", default = "unknown")).
From 01447f599c89e7faebc389e7a430c22ab5af9c7a Mon Sep 17 00:00:00 2001
From: Tim Yardley
Date: Wed, 18 Dec 2019 22:11:51 -0600
Subject: [PATCH 8/9] revert Global.scala change... not sure where that came
from
---
app/Global.scala | 223 +++++++++++++++++++++++++++++++++++------------
1 file changed, 165 insertions(+), 58 deletions(-)
diff --git a/app/Global.scala b/app/Global.scala
index 3f5c264b9..24adc32b8 100644
--- a/app/Global.scala
+++ b/app/Global.scala
@@ -1,71 +1,178 @@
-import models.UUID
-import play.api.mvc._
-import com.mongodb.casbah.Imports._
-import org.bson.types.ObjectId
-import play.api.libs.json._
-import play.api.data.validation.ValidationError
-
-object Binders {
-
- type ObjectId = org.bson.types.ObjectId
-
- /**
- * QueryString binder for ObjectId
- */
- implicit def objectIdQueryStringBindable = new QueryStringBindable[ObjectId] {
- def bind(key: String, params: Map[String, Seq[String]]) = params.get(key).flatMap(_.headOption).map { value =>
- if (ObjectId.isValid(value))
- Right(new ObjectId(value))
- else
- Left("Cannot parse parameter " + key + " as ObjectId")
+import java.io.{PrintWriter, StringWriter}
+import java.time.Duration
+import java.time.ZonedDateTime
+import java.time.temporal.ChronoUnit
+
+import play.api.{Application, GlobalSettings}
+import play.api.Logger
+import play.filters.gzip.GzipFilter
+import play.libs.Akka
+import securesocial.core.SecureSocial
+import services.{AppConfiguration, AppConfigurationService, CollectionService, DI, DatasetService, FileService, SpaceService, UserService}
+
+import scala.concurrent.Future
+import scala.concurrent.duration._
+import play.api.libs.concurrent.Execution.Implicits._
+import models._
+import java.util.{Calendar, Date}
+
+import play.api.mvc.{RequestHeader, WithFilters}
+import play.api.mvc.Results._
+import akka.actor.Cancellable
+import filters.CORSFilter
+import julienrf.play.jsonp.Jsonp
+import play.Play
+import play.api.Play.configuration
+import play.api.libs.json.Json._
+
+/**
+ * Configure application. Ensure mongo indexes if mongo plugin is enabled.
+ *
+ * @author Luigi Marini
+ */
+object Global extends WithFilters(new GzipFilter(), new Jsonp(), CORSFilter()) with GlobalSettings {
+ var extractorTimer: Cancellable = null
+ var jobTimer: Cancellable = null
+ var archivalTimer: Cancellable = null
+
+
+ override def onStart(app: Application) {
+ val appConfig: AppConfigurationService = DI.injector.getInstance(classOf[AppConfigurationService])
+ val files: FileService = DI.injector.getInstance(classOf[FileService])
+
+ ServerStartTime.startTime = Calendar.getInstance().getTime
+ Logger.debug("\n----Server Start Time----" + ServerStartTime.startTime + "\n \n")
+
+ val users: UserService = DI.injector.getInstance(classOf[UserService])
+
+ // set the default ToS version
+ AppConfiguration.setDefaultTermsOfServicesVersion()
+
+ // add all new admins
+ users.updateAdmins()
+
+ // create default roles
+ if (users.listRoles().isEmpty) {
+ Logger.debug("Ensuring roles exist")
+ users.updateRole(Role.Admin)
+ users.updateRole(Role.Editor)
+ users.updateRole(Role.Viewer)
}
- def unbind(key: String, value: ObjectId) = key + "=" + value.toString
- }
- /**
- * Path binder for ObjectId.
- */
- implicit def objectIdPathBindable = new PathBindable[ObjectId] {
- def bind(key: String, value: String) = {
- if (ObjectId.isValid(value))
- Right(new ObjectId(value))
- else
- Left("Cannot parse parameter " + key + " as ObjectId")
+ // set default metadata definitions
+ MetadataDefinition.registerDefaultDefinitions()
+
+ val archiveEnabled = Play.application.configuration.getBoolean("archiveEnabled", false)
+ if (archiveEnabled && archivalTimer == null) {
+ val archiveDebug = Play.application.configuration.getBoolean("archiveDebug", false)
+ val interval = if (archiveDebug) { 5 minutes } else { 1 day }
+
+ // Determine time until next midnight
+ val now = ZonedDateTime.now
+ val midnight = now.truncatedTo(ChronoUnit.DAYS)
+ val sinceLastMidnight = Duration.between(midnight, now).getSeconds
+ val delay = if (archiveDebug) { 10 seconds } else {
+ (Duration.ofDays(1).getSeconds - sinceLastMidnight) seconds
+ }
+
+ Logger.info("Starting archival loop - first iteration in " + delay + ", next iteration after " + interval)
+ archivalTimer = Akka.system.scheduler.schedule(delay, interval) {
+ Logger.info("Starting auto archive process...")
+ files.autoArchiveCandidateFiles()
+ }
}
- def unbind(key: String, value: ObjectId) = value.toString
+
+ if (extractorTimer == null) {
+ extractorTimer = Akka.system().scheduler.schedule(0 minutes, 5 minutes) {
+ ExtractionInfoSetUp.updateExtractorsInfo()
+ }
+ }
+
+ if (jobTimer == null) {
+ jobTimer = Akka.system().scheduler.schedule(0 minutes, 1 minutes) {
+ JobsScheduler.runScheduledJobs()
+ }
+ }
+
+ // Get database counts from appConfig; generate them if unavailable or user count = 0
+ appConfig.getProperty[Long]("countof.bytes") match {
+ case Some(filesBytes) =>
+ Logger.info("Byte count found in appConfig; skipping database counting")
+ case None => {
+ // Reset byte count to zero before incrementing
+ appConfig.resetCount('bytes)
+
+ Logger.info("Byte count not found in appConfig; scheduling database counting in 10s...")
+ Akka.system().scheduler.scheduleOnce(10 seconds) {
+ Logger.debug("Initializing appConfig byte count...")
+ val files: FileService = DI.injector.getInstance(classOf[FileService])
+
+ // Store the byte count in appConfig so it can be fetched quickly later
+ appConfig.incrementCount('bytes, files.bytes())
+ Logger.info("Initialized appConfig byte count")
+ }
+ }
+ }
+
+ Logger.info("Application has started")
}
- /**
- * Convert a ObjectId to a Javascript String
- */
- implicit def objectIdJavascriptLitteral = new JavascriptLitteral[ObjectId] {
- def to(value: ObjectId) = value.toString
+ override def onStop(app: Application) {
+ extractorTimer.cancel()
+ jobTimer.cancel()
+ Logger.info("Application shutdown")
+ }
+
+ private lazy val injector = services.DI.injector
+ private lazy val users: UserService = DI.injector.getInstance(classOf[UserService])
+
+ /** Used for dynamic controller dispatcher **/
+ override def getControllerInstance[A](clazz: Class[A]) = {
+ injector.getInstance(clazz)
+ }
+
+ override def onError(request: RequestHeader, ex: Throwable) = {
+ val sw = new StringWriter()
+ val pw = new PrintWriter(sw)
+ ex.printStackTrace(pw)
+
+ if (request.path.contains("/api/")) {
+ Future(InternalServerError(toJson(Map("status" -> "error",
+ "request" -> request.toString(),
+ "exception" -> sw.toString.replace("\n", "\\n")))))
+ } else {
+ implicit val user = SecureSocial.currentUser(request) match{
+ case Some(identity) => users.findByIdentity(identity)
+ case None => None
+ }
+ Future(InternalServerError(views.html.errorPage(request, sw.toString)(user)))
+ }
}
- /**
- * UUID path binder
- */
- implicit def uuidPathBinder(implicit stringBinder: PathBindable[String]) = new PathBindable[UUID] {
- override def bind(key: String, value: String): Either[String, UUID] = {
- if (UUID.isValid(value))
- Right(UUID(value))
- else
- Left(s"Cannot parse parameter $key")
+ override def onHandlerNotFound(request: RequestHeader) = {
+ if (request.path.contains("/api/")) {
+ Future(NotFound(toJson(Map("status" -> "not found",
+ "request" -> request.toString()))))
+ } else {
+ implicit val user = SecureSocial.currentUser(request) match {
+ case Some(identity) => users.findByIdentity(identity)
+ case None => None
+ }
+ Future(NotFound(views.html.errorPage(request, "Not found")(user)))
}
- override def unbind(key: String, uuid: UUID): String = uuid.stringify
}
- /**
- * UUID query binder
- */
- implicit def uuidQueryBinder(implicit stringBinder: QueryStringBindable[String]) = new QueryStringBindable[UUID] {
- override def bind(key: String, params: Map[String, Seq[String]]): Option[Either[String, UUID]] = {
- val value = params(key)(0)
- if (UUID.isValid(value))
- Some(Right(UUID(value)))
- else
- Some(Left(s"Cannot parse parameter $key"))
+ override def onBadRequest(request: RequestHeader, error: String) = {
+ if (request.path.contains("/api/")) {
+ Future(BadRequest(toJson(Map("status" -> ("bad request"),
+ "message" -> error,
+ "request" -> request.toString()))))
+ } else {
+ implicit val user = SecureSocial.currentUser(request) match {
+ case Some(identity) => users.findByIdentity(identity)
+ case None => None
+ }
+ Future(BadRequest(views.html.errorPage(request, error)(user)))
}
- override def unbind(key: String, uuid: UUID): String = uuid.stringify
}
}
From 52b68d2a8c7b2e55308dc422da9f09a3a8501944 Mon Sep 17 00:00:00 2001
From: Tim Yardley
Date: Thu, 9 Jan 2020 10:28:04 -0600
Subject: [PATCH 9/9] cleanup spurious beta character
---
app/services/ToolManagerPlugin.scala | 2 +-
app/services/mongodb/MongoDBFileService.scala | 2 +-
2 files changed, 2 insertions(+), 2 deletions(-)
diff --git a/app/services/ToolManagerPlugin.scala b/app/services/ToolManagerPlugin.scala
index 4b1c66bc2..b81e9b0df 100644
--- a/app/services/ToolManagerPlugin.scala
+++ b/app/services/ToolManagerPlugin.scala
@@ -310,7 +310,7 @@ class ToolManagerPlugin(application: Application) extends Plugin {
val apipath = play.Play.application().configuration().getString("toolmanagerURI") + "/instances/" + instance.toolPath
val statusRequest: Future[Response] = url(apipath).put(Json.obj(
- "dataset" -> (dsURL.replace("/datasets", "/api/datasets") + ß"/download"),
+ "dataset" -> (dsURL.replace("/datasets", "/api/datasets") + "/download"),
"key" -> play.Play.application().configuration().getString("commKey"),
"id" -> instance.externalId.toString,
"uploaderId" -> oId,
diff --git a/app/services/mongodb/MongoDBFileService.scala b/app/services/mongodb/MongoDBFileService.scala
index a68ada9b4..50d9c5a29 100644
--- a/app/services/mongodb/MongoDBFileService.scala
+++ b/app/services/mongodb/MongoDBFileService.scala
@@ -918,7 +918,7 @@ class MongoDBFileService @Inject() (
def searchUserMetadataFormulateQuery(requestedMetadataQuery: Any): List[File] = {
Logger.debug("top: " + requestedMetadataQuery.asInstanceOf[java.util.LinkedHashMap[String,Any]].toString() )
var theQuery = searchMetadataFormulateQuery(requestedMetadataQuery.asInstanceOf[java.util.LinkedHashMap[String,Any]], "userMetadata")
- Logger.debug("thequery: " + ßtheQuery.toString)
+ Logger.debug("thequery: " + theQuery.toString)
FileDAO.find(theQuery).toList
}