MongoDBFileService.scala

package services.mongodb

import play.api.mvc.Request
import services._
import models._
import com.mongodb.casbah.commons.{Imports, MongoDBObject}

import java.text.SimpleDateFormat
import _root_.util.{License, Parsers, SearchUtils}

import scala.collection.mutable.ListBuffer
import Transformation.LidoToCidocConvertion

import java.util.{ArrayList, Calendar, Date}
import java.io._
import org.apache.commons.io.FileUtils
import org.json.JSONObject
import play.api.libs.json.{JsValue, Json}
import com.mongodb.util.JSON

import java.nio.file.{FileSystems, Files}
import java.nio.file.attribute.BasicFileAttributes
import java.time.Instant
import collection.JavaConverters._
import scala.collection.JavaConversions._
import javax.inject.{Inject, Singleton}
import com.mongodb.casbah.WriteConcern
import play.api.Logger

import scala.util.parsing.json.JSONArray
import play.api.libs.json.JsArray
import models.File
import play.api.libs.json.JsObject
import com.novus.salat.dao.{ModelCompanion, SalatDAO}
import MongoContext.context
import play.api.Play._
import com.mongodb.casbah.Imports._
import models.FileStatus.FileStatus
import org.bson.types.ObjectId

import java.time.temporal.ChronoUnit
import scala.concurrent.duration.FiniteDuration


/**
 * Use mongo for both metadata and blobs.
 *
 *
 */
@Singleton
class MongoDBFileService @Inject() (
  datasets: DatasetService,
  collections: CollectionService,
  spaces: SpaceService,
  sections: SectionService,
  comments: CommentService,
  previews: PreviewService,
  thumbnails: ThumbnailService,
  threeD: ThreeDService,
  sparql: RdfSPARQLService,
  storage: ByteStorageService,
  userService: UserService,
  folders: FolderService,
  metadatas: MetadataService,
  events: EventService,
  routing: ExtractorRoutingService,
  appConfig: AppConfigurationService,
  esqueue: ElasticsearchQueue) extends FileService {

  object MustBreak extends Exception {}

  /**
   * Count all files
   */
  def count(): Long = {
    FileDAO.count(MongoDBObject())
  }

  def statusCount(): Map[FileStatus, Long] = {
    FileStatus.values.map(x =>
      (x, FileDAO.dao.count(MongoDBObject("status" -> x.toString)))
    ).toMap
  }

  def bytes(): Long = {
    FileDAO.dao.find(MongoDBObject()).map(_.length).sum
  }

  def save(file: File): Unit = {
    FileDAO.save(file, WriteConcern.Safe)
  }

  /**
   * List all files.
   */
  def listFiles(): List[File] = {
    (for (file <- FileDAO.find(MongoDBObject())) yield file).toList
  }

  /**
   * List all files in the system that are not intermediate result files generated by the extractors.
   */
  def listFilesNotIntermediate(): List[File] = {
    (for (file <- FileDAO.find("isIntermediate" $ne true)) yield file).toList
  }

  /**
   * List files after a specified date.
   */
  def listFilesAfter(date: String, limit: Int): List[File] = {
    val order = MongoDBObject("uploadDate" -> -1)
    if (date == "") {
      FileDAO.find("isIntermediate" $ne true).sort(order).limit(limit).toList
    } else {
      val sinceDate = new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss.SSS").parse(date)
      Logger.debug("After " + sinceDate)
      FileDAO.find($and("isIntermediate" $ne true, "uploadDate" $lt sinceDate)).sort(order).limit(limit).toList
    }
  }

  /**
   * List files before a specified date.
   */
  def listFilesBefore(date: String, limit: Int): List[File] = {
    var order = MongoDBObject("uploadDate" -> -1)
    if (date == "") {
      FileDAO.find("isIntermediate" $ne true).sort(order).limit(limit).toList
    } else {
      order = MongoDBObject("uploadDate" -> 1)
      val sinceDate = new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss.SSS").parse(date)
      Logger.debug("Before " + sinceDate)
      FileDAO.find($and("isIntermediate" $ne true, "uploadDate" $gt sinceDate)).sort(order).limit(limit).toList.reverse
    }
  }

  /**
   * List files specific to a user after a specified date.
   */
  def listUserFilesAfter(date: String, limit: Int, email: String): List[File] = {
    val order = MongoDBObject("uploadDate"-> -1 )
    if (date == "") {
      FileDAO.find(("isIntermediate" $ne true) ++ ("author.email" $eq email)).sort(order).limit(limit).toList
    } else {
      val sinceDate = new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss").parse(date)
      FileDAO.find(("isIntermediate" $ne true) ++ ("uploadDate" $lt sinceDate) ++ ("author.email" -> email))
        .sort(order).limit(limit).toList
    }
  }

  /**
   * List files specific to a user before a specified date.
   */
  def listUserFilesBefore(date: String, limit: Int, email: String): List[File] = {
    var order = MongoDBObject("uploadDate"-> -1)
    if (date == "") {
      FileDAO.find(("isIntermediate" $ne true) ++ ("author.email" $eq email)).sort(order).limit(limit).toList
    } else {
      order = MongoDBObject("uploadDate"-> 1)
      val sinceDate = new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss").parse(date)
      FileDAO.find(("isIntermediate" $ne true) ++ ("uploadDate" $gt sinceDate) ++ ("author.email" $eq email))
        .sort(order).limit(limit).toList.reverse
    }
  }

  /**
    * Submit a single archival operation to the appropriate queue/extractor
    */
  def submitArchivalOperation(file: File, id:UUID, host: String, parameters: JsObject, apiKey: Option[String], user: Option[User]) = {
    val idAndFlags = ""
    val extra = Map("filename" -> file.filename,
      "parameters" -> parameters,
      "action" -> "manual-submission")
    val showPreviews = file.showPreviews
    val newFlags = if (showPreviews.equals("FileLevel"))
      idAndFlags + "+filelevelshowpreviews"
    else if (showPreviews.equals("None"))
      idAndFlags + "+nopreviews"
    else
      idAndFlags

    val originalId = if (!file.isIntermediate) {
      file.id.toString()
    } else {
      idAndFlags
    }

    var datasetId: UUID = null
    // search datasets containing this file, either directly under dataset or indirectly.
    val datasetslists: List[Dataset] = datasets.findByFileIdAllContain(id)
    // Note, we assume only at most one dataset will contain a given file.
    if (0 != datasetslists.length) {
      datasetId = datasetslists.head.id
    }
    val extractorId = play.Play.application().configuration().getString("archiveExtractorId")
    routing.submitFileManually(new UUID(originalId), file, host, extractorId, extra,
      datasetId, newFlags, apiKey, user)
    Logger.info("Sent archive request for file " + id)
  }

  /**
    * Submit all archival candidates to the appropriate queue/extractor.
    * This may be expanded to support per-space configuration in the future.
    *
    * Reads the following parameters from Clowder configuration:
    *     - archiveAutoAfterInactiveCount - timeout after which files are considered
    *         to be candidates for archival (see below)
    *     - archiveAutoAfterInactiveUnits - time unit that should be used for the timeout (see below)
    *     - archiveAutoAboveMinimumStorageSize - files below this size (in Bytes) should not be archived
    *     - clowder.rabbitmq.clowderurl - the Clowder hostname to pass to the archival extractor
    *     - commKey - the admin key to pass to the archival extractor
    *
    * Archival candidates are currently defined as follows:
    *    - file's size must be greater than `archiveAutoAboveMinimumStorageSize` Bytes
    *    - file's age must be greater than `archiveAutoAfterInactiveCount` * `archiveAutoAfterInactiveUnits`
    *         (e.g. 10 days old)
    *    - AND one of the following must be true:
    *       - file has never been downloaded (0 downloads)
    *                 OR
    *       - file has not been downloaded in the past `archiveAutoAfterInactiveCount` `archiveAutoAfterInactiveUnits`
    *
    *
    */
  def autoArchiveCandidateFiles() = {
    val timeout: Option[Long] = configuration(play.api.Play.current).getLong("archiveAutoAfterInactiveCount")
    timeout match {
      case None => Logger.info("No archival auto inactivity timeout set - skipping auto archival loop.")
      case Some(inactiveTimeout) => {
        if (inactiveTimeout == 0) {
          Logger.info("Archival auto inactivity timeout set to 0 - skipping auto archival loop.")
        } else {
          val unit = configuration(play.api.Play.current).getString("archiveAutoAfterInactiveUnits").getOrElse("days")
          val timeoutAgo = FiniteDuration(inactiveTimeout, unit)

          // Query for files that haven't been accessed for at least this many units
          val since = Instant.now().minus(timeoutAgo.length.toLong, ChronoUnit.valueOf(timeoutAgo.unit.toString)).toString + "-00:00"
          val notDownloadedWithinTimeout = ("stats.last_downloaded" $lt Parsers.fromISO8601(since)) ++ ("status" $eq FileStatus.PROCESSED.toString)

          // Include files that have never been downloaded, but make sure they are old enough
          val neverDownloaded = ("stats.downloads" $eq 0) ++ ("uploadDate" $lt Parsers.fromISO8601(since)) ++ ("status" $eq FileStatus.PROCESSED.toString)

          // TODO: How to get host / apiKey / admin internally without a request?
          val host = configuration(play.api.Play.current).getString("clowder.rabbitmq.clowderurl").getOrElse("http://localhost:9000")
          val adminApiKey = configuration(play.api.Play.current).getString("commKey")
          val adminUser = Option(userService.getAdmins.head)
          val params = JsObject(Seq.empty[(String, JsValue)]) + FileService.ARCHIVE_PARAMETER

          // Submit our queries and determine the union
          val ndFiles = FileDAO.find(neverDownloaded).toList
          val ndwtoFiles = FileDAO.find(notDownloadedWithinTimeout).toList
          val matchingFiles = ndFiles.union(ndwtoFiles)
          Logger.info("Archival candidates found: " + matchingFiles.length)

          // Exclude candidates that do not exceed our minimum file size threshold
          val minSize = configuration(play.api.Play.current).getLong("archiveAutoAboveMinimumStorageSize").getOrElse(1000000L)

          // Loop all candidate files and submit each one for archival
          for (file <- matchingFiles) {
            if (file.length > minSize) {
              Logger.debug("Submitting file " + file.id + " for archival")
              submitArchivalOperation(file, file.id, host, params, adminApiKey, adminUser)
            } else {
              Logger.debug("Skipping file " + file.id + ": below threshold (" +
                file.length.toString + " B < " + minSize.toString + " B)")
            }
          }
          Logger.info("Auto archival loop completed successfully")
        }
      }
    }
  }

  def latest(): Option[File] = {
    val results = FileDAO.find("isIntermediate" $ne true).sort(MongoDBObject("uploadDate" -> -1)).limit(1).toList
    if (results.size > 0)
      Some(results(0))
    else
      None
  }

  def latest(i: Int): List[File] = {
    FileDAO.find(MongoDBObject()).sort(MongoDBObject("uploadDate" -> -1)).limit(i).toList
  }

  def first(): Option[File] = {
    val results = FileDAO.find("isIntermediate" $ne true).sort(MongoDBObject("uploadDate" -> 1)).limit(1).toList
    if (results.size > 0)
      Some(results(0))
    else
      None
  }

  /**
   * Save blob.
   */
  def save(inputStream: InputStream, filename: String, contentLength: Long, contentType: Option[String], author: MiniUser, showPreviews: String = "DatasetLevel"): Option[File] = {
    ByteStorageService.save(inputStream, FileDAO.COLLECTION, contentLength) match {
      case Some(x) => {
        val file = File(UUID.generate(), x._1, filename, filename, author, new Date(), util.FileUtils.getContentType(filename, contentType),
          x._3, x._2, showPreviews = showPreviews, licenseData = License.fromAppConfig(), stats = new Statistics())
        FileDAO.save(file)
        Some(file)
      }
      case None => None
    }
  }

  /**
   * Get blob.
   */
  def getBytes(id: UUID): Option[(InputStream, String, String, Long)] = {
    get(id).flatMap { x =>
      ByteStorageService.load(x.loader, x.loader_id, FileDAO.COLLECTION).map((_, x.filename, x.contentType, x.length))
    }
  }

  def indexAll(idx: Option[String] = None) = {
    // Bypass Salat in case any of the file records are malformed to continue past them
    val trashedIds = datasets.getTrashedIds()
    FileDAO.dao.collection.find(MongoDBObject(), MongoDBObject("_id" -> 1)).foreach(f => {
      val fid = new UUID(f.get("_id").toString)
      if (!trashedIds.contains(fid))
        index(fid, idx)
    })
  }

  def index(id: UUID, idx: Option[String] = None) {
    try
      esqueue.queue("index_file", new ResourceRef('file, id), new ElasticsearchParameters(index=idx))
    catch {
      case except: Throwable => Logger.error(s"Error queuing file ${id.stringify}: ${except}")
      case _ => Logger.error(s"Error queuing file ${id.stringify}")
    }
  }

  /**
    * Directly insert a file into the db (even with a local path)
    */
  def insert(file: File): Option[String] = {
    FileDAO.insert(file).map(_.toString)
  }

  /**
   * Return a list of tags and counts found in sections
   */
  def getTags(user: Option[User]): Map[String, Long] = {
    val filter = MongoDBObject("tags" -> MongoDBObject("$not" -> MongoDBObject("$size" -> 0)))
    var tags = scala.collection.mutable.Map[String, Long]()
    FileDAO.dao.find(buildTagFilter(user) ++ filter).foreach{ x =>
      x.tags.foreach{ t =>
        tags.put(t.name, tags.get(t.name).getOrElse(0L) + 1L)
      }
    }
    tags.toMap
  }

  private def buildTagFilter(user: Option[User]): MongoDBObject = {
    if (user.isDefined && user.get.superAdminMode)
      return MongoDBObject()

    val orlist = collection.mutable.ListBuffer.empty[MongoDBObject]

    // all files where user is the author
    user.foreach{u => orlist += MongoDBObject("author._id" -> new ObjectId(u.id.stringify))}

    // Get all files in all datasets you have access to.
    val datasetsList = datasets.listUser(user)
    val foldersList = folders.findByParentDatasetIds(datasetsList.map(x => x.id))
    val fileIds = datasetsList.map(x => x.files) ++ foldersList.map(x => x.files)
    orlist += ("_id" $in fileIds.flatten.map(x => new ObjectId(x.stringify)))

    $or(orlist.map(_.asDBObject))
  }

  def modifyRDFOfMetadataChangedFiles() {
    val changedFiles = findMetadataChangedFiles()
    for (changedFile <- changedFiles) {
      modifyRDFUserMetadata(changedFile.id)
    }
  }

  def modifyRDFUserMetadata(id: UUID, mappingNumber: String = "1") = { implicit request: Request[Any] =>
    sparql.removeFileFromGraphs(id, "rdfCommunityGraphName")
    get(id) match {
      case Some(file) => {
        val theJSON = getUserMetadataJSON(id)
        val fileSep = System.getProperty("file.separator")
        val tmpDir = System.getProperty("java.io.tmpdir")
        var resultDir = tmpDir + fileSep + "clowder__rdfuploadtemporaryfiles" + fileSep + UUID.generate.stringify
        val resultDirFile = new java.io.File(resultDir)
        resultDirFile.mkdirs()

        if (!theJSON.replaceAll(" ", "").equals("{}")) {
          val xmlFile = jsonToXML(theJSON)
          new LidoToCidocConvertion(play.api.Play.configuration.getString("filesxmltordfmapping.dir_" + mappingNumber).getOrElse(""), xmlFile.getAbsolutePath(), resultDir)
          xmlFile.delete()
        }
        else {
          new java.io.File(resultDir + fileSep + "Results.rdf").createNewFile()
        }
        val resultFile = new java.io.File(resultDir + fileSep + "Results.rdf")

        //Connecting RDF metadata with the entity describing the original file
        val rootNodes = new ArrayList[String]()
        val rootNodesFile = play.api.Play.configuration.getString("rootNodesFile").getOrElse("")
        Logger.debug(rootNodesFile)
        if (!rootNodesFile.equals("*")) {
          val rootNodesReader = new BufferedReader(new FileReader(new java.io.File(rootNodesFile)))
          var line = rootNodesReader.readLine()
          while (line != null) {
            Logger.debug((line == null).toString())
            rootNodes.add(line.trim())
            line = rootNodesReader.readLine()
          }
          rootNodesReader.close()
        }

        val resultFileConnected = java.io.File.createTempFile("ResultsConnected", ".rdf")

        val fileWriter = new BufferedWriter(new FileWriter(resultFileConnected))
        val fis = new FileInputStream(resultFile)
        val data = new Array[Byte](resultFile.length().asInstanceOf[Int])
        fis.read(data)
        fis.close()
        resultFile.delete()
        FileUtils.deleteDirectory(resultDirFile)
        //
        val s = new String(data, "UTF-8")
        val rdfDescriptions = s.split("<rdf:Description")
        fileWriter.write(rdfDescriptions(0))
        var i = 0
        for (i <- 1 to (rdfDescriptions.length - 1)) {
          fileWriter.write("<rdf:Description" + rdfDescriptions(i))
          if (rdfDescriptions(i).contains("<rdf:type")) {
            var isInRootNodes = false
            if (rootNodesFile.equals("*"))
              isInRootNodes = true
            else {
              var j = 0
              try {
                for (j <- 0 to (rootNodes.size() - 1)) {
                  if (rdfDescriptions(i).contains("\"" + rootNodes.get(j) + "\"")) {
                    isInRootNodes = true
                    throw MustBreak
                  }
                }
              } catch {
                case MustBreak =>
              }
            }

            if (isInRootNodes) {
              val theResource = rdfDescriptions(i).substring(rdfDescriptions(i).indexOf("\"") + 1, rdfDescriptions(i).indexOf("\"", rdfDescriptions(i).indexOf("\"") + 1))
              // TODO RK : need to make sure we know if it is https
              var connection = "<rdf:Description rdf:about=\"" + api.routes.Files.get(id).absoluteURL(false)
              connection = connection + "\"><P129_is_about xmlns=\"http://www.cidoc-crm.org/rdfs/cidoc_crm_v5.0.2.rdfs#\" rdf:resource=\"" + theResource
              connection = connection + "\"/></rdf:Description>"
              fileWriter.write(connection)
            }
          }
        }
        fileWriter.close()

        sparql.addFromFile(id, resultFileConnected, "file")
        resultFileConnected.delete()

        sparql.addFileToGraph(id, "rdfCommunityGraphName")

        setUserMetadataWasModified(id, false)
      }
      case None => {}
    }
  }

  def jsonToXML(theJSON: String): java.io.File = {

    val jsonObject = new JSONObject(theJSON)
    var xml = org.json.XML.toString(jsonObject)

    //Remove spaces from XML tags
    var currStart = xml.indexOf("<")
    var currEnd = -1
    var xmlNoSpaces = ""
    while (currStart != -1) {
      xmlNoSpaces = xmlNoSpaces + xml.substring(currEnd + 1, currStart)
      currEnd = xml.indexOf(">", currStart + 1)
      xmlNoSpaces = xmlNoSpaces + xml.substring(currStart, currEnd + 1).replaceAll(" ", "_")
      currStart = xml.indexOf("<", currEnd + 1)
    }
    xmlNoSpaces = xmlNoSpaces + xml.substring(currEnd + 1)

    val xmlFile = java.io.File.createTempFile("xml", ".xml")
    val fileWriter = new BufferedWriter(new FileWriter(xmlFile))
    fileWriter.write(xmlNoSpaces)
    fileWriter.close()

    return xmlFile
  }

  def getXMLMetadataJSON(id: UUID): String = {
    FileDAO.dao.collection.findOneByID(new ObjectId(id.stringify)) match {
      case None => "{}"
      case Some(x) => {
        x.getAs[DBObject]("xmlMetadata") match {
          case Some(y) => {
            val returnedMetadata = com.mongodb.util.JSON.serialize(x.getAs[DBObject]("xmlMetadata").get)
            returnedMetadata
          }
          case None => "{}"
        }
      }
    }
  }

  def removeTags(id: UUID, tags: List[String]) {
    Logger.debug("Removing tags in file " + id + " : " + tags)
    val file = get(id).get
    val existingTags = file.tags.map(_.name)
    Logger.debug("existingTags after user and extractor filtering: " + existingTags.toString)
    // Only remove existing tags.
    tags.intersect(existingTags).map {
      tag =>
        FileDAO.update(MongoDBObject("_id" -> new ObjectId(id.stringify)), $pull("tags" -> MongoDBObject("name" -> tag)), false, false, WriteConcern.Safe)
    }
  }

  def addMetadata(fileId: UUID, metadata: JsValue) {
    val doc = JSON.parse(Json.stringify(metadata)).asInstanceOf[DBObject]
    FileDAO.update(MongoDBObject("_id" -> new ObjectId(fileId.stringify)), $addToSet("metadata" -> doc), false, false, WriteConcern.Safe)
  }

  def updateMetadata(fileId: UUID, metadata: JsValue, extractor_id: String) {
    val doc = JSON.parse(Json.stringify(metadata)).asInstanceOf[DBObject]
    FileDAO.findOneById(new ObjectId(fileId.stringify)) match {
      case None => None
      case Some(file) => {
        FileDAO.update(MongoDBObject("_id" -> new ObjectId(fileId.stringify), "metadata.extractor_id" -> extractor_id), $set("metadata.$" -> doc), false, false, WriteConcern.Safe)
      }
    }
  }

  def get(id: UUID): Option[File] = {
    FileDAO.findOneById(new ObjectId(id.stringify)) match {
      case Some(file) => {
        val previewsByFile = previews.findByFileId(file.id)
        val sectionsByFile = sections.findByFileId(file.id)
        val sectionsWithPreviews = sectionsByFile.map { s =>
          val p = PreviewDAO.findOne(MongoDBObject("section_id"->s.id))
          s.copy(preview = p)
        }
        Some(file.copy(sections = sectionsWithPreviews, previews = previewsByFile))
      }
      case None => None
    }
  }

  def get(ids: List[UUID]): DBResult[File] = {
    if (ids.length == 0) return DBResult(List.empty, List.empty)

    val query = MongoDBObject("_id" -> MongoDBObject("$in" -> ids.map(id => new ObjectId(id.stringify))))
    val found = FileDAO.find(query).toList
    val notFound = ids.diff(found.map(_.id))

    if (notFound.length > 0)
      Logger.error("Not all file IDs found for bulk get request")
    return DBResult(found, notFound)
  }

  override def setStatus(id: UUID, status: FileStatus): Unit = {
    FileDAO.dao.update(MongoDBObject("_id" -> new ObjectId(id.toString())), $set("status" -> status.toString))
  }

  def listOutsideDataset(dataset_id: UUID): List[File] = {
    datasets.get(dataset_id) match{
      case Some(dataset) => {
        val list = for (file <- FileDAO.findAll(); if(!isInDataset(file,dataset) && !file.isIntermediate)) yield file
        return list.toList
      }
      case None =>{
        return FileDAO.findAll.toList
      }
    }
  }

  def isInDataset(file: File, dataset: Dataset): Boolean = {
    for(dsFile <- dataset.files){
      if(dsFile == file.id)
        return true
    }
    return false
  }

  //Not used yet
  def getMetadata(id: UUID): scala.collection.immutable.Map[String,Any] = {
    FileDAO.dao.collection.findOneByID(new ObjectId(id.stringify)) match {
      case None => new scala.collection.immutable.HashMap[String,Any]
      case Some(x) => {
        val returnedMetadata = x.getAs[DBObject]("metadata").get.toMap.asScala.asInstanceOf[scala.collection.mutable.Map[String,Any]].toMap
        returnedMetadata
      }
    }
  }

  def getUserMetadata(id: UUID): scala.collection.mutable.Map[String,Any] = {
    FileDAO.dao.collection.findOneByID(new ObjectId(id.stringify)) match {
      case None => new scala.collection.mutable.HashMap[String,Any]
      case Some(x) => {
        x.getAs[DBObject]("userMetadata") match{
          case Some(y)=>{
            val returnedMetadata = x.getAs[DBObject]("userMetadata").get.toMap.asScala.asInstanceOf[scala.collection.mutable.Map[String,Any]]
            returnedMetadata
          }
          case None => new scala.collection.mutable.HashMap[String,Any]
        }
      }
    }
  }

  def getUserMetadataJSON(id: UUID): String = {
    FileDAO.dao.collection.findOneByID(new ObjectId(id.stringify)) match {
      case None => "{}"
      case Some(x) => {
        x.getAs[DBObject]("userMetadata") match{
          case Some(y)=>{
            val returnedMetadata = com.mongodb.util.JSON.serialize(x.getAs[DBObject]("userMetadata").get)
            returnedMetadata
          }
          case None => "{}"
        }
      }
    }
  }

  def getTechnicalMetadataJSON(id: UUID): String = {
    FileDAO.dao.collection.findOneByID(new ObjectId(id.stringify)) match {
      case None => "{}"
      case Some(x) => {
        x.getAs[DBObject]("metadata") match{
          case Some(y)=>{
            val returnedMetadata = com.mongodb.util.JSON.serialize(x.getAs[DBObject]("metadata").get)
            returnedMetadata
          }
          case None => "{}"
        }
      }
    }
  }

  /** Change the metadataCount field for a file */
  def incrementMetadataCount(id: UUID, count: Long) = {
    FileDAO.update(MongoDBObject("_id" -> new ObjectId(id.stringify)), $inc("metadataCount" -> count), false, false, WriteConcern.Safe)
  }
  
  /**
   *  Add versus descriptors to the Versus.descriptors collection associated to a file
   *
   */
 def addVersusMetadata(id:UUID,json:JsValue){
    val doc = JSON.parse(Json.stringify(json)).asInstanceOf[DBObject].toMap
              .asScala.asInstanceOf[scala.collection.mutable.Map[String,Any]].toMap
       VersusDAO.insert(new Versus(id,doc),WriteConcern.Safe)
       Logger.debug("--Added versus descriptors in json format received from versus to the metadata field --")
  }
 
/**
 * Get Versus descriptors as Json Array for a file
 */
  def getVersusMetadata(id: UUID): Option[JsValue] = {
    val versusDescriptors = VersusDAO.find(MongoDBObject("fileId" -> new ObjectId(id.stringify)))
    var vdArray = new JsArray()
    for (vd <- versusDescriptors) {
      var x = com.mongodb.util.JSON.serialize(vd.asInstanceOf[Versus].descriptors("versus_descriptors"))
      vdArray = vdArray :+ Json.parse(x)
      Logger.debug("array=" + vdArray.toString)
    }
    Some(vdArray)
  } 
   /*convert list of JsObject to JsArray*/
  def getJsonArray(list: List[JsObject]): JsArray = {
    list.foldLeft(JsArray())((acc, x) => acc ++ Json.arr(x))
  }

  def addUserMetadata(id: UUID, json: String) {
    Logger.debug("Adding/modifying user metadata to file " + id + " : " + json)
    val md = com.mongodb.util.JSON.parse(json).asInstanceOf[DBObject]
    FileDAO.update(MongoDBObject("_id" -> new ObjectId(id.stringify)), $set("userMetadata" -> md), false, false, WriteConcern.Safe)
  }

  def addXMLMetadata(id: UUID, json: String) {
    Logger.debug("Adding/modifying XML file metadata to file " + id + " : " + json)
    val md = com.mongodb.util.JSON.parse(json).asInstanceOf[DBObject]
    FileDAO.update(MongoDBObject("_id" -> new ObjectId(id.stringify)), $set("xmlMetadata" -> md), false, false, WriteConcern.Safe)
  }

  def findByTag(tag: String, user: Option[User]): List[File] = {
    FileDAO.find(buildTagFilter(user) ++ MongoDBObject("tags.name" -> tag)).toList
  }

  def findByTag(tag: String, start: String, limit: Integer, reverse: Boolean, user: Option[User]): List[File] = {

    var filter = if (start == "") {
      MongoDBObject("tags.name" -> tag)
    } else {
      if (reverse) {
        MongoDBObject("tags.name" -> tag) ++ ("uploadDate" $gte Parsers.fromISO8601(start))
      } else {
        MongoDBObject("tags.name" -> tag) ++ ("uploadDate" $lte Parsers.fromISO8601(start))
      }
    }
    if(!(configuration(play.api.Play.current).getString("permissions").getOrElse("public") == "public")) {
      filter = buildTagFilter(user) ++ filter
    }
    val order = if (reverse) {
      MongoDBObject("uploadDate" -> 1, "filename" -> 1)
    } else {
      MongoDBObject("uploadDate" -> -1, "filename" -> 1)
    }
    FileDAO.dao.find(filter).sort(order).limit(limit).toList
  }

  def findIntermediates(): List[File] = {
    FileDAO.find(MongoDBObject("isIntermediate" -> true)).toList
  }
  
  /**
   * Implementation of updateLicenseing defined in services/FileService.scala.
   */
  def updateLicense(id: UUID, licenseType: String, rightsHolder: String, licenseText: String, licenseUrl: String,
    allowDownload: String) {
      val licenseData = models.LicenseData(m_licenseType = licenseType, m_rightsHolder = rightsHolder,
        m_licenseText = licenseText, m_licenseUrl = licenseUrl, m_allowDownload = allowDownload.toBoolean)
      val result = FileDAO.update(MongoDBObject("_id" -> new ObjectId(id.stringify)), 
          $set("licenseData" -> LicenseData.toDBObject(licenseData)), 
          false, false, WriteConcern.Safe);      
  }

  // ---------- Tags related code starts ------------------
  // Input validation is done in api.Files, so no need to check again.
  def addTags(id: UUID, userIdStr: Option[String], eid: Option[String], tags: List[String]) : List[Tag] = {
    Logger.debug("Adding tags to file " + id + " : " + tags)

    var tagsAdded : ListBuffer[Tag] = ListBuffer.empty[Tag]
    val file = get(id).get
    val existingTags = file.tags.filter(x => userIdStr == x.userId && eid == x.extractor_id).map(_.name)
    val createdDate = new Date
    val maxTagLength = play.api.Play.configuration.getInt("clowder.tagLength").getOrElse(100)
    tags.foreach(tag => {
      val shortTag = if (tag.length > maxTagLength) {
        Logger.error("Tag is truncated to " + maxTagLength + " chars : " + tag)
        tag.substring(0, maxTagLength)
      } else {
        tag
      }
      // Only add tags with new values.
      if (!existingTags.contains(shortTag)) {
        val tagObj = models.Tag(name = shortTag, userId = userIdStr, extractor_id = eid, created = createdDate)
        tagsAdded += tagObj
        FileDAO.update(MongoDBObject("_id" -> new ObjectId(id.stringify)), $addToSet("tags" -> Tag.toDBObject(tagObj)), false, false, WriteConcern.Safe)
      }
    })
    tagsAdded.toList
  }

  def removeAllTags(id: UUID) {
    FileDAO.update(MongoDBObject("_id" -> new ObjectId(id.stringify)), $set("tags" -> List()), false, false,
      WriteConcern.Safe)
  }
  // ---------- Tags related code ends ------------------

  def comment(id: UUID, comment: Comment) {
    FileDAO.update(MongoDBObject("_id" -> new ObjectId(id.stringify)), $addToSet("comments" -> Comment.toDBObject(comment)),
      false, false, WriteConcern.Safe)
  }
  
  def setIntermediate(id: UUID){
    FileDAO.update(MongoDBObject("_id" -> new ObjectId(id.stringify)), $set("isIntermediate" -> Some(true)), false, false,
      WriteConcern.Safe)
  }

  def renameFile(id: UUID, newName: String){
    events.updateObjectName(id, newName)
        get(id) match{
      case Some(file) => {
        if(file.originalname.length >0) {
      
    FileDAO.update(MongoDBObject("_id" -> new ObjectId(id.stringify)), $set("filename" -> newName), false, false,
      WriteConcern.Safe)
        } else {
    FileDAO.update(MongoDBObject("_id" -> new ObjectId(id.stringify)), $set("filename" -> newName, "originalname" -> file.filename ), false, false,
      WriteConcern.Safe)
          
        }
      }
    }
  }

  def setContentType(id: UUID, newType: String){
    FileDAO.update(MongoDBObject("_id" -> new ObjectId(id.stringify)), $set("contentType" -> newType), false, false,
      WriteConcern.Safe)
  }

  def setUserMetadataWasModified(id: UUID, wasModified: Boolean){
    FileDAO.update(MongoDBObject("_id" -> new ObjectId(id.stringify)), $set("userMetadataWasModified" -> Some(wasModified)),
      false, false, WriteConcern.Safe)
  }

  def removeFile(id: UUID, host: String, apiKey: Option[String], user: Option[User]) : Boolean = {
    get(id) match{
      case Some(file) => {
          if(!file.isIntermediate){
            val fileDatasets = datasets.findByFileIdDirectlyContain(file.id)
            for(fileDataset <- fileDatasets){
              datasets.removeFile(fileDataset.id, id)
              if(!file.xmlMetadata.isEmpty){
                datasets.index(fileDataset.id)
              }

              if(!file.thumbnail_id.isEmpty && !fileDataset.thumbnail_id.isEmpty){
                if(file.thumbnail_id.get.equals(fileDataset.thumbnail_id.get)){
                  datasets.newThumbnail(fileDataset.id)
                  collections.get(fileDataset.collections).found.foreach(collection => {
                    if(!collection.thumbnail_id.isEmpty){
                      if(collection.thumbnail_id.get.equals(fileDataset.thumbnail_id.get)){
                        collections.createThumbnail(collection.id)
                      }
                    }
                  })
                }
              }

            }

            val fileFolders = folders.findByFileId(file.id)
            for(fileFolder <- fileFolders) {
              folders.removeFile(fileFolder.id, file.id)
            }
            for(section <- sections.findByFileId(file.id)){
              sections.removeSection(section)
            }
            for(comment <- comments.findCommentsByFileId(id)){
              comments.removeComment(comment)
            }
            for(texture <- threeD.findTexturesByFileId(file.id)){
              ThreeDTextureDAO.removeById(new ObjectId(texture.id.stringify))
            }
            for (follower <- file.followers) {
              userService.unfollowFile(follower, id)
            }
          }


          // delete the actual file
          val fileSize = if(isLastPointingToLoader(file.loader, file.loader_id)) {
            for(preview <- previews.findByFileId(file.id)){
              previews.removePreview(preview)
            }
            if(!file.thumbnail_id.isEmpty)
              thumbnails.remove(UUID(file.thumbnail_id.get))
            ByteStorageService.delete(file.loader, file.loader_id, FileDAO.COLLECTION)
            file.length
          } else {
            0
          }

          import UUIDConversions._
          FileDAO.removeById(file.id)
          appConfig.incrementCount('files, -1)
          appConfig.incrementCount('bytes, -1 * fileSize)
          current.plugin[ElasticsearchPlugin].foreach {
            _.delete(id.stringify)
          }

          // finally remove metadata - if done before file is deleted, document metadataCounts won't match
          metadatas.removeMetadataByAttachTo(ResourceRef(ResourceRef.file, id), host, apiKey, user)
          true
      }
      case None => {
        Logger.debug("File not found")
        false
      }
    }
  }

  def isLastPointingToLoader(loader: String, loader_id: String): Boolean = {
    val result = FileDAO.find(MongoDBObject("loader" -> loader, "loader_id" -> loader_id))
    result.size == 1
  }
  def removeTemporaries(){
    val cal = Calendar.getInstance()
    val timeDiff = play.Play.application().configuration().getInt("rdfTempCleanup.removeAfter")
    cal.add(Calendar.MINUTE, -timeDiff)
    val oldDate = cal.getTime()

    val tmpDir = System.getProperty("java.io.tmpdir")
    val filesep = System.getProperty("file.separator")
    val rdfTmpDir = new java.io.File(tmpDir + filesep + "clowder__rdfdumptemporaryfiles")
    if(!rdfTmpDir.exists()){
      rdfTmpDir.mkdir()
    }

    val listOfFiles = rdfTmpDir.listFiles()
    for(currFileDir <- listOfFiles){
      val currFile = currFileDir.listFiles()(0)
      val attrs = Files.readAttributes(FileSystems.getDefault().getPath(currFile.getAbsolutePath()),  classOf[BasicFileAttributes])
      val timeCreated = new Date(attrs.creationTime().toMillis())
      if(timeCreated.compareTo(oldDate) < 0){
        currFile.delete()
        currFileDir.delete()
      }
    }
  }

  def findMetadataChangedFiles(): List[File] = {
    FileDAO.find(MongoDBObject("userMetadataWasModified" -> true)).toList
  }

  def searchAllMetadataFormulateQuery(requestedMetadataQuery: Any): List[File] = {
    Logger.debug("top: "+ requestedMetadataQuery.asInstanceOf[java.util.LinkedHashMap[String,Any]].toString()  )
    var theQuery =  searchMetadataFormulateQuery(requestedMetadataQuery.asInstanceOf[java.util.LinkedHashMap[String,Any]], "all")
    Logger.debug("thequery: "+theQuery.toString)
    FileDAO.find(theQuery).toList
  }

  def searchUserMetadataFormulateQuery(requestedMetadataQuery: Any): List[File] = {
    Logger.debug("top: "+ requestedMetadataQuery.asInstanceOf[java.util.LinkedHashMap[String,Any]].toString()  )
    var theQuery =  searchMetadataFormulateQuery(requestedMetadataQuery.asInstanceOf[java.util.LinkedHashMap[String,Any]], "userMetadata")
    Logger.debug("thequery: "+theQuery.toString)
    FileDAO.find(theQuery).toList
  }

  def searchMetadataFormulateQuery(requestedMap: java.util.LinkedHashMap[String,Any], root: String): MongoDBObject = {
    Logger.debug("req: "+ requestedMap)
    var queryMap = MongoDBList()
    var builder = MongoDBList()
    var orFound = false
    for((reqKey, reqValue) <- requestedMap){
      val keyTrimmed = reqKey.replaceAll("__[0-9]+$","")

      if(keyTrimmed.equals("OR")){
        queryMap.add(MongoDBObject("$and" ->  builder))
        builder = MongoDBList()
        orFound = true
      }
      else{
        var actualKey = keyTrimmed
        if(keyTrimmed.endsWith("__not")){
          actualKey = actualKey.substring(0, actualKey.length()-5)
        }

        if(!root.equals("all")){

          if(!root.equals(""))
            actualKey = root + "." + actualKey

          if(reqValue.isInstanceOf[String]){
            val currValue = reqValue.asInstanceOf[String]
                        
            if(keyTrimmed.endsWith("__not")){
              if(currValue.contains(" IGNORE CASE") || currValue.contains(" ANYWHERE")){
                var realValue = currValue.replace(" IGNORE CASE", "").replace(" ANYWHERE", "");                
                if(!currValue.contains(" ANYWHERE")){
                  realValue = "^"+realValue+"$";
                }
                if(currValue.contains(" IGNORE CASE")){
                  realValue = "(?i)"+realValue;
                }
                builder += MongoDBObject(actualKey -> MongoDBObject("$not" ->  realValue.r))
              }
              else{
                builder += MongoDBObject(actualKey -> MongoDBObject("$ne" ->  currValue))
              }
            }
            else{
              if(currValue.contains(" IGNORE CASE") || currValue.contains(" ANYWHERE")){
                var realValue = currValue.replace(" IGNORE CASE", "").replace(" ANYWHERE", "");                
                if(!currValue.contains(" ANYWHERE")){
                  realValue = "^"+realValue+"$";
                }
                if(currValue.contains(" IGNORE CASE")){
                  realValue = "(?i)"+realValue;
                }
                builder += MongoDBObject(actualKey -> realValue.r)
              }
              else{
                builder += MongoDBObject(actualKey -> currValue)
              }
            }
          }else{
            //recursive
            if(root.equals("userMetadata")){
              val currValue =  searchMetadataFormulateQuery(reqValue.asInstanceOf[java.util.LinkedHashMap[String,Any]], "")
              val elemMatch = actualKey $elemMatch currValue
              builder.add(elemMatch)
            }
            else{
              val currValue =  searchMetadataFormulateQuery(reqValue.asInstanceOf[java.util.LinkedHashMap[String,Any]], actualKey)
              builder += currValue
            }
          }
        } else {
          var objectForEach = MongoDBList()
          val allRoots = Map(1 -> "userMetadata", 2 -> "metadata", 3 -> "xmlMetadata")
          allRoots.keys.foreach{ i =>
            var tempActualKey = allRoots(i) + "." + actualKey

            if(reqValue.isInstanceOf[String]){
              val currValue = reqValue.asInstanceOf[String]
              if(keyTrimmed.endsWith("__not")){
                if(currValue.contains(" IGNORE CASE") || currValue.contains(" ANYWHERE")){
	                var realValue = currValue.replace(" IGNORE CASE", "").replace(" ANYWHERE", "");                
	                if(!currValue.contains(" ANYWHERE")){
	                  realValue = "^"+realValue+"$";
	                }
	                if(currValue.contains(" IGNORE CASE")){
	                  realValue = "(?i)"+realValue;
	                }
	                objectForEach += MongoDBObject(tempActualKey -> MongoDBObject("$not" ->  realValue.r))
                }
                else{
                	objectForEach += MongoDBObject(tempActualKey -> MongoDBObject("$ne" ->  currValue))
                }
              }
              else{
                if(currValue.contains(" IGNORE CASE") || currValue.contains(" ANYWHERE")){
	                var realValue = currValue.replace(" IGNORE CASE", "").replace(" ANYWHERE", "");                
	                if(!currValue.contains(" ANYWHERE")){
	                  realValue = "^"+realValue+"$";
	                }
	                if(currValue.contains(" IGNORE CASE")){
	                  realValue = "(?i)"+realValue;
	                }
	                objectForEach += MongoDBObject(tempActualKey -> realValue.r)
                }
                else{
                	objectForEach += MongoDBObject(tempActualKey -> currValue)
                }
              }
            }else{
              //recursive
              if(allRoots(i).equals("userMetadata")){
                val currValue =  searchMetadataFormulateQuery(reqValue.asInstanceOf[java.util.LinkedHashMap[String,Any]], "")
                val elemMatch = tempActualKey $elemMatch currValue
                objectForEach.add(elemMatch)
              }
              else{
                val currValue =  searchMetadataFormulateQuery(reqValue.asInstanceOf[java.util.LinkedHashMap[String,Any]], tempActualKey)
                objectForEach += currValue
              }
            }
          }

          builder.add(MongoDBObject("$or" ->  objectForEach))

        }
      }
    }

    if(orFound){
      queryMap.add(MongoDBObject("$and" ->  builder))
      return MongoDBObject("$or" ->  queryMap)
    }
    else if(!builder.isEmpty)  {
      return MongoDBObject("$and" ->  builder)
    }
    else if(!root.equals("")){
      return (root $exists true)
    }
    else{
      return new MongoDBObject()
    }
  }

  def removeOldIntermediates(apiKey: Option[String], user: Option[User]){
    val cal = Calendar.getInstance()
    val timeDiff = play.Play.application().configuration().getInt("intermediateCleanup.removeAfter")
    cal.add(Calendar.HOUR, -timeDiff)
    val oldDate = cal.getTime()
    val fileList = FileDAO.find($and("isIntermediate" $eq true, "uploadDate" $lt oldDate)).toList
    for(file <- fileList)
      removeFile(file.id, "", apiKey, user)
  }

  /**
   * Update thumbnail used to represent this dataset.
   */
  def updateThumbnail(fileId: UUID, thumbnailId: UUID) {
    FileDAO.update(MongoDBObject("_id" -> new ObjectId(fileId.stringify)),
      $set("thumbnail_id" -> thumbnailId.stringify), false, false, WriteConcern.Safe)
  }

  def dumpAllFileMetadata(): List[String] = {
		    Logger.debug("Dumping metadata of all files.")

		    val fileSep = System.getProperty("file.separator")
		    val lineSep = System.getProperty("line.separator")
		    var fileMdDumpDir = play.api.Play.configuration.getString("filedump.dir").getOrElse("")
			if(!fileMdDumpDir.endsWith(fileSep))
				fileMdDumpDir = fileMdDumpDir + fileSep
			var fileMdDumpMoveDir = play.api.Play.configuration.getString("filedumpmove.dir").getOrElse("")
			if(fileMdDumpMoveDir.equals("")){
				Logger.warn("Will not move dumped files metadata to staging directory. No staging directory set.")
			}
			else{
			    if(!fileMdDumpMoveDir.endsWith(fileSep))
				  fileMdDumpMoveDir = fileMdDumpMoveDir + fileSep
			}

			var unsuccessfulDumps: ListBuffer[String] = ListBuffer.empty

			for(file <- FileDAO.findAll){
			  try{
				  val fileId = file.id.toString

				  val fileTechnicalMetadata = getTechnicalMetadataJSON(file.id)
				  val fileUserMetadata = getUserMetadataJSON(file.id)
				  if(fileTechnicalMetadata != "{}" || fileUserMetadata != "{}"){

				    val filenameNoExtension = file.filename.substring(0, file.filename.lastIndexOf("."))
				    val filePathInDirs = fileId.charAt(fileId.length()-3)+ fileSep + fileId.charAt(fileId.length()-2)+fileId.charAt(fileId.length()-1)+ fileSep + fileId + fileSep + filenameNoExtension + "__metadata.txt"
				    val mdFile = new java.io.File(fileMdDumpDir + filePathInDirs)
				    mdFile.getParentFile().mkdirs()

				    val fileWriter =  new BufferedWriter(new FileWriter(mdFile))
					fileWriter.write(fileTechnicalMetadata + lineSep + lineSep + fileUserMetadata)
					fileWriter.close()

					if(!fileMdDumpMoveDir.equals("")){
					  try{
						  val mdMoveFile = new java.io.File(fileMdDumpMoveDir + filePathInDirs)
					      mdMoveFile.getParentFile().mkdirs()

						  if(mdFile.renameTo(mdMoveFile)){
			            	Logger.debug("File metadata dumped and moved to staging directory successfully.")
						  }else{
			            	Logger.warn("Could not move dumped file metadata to staging directory.")
			            	throw new Exception("Could not move dumped file metadata to staging directory.")
						  }
					  }catch {case ex:Exception =>{
						  val badFileId = file.id.toString
						  Logger.error("Unable to stage dumped metadata of file with id "+badFileId+": "+ex.printStackTrace())
						  unsuccessfulDumps += badFileId
					  }}
					}
				  }

			  }catch {case ex:Exception =>{
			    val badFileId = file.id.toString
			    Logger.error("Unable to dump metadata of file with id "+badFileId+": "+ex.printStackTrace())
			    unsuccessfulDumps += badFileId
			  }}
			}

		    return unsuccessfulDumps.toList

	}

  def addFollower(id: UUID, userId: UUID) {
    FileDAO.update(MongoDBObject("_id" -> new ObjectId(id.stringify)),
                    $addToSet("followers" -> new ObjectId(userId.stringify)), false, false, WriteConcern.Safe)
  }

  def removeFollower(id: UUID, userId: UUID) {
    FileDAO.update(MongoDBObject("_id" -> new ObjectId(id.stringify)),
                    $pull("followers" -> new ObjectId(userId.stringify)), false, false, WriteConcern.Safe)
  }

  def updateDescription(id: UUID, description: String) {
    val result = FileDAO.update(MongoDBObject("_id" -> new ObjectId(id.stringify)),
      $set("description" -> description),
      false, false, WriteConcern.Safe)

  }

  def updateAuthorFullName(userId: UUID, fullName: String) {
    FileDAO.update(MongoDBObject("author._id" -> new ObjectId(userId.stringify)),
      $set("author.fullName" -> fullName), false, true, WriteConcern.Safe)
  }

  def incrementViews(id: UUID, user: Option[User]): (Int, Date) = {
    Logger.debug("updating views for file "+id.toString)
    val viewdate = new Date

    val updated = FileDAO.dao.collection.findAndModify(
      query=MongoDBObject("_id" -> new ObjectId(id.stringify)),
      update=$inc("stats.views" -> 1) ++ $set("stats.last_viewed" -> viewdate),
      upsert=true, fields=null, sort=null, remove=false, returnNew=true)

    user match {
      case Some(u) => {
        Logger.debug("updating views for user "+u.toString)
        FileStats.update(MongoDBObject("user_id" -> new ObjectId(u.id.stringify), "resource_id" -> new ObjectId(id.stringify), "resource_type" -> "file"),
          $inc("views" -> 1) ++ $set("last_viewed" -> viewdate), true, false, WriteConcern.Safe)
      }
      case None => {}
    }

    // Return updated count & updated date
    return updated match {
      case Some(u) => (u.get("stats").asInstanceOf[BasicDBObject].get("views").asInstanceOf[Int], viewdate)
      case None => (0, viewdate)
    }
  }

  def incrementDownloads(id: UUID, user: Option[User], dateOnly: Boolean = false) = {
    Logger.debug("updating downloads for file "+id.toString)
    val fileOp = if (dateOnly) {
      $set("stats.last_downloaded" -> new Date)
    } else {
      $inc("stats.downloads" -> 1) ++ $set("stats.last_downloaded" -> new Date)
    }
    FileDAO.update(MongoDBObject("_id" -> new ObjectId(id.stringify)), fileOp, true, false, WriteConcern.Safe)

    user match {
      case Some(u) => {
        Logger.debug("updating downloads for user "+u.toString)
        val userOp = if (dateOnly) {
          $set("last_downloaded" -> new Date)
        } else {
          $inc("downloads" -> 1) ++ $set("last_downloaded" -> new Date)
        }
        FileStats.update(MongoDBObject("user_id" -> new ObjectId(u.id.stringify), "resource_id" -> new ObjectId(id.stringify), "resource_type" -> "file"),
          userOp, true, false, WriteConcern.Safe)
      }
      case None => {}
    }
  }

  def getIterator(space: Option[String], since: Option[String], until: Option[String]): Iterator[File] = {
    var query = MongoDBObject()
    space.foreach(spid => {
      // If space is specified, we have to get that association from datasets for now
      val dsresults = datasets.getIterator(space, None, None) // ignore time filters (no bearing on files)
      while (dsresults.hasNext) {
        query += ("child_of" -> new ObjectId(dsresults.next.id.stringify))
      }
    })
    since.foreach(t => query = query ++ ("uploadDate" $gte Parsers.fromISO8601(t)))
    until.foreach(t => query = query ++ ("uploadDate" $lte Parsers.fromISO8601(t)))
    FileDAO.find(query)
  }

  def isInTrash(id: UUID): Boolean = {
    var foundTrash = false
    datasets.findByFileIdAllContain(id).foreach(ds => {
      if (ds.trash)
        foundTrash = true
    })
    foundTrash
  }
}

object FileDAO extends ModelCompanion[File, ObjectId] {
  val COLLECTION = "uploads"

  val dao = current.plugin[MongoSalatPlugin] match {
    case None => throw new RuntimeException("No MongoSalatPlugin");
    case Some(x) => new SalatDAO[File, ObjectId](collection = x.collection(COLLECTION)) {}
  }
}

object VersusDAO extends ModelCompanion[Versus,ObjectId]{
    val dao = current.plugin[MongoSalatPlugin] match {
    case None => throw new RuntimeException("No MongoSalatPlugin");
    case Some(x) => new SalatDAO[Versus, ObjectId](collection = x.collection("versus.descriptors")) {}
  }
}

object FileStats extends ModelCompanion[StatisticUser, ObjectId] {
  val dao = current.plugin[MongoSalatPlugin] match {
    case None => throw new RuntimeException("No MongoSalatPlugin");
    case Some(x) => new SalatDAO[StatisticUser, ObjectId](collection = x.collection("statistics.users")) {}
  }
}