Merge branch 'release-0.6.0'

Author: dszeto

bin/build.sh

@@ -72,14 +72,24 @@ else
     $SBT $CLEAN update assembly
 
     # Build process itemrec Top-k Items Collector
-    echo "Going to build PredictionIO Top-k Items Collector Assembly..."
+    echo "Going to build PredictionIO ItemRec Top-k Items Collector Assembly..."
     cd $BASE/process/engines/itemrec/evaluations/scala/topkitems
     $SBT $CLEAN update assembly
 
     # Build process itemsim algo assembly
-    echo "Going to build PredictionIO Process ItemRec Algorithms Assembly..."
+    echo "Going to build PredictionIO Process ItemSim Algorithms Assembly..."
     cd $BASE/process/engines/itemsim/algorithms/hadoop/scalding
     $SBT $CLEAN update assembly
+
+    # Build process itemsim eval assembly
+    echo "Going to build PredictionIO Process ItemSim Evaluations Assembly..."
+    cd $BASE/process/engines/itemsim/evaluations/hadoop/scalding
+    $SBT $CLEAN update assembly
+
+    # Build process itemsim Top-k Items Collector
+    echo "Going to build PredictionIO ItemSim Top-k Items Collector Assembly..."
+    cd $BASE/process/engines/itemsim/evaluations/scala/topkitems
+    $SBT $CLEAN update assembly
 fi
 
 # Build connection check tool

bin/change-version.py

@@ -44,7 +44,8 @@ def change(filename, oldversion, newversion):
     'process/engines/itemrec/evaluations/hadoop/scalding/trainingtestsplit/build.sbt',
     'process/engines/itemrec/evaluations/scala/*/build.sbt',
     'process/engines/itemsim/algorithms/hadoop/scalding/build.sbt',
-    'process/engines/itemsim/algorithms/hadoop/scalding/itemsimcf/build.sbt',
+    'process/engines/itemsim/evaluations/hadoop/scalding/build.sbt',
+    'process/engines/itemsim/evaluations/scala/*/build.sbt',
     'servers/*/project/Build.scala',
     'servers/scheduler/conf/application.conf',
     'tools/*/build.sbt',

bin/common.sh

@@ -2,7 +2,7 @@
 
 # This script should be sourced with $BASE set to the base of the repository
 
-VERSION=0.5.2
+VERSION=0.6.0
 
 # Play framework related
 PLAY_OPTS=

bin/package.sh

@@ -47,19 +47,24 @@ cp -R $DIST_DIR/bin $PACKAGE_DIR
 cp $BASE/bin/quiet.sh $PACKAGE_DIR/bin
 cp -R $DIST_DIR/conf $PACKAGE_DIR
 
-cp "$BASE/process/engines/itemrec/algorithms/hadoop/scalding/target/scala-2.9.2/PredictionIO-Process-ItemRec-Algorithms-Hadoop-Scalding-assembly-$VERSION.jar" "$PACKAGE_DIR/lib"
+cp "$BASE/process/engines/itemrec/algorithms/hadoop/scalding/target/scala-2.10/PredictionIO-Process-ItemRec-Algorithms-Hadoop-Scalding-assembly-$VERSION.jar" "$PACKAGE_DIR/lib"
 cp "$BASE/process/engines/itemrec/algorithms/scala/mahout/target/scala-2.10/PredictionIO-Process-ItemRec-Algorithms-Scala-Mahout-assembly-$VERSION.jar" "$PACKAGE_DIR/lib"
-cp "$BASE/process/engines/itemrec/evaluations/hadoop/scalding/target/scala-2.9.2/PredictionIO-Process-ItemRec-Evaluations-Hadoop-Scalding-assembly-$VERSION.jar" "$PACKAGE_DIR/lib"
+cp "$BASE/process/engines/itemrec/evaluations/hadoop/scalding/target/scala-2.10/PredictionIO-Process-ItemRec-Evaluations-Hadoop-Scalding-assembly-$VERSION.jar" "$PACKAGE_DIR/lib"
 cp "$BASE/process/engines/itemrec/evaluations/scala/topkitems/target/scala-2.10/PredictionIO-Process-ItemRec-Evaluations-TopKItems-assembly-$VERSION.jar" "$PACKAGE_DIR/lib"
-cp "$BASE/process/engines/itemrec/evaluations/scala/trainingtestsplit/target/scala-2.9.2/PredictionIO-Process-ItemRec-Evaluations-Scala-TrainingTestSplitTime-assembly-$VERSION.jar" "$PACKAGE_DIR/lib"
+cp "$BASE/process/engines/itemrec/evaluations/scala/trainingtestsplit/target/scala-2.10/PredictionIO-Process-ItemRec-Evaluations-Scala-TrainingTestSplitTime-assembly-$VERSION.jar" "$PACKAGE_DIR/lib"
 cp "$BASE/process/engines/itemrec/evaluations/scala/paramgen/target/scala-2.10/PredictionIO-Process-ItemRec-Evaluations-ParamGen-assembly-$VERSION.jar" "$PACKAGE_DIR/lib"
-cp "$BASE/process/engines/itemsim/algorithms/hadoop/scalding/target/scala-2.9.2/PredictionIO-Process-ItemSim-Algorithms-Hadoop-Scalding-assembly-$VERSION.jar" "$PACKAGE_DIR/lib"
+cp "$BASE/process/engines/itemsim/algorithms/hadoop/scalding/target/scala-2.10/PredictionIO-Process-ItemSim-Algorithms-Hadoop-Scalding-assembly-$VERSION.jar" "$PACKAGE_DIR/lib"
+cp "$BASE/process/engines/itemsim/evaluations/hadoop/scalding/target/scala-2.10/PredictionIO-Process-ItemSim-Evaluations-Hadoop-Scalding-assembly-$VERSION.jar" "$PACKAGE_DIR/lib"
+cp "$BASE/process/engines/itemsim/evaluations/scala/topkitems/target/scala-2.10/PredictionIO-Process-ItemSim-Evaluations-TopKItems-assembly-$VERSION.jar" "$PACKAGE_DIR/lib"
 cp -n $BASE/tools/conncheck/target/pack/lib/* $PACKAGE_DIR/lib
 cp -n $BASE/tools/migration/0.5/appdata/target/pack/lib/* $PACKAGE_DIR/lib
 cp -n $BASE/tools/settingsinit/target/pack/lib/* $PACKAGE_DIR/lib
 cp -n $BASE/tools/softwaremanager/target/pack/lib/* $PACKAGE_DIR/lib
 cp -n $BASE/tools/users/target/pack/lib/* $PACKAGE_DIR/lib
 
+mkdir -p $PACKAGE_DIR/vendors/mahout-distribution-0.8
+cp $VENDOR_MAHOUT/mahout-core-0.8-job.jar $PACKAGE_DIR/vendors/mahout-distribution-0.8
+
 cd $DIST_DIR/target
 rm "$PACKAGE_NAME.zip"
 zip -q -r "$PACKAGE_NAME.zip" "$PACKAGE_NAME"

bin/vendors.sh

@@ -27,29 +27,22 @@ install_play () {
 }
 
 install_mahout () {
-	echo "Going to download and build Apache Mahout 0.8 Build 1993..."
-	mkdir -p $LIB_MAHOUT
-	cd $LIB_MAHOUT
-	if [ ! -f mahout-core-0.8-SNAPSHOT.jar -o ! -f mahout-math-0.8-SNAPSHOT.jar ] ; then
-		rm -rf $LIB_MAHOUT
-		mkdir -p $LIB_MAHOUT
-		cd $LIB_MAHOUT
-		curl -o mahout-core-0.8-SNAPSHOT.jar http://download.prediction.io/mahout-snapshots/1993/mahout-core-0.8-SNAPSHOT.jar
-		curl -o mahout-math-0.8-SNAPSHOT.jar http://download.prediction.io/mahout-snapshots/1993/mahout-math-0.8-SNAPSHOT.jar
-	fi
-	mkdir -p $VENDOR_MAHOUT
-	cd $VENDOR_MAHOUT
-	if [ ! -f mahout-core-0.8-SNAPSHOT-job.jar ] ; then
-		curl -o mahout-core-0.8-SNAPSHOT-job.jar http://download.prediction.io/mahout-snapshots/1993/mahout-core-0.8-SNAPSHOT-job.jar
-	fi
+	echo "Going to download and install Apache Mahout 0.8..."
+	mkdir -p $VENDORS_PATH
+	cd $VENDORS_PATH
+	echo "Retrieving Apache mirror list..."
+	curl -o apache_mahout_mirrors.txt http://www.apache.org/dyn/closer.cgi/mahout/0.8/mahout-distribution-0.8.tar.gz
+	MAHOUT_URL=$(cat apache_mahout_mirrors.txt | grep -m 1 "<strong>.*</strong>" | sed 's/.*<strong>//' | sed 's/<\/strong>.*//')
+	echo "Found mirror: $MAHOUT_URL"
+	curl -O $MAHOUT_URL
+	tar zxvf mahout-distribution-0.8.tar.gz
 }
 
 # Third party software
 VENDORS_PATH="$BASE/vendors"
 VENDOR_SBT="$VENDORS_PATH/sbt-0.12.3/sbt"
 VENDOR_PLAY="$VENDORS_PATH/play-2.1.1/play"
-VENDOR_MAHOUT="$VENDORS_PATH/mahout-0.8-snapshot"
-LIB_MAHOUT="$BASE/process/engines/itemrec/algorithms/scala/mahout/commons/lib"
+VENDOR_MAHOUT="$VENDORS_PATH/mahout-distribution-0.8"
 
 # Detect existing installations in search path
 # Do not use existing sbt to enforce JVM settings
@@ -79,11 +72,11 @@ else
 	exit 1
 fi
 
-if [ -r "$LIB_MAHOUT/mahout-core-0.8-SNAPSHOT.jar" -a -r "$LIB_MAHOUT/mahout-math-0.8-SNAPSHOT.jar" -a -r "$VENDOR_MAHOUT/mahout-core-0.8-SNAPSHOT-job.jar" ] ; then
-	echo "Using Apache Mahout 0.8 Build 1993 in vendors."
+if [ -r "$VENDOR_MAHOUT/mahout-core-0.8-job.jar" ] ; then
+	echo "Using Apache Mahout 0.8 in vendors."
 elif install_mahout ; then
 	echo ""
 else
-	echo "Unable to locate Apache Mahout 0.8 Build 1993 and automatic installation failed. Aborting." >&2
+	echo "Unable to locate Apache Mahout 0.8 and automatic installation failed. Aborting." >&2
 	exit 1
 fi

commons/build.sbt

@@ -1,21 +1,19 @@
 name := "PredictionIO Commons"
 
-version := "0.5.2"
+version := "0.6.0"
 
 organization := "io.prediction"
 
-scalaVersion := "2.10.0"
-
-crossScalaVersions := Seq("2.9.2", "2.10.0")
+scalaVersion := "2.10.2"
 
 scalacOptions in (Compile, doc) ++= Opts.doc.title("PredictionIO Commons API Documentation")
 
 libraryDependencies ++= Seq(
-  "com.github.nscala-time" %% "nscala-time" % "0.2.0",
-  "com.twitter" %% "chill" % "0.2.2",
-  "com.typesafe" % "config" % "1.0.0",
+  "com.github.nscala-time" %% "nscala-time" % "0.4.2",
+  "com.twitter" %% "chill" % "0.2.3",
+  "com.typesafe" % "config" % "1.0.2",
   "org.mongodb" %% "casbah" % "2.6.2",
-  "org.specs2" %% "specs2" % "1.12.3" % "test"
+  "org.specs2" %% "specs2" % "1.14" % "test"
 )
 
 publishTo := Some(Resolver.file("file",  new File(Path.userHome.absolutePath+"/.m2/repository")))

commons/src/main/scala/io/prediction/commons/Config.scala

@@ -630,6 +630,16 @@ class Config {
     }
   }
 
+  /** Obtains an ItemSimScores object with configured backend type. */
+  def getModeldataItemSimScores(): modeldata.ItemSimScores = {
+    modeldataDbType match {
+      case "mongodb" => {
+        new modeldata.mongodb.MongoItemSimScores(modeldataMongoDb.get)
+      }
+      case _ => throw new RuntimeException("Invalid modeldata database type: " + modeldataDbType)
+    }
+  }
+
   /** Obtains an ItemRecScores object with configured backend type. */
   def getModeldataTrainingItemRecScores(): modeldata.ItemRecScores = {
     modeldataDbType match {
@@ -639,4 +649,14 @@ class Config {
       case _ => throw new RuntimeException("Invalid modeldata database type: " + modeldataTrainingDbType)
     }
   }
+
+  /** Obtains an ItemSimScores object with configured backend type. */
+  def getModeldataTrainingItemSimScores(): modeldata.ItemSimScores = {
+    modeldataDbType match {
+      case "mongodb" => {
+        new modeldata.mongodb.MongoItemSimScores(modeldataTrainingMongoDb.get)
+      }
+      case _ => throw new RuntimeException("Invalid modeldata database type: " + modeldataTrainingDbType)
+    }
+  }
 }

commons/src/main/scala/io/prediction/commons/appdata/Items.scala

@@ -38,6 +38,9 @@ trait Items {
   /** Get an item by ID. */
   def get(appid: Int, id: String): Option[Item]
 
+  /** Find all items by App ID. */
+  def getByAppid(appid: Int): Iterator[Item]
+
   /** Get items by IDs. */
   def getByIds(appid: Int, ids: Seq[String]): Seq[Item]
 

commons/src/main/scala/io/prediction/commons/appdata/mongodb/MongoItems.scala

@@ -39,6 +39,8 @@ class MongoItems(db: MongoDB) extends Items {
     itemColl.findOne(MongoDBObject("_id" -> idWithAppid(appid, id))) map { dbObjToItem(_) }
   }
 
+  def getByAppid(appid: Int) = new MongoItemsIterator(itemColl.find(MongoDBObject("appid" -> appid)))
+
   def getByIds(appid: Int, ids: Seq[String]) = {
     itemColl.find(MongoDBObject("_id" -> MongoDBObject("$in" -> ids.map(idWithAppid(appid, _))))).toList map { dbObjToItem(_) }
   }
@@ -90,4 +92,9 @@ class MongoItems(db: MongoDB) extends Items {
       attributes = Option(getAttributesFromDBObject(dbObj)).filter(!_.isEmpty)
     )
   }
+
+  class MongoItemsIterator(it: MongoCursor) extends Iterator[Item] {
+    def next = dbObjToItem(it.next)
+    def hasNext = it.hasNext
+  }
 }

commons/src/main/scala/io/prediction/commons/modeldata/ItemRecScores.scala

@@ -9,6 +9,7 @@ import io.prediction.commons.settings.{Algo, App, OfflineEval}
   * @param iid Item ID.
   * @param score Recommendation score.
   * @param itypes Item types of the item recommended. Copied from the item when a batch mode algorithm is run.
+  * @param appid App ID of this record.
   * @param algoid Algo ID of this record.
   * @param modelset Model data set.
   * @param id ItemRecScore ID (optional field used internally for sorting)
@@ -17,7 +18,7 @@ case class ItemRecScore(
   uid: String,
   iid: String,
   score: Double,
-  itypes: List[String],
+  itypes: Seq[String],
   appid: Int,
   algoid: Int,
   modelset: Boolean,
@@ -26,14 +27,14 @@ case class ItemRecScore(
 
 /** Base trait for implementations that interact with itemrec scores in the backend data store. */
 trait ItemRecScores {
-  /** Insert an ItemSimScore and return it with a real ID, if any (database vendor dependent). */
+  /** Insert an ItemRecScore and return it with a real ID, if any (database vendor dependent). */
   def insert(itemRecScore: ItemRecScore): ItemRecScore
 
-  /** Get the top N ItemSimScore ranked by score in descending order.
+  /** Get the top N ItemRecScore ranked by score in descending order.
     *
-    * @param after Returns the next top N results after the provided ItemSimScore, if provided.
+    * @param after Returns the next top N results after the provided ItemRecScore, if provided.
     */
-  def getTopN(uid: String, n: Int, itypes: Option[List[String]], after: Option[ItemRecScore])(implicit app: App, algo: Algo, offlineEval: Option[OfflineEval] = None): Iterator[ItemRecScore]
+  def getTopN(uid: String, n: Int, itypes: Option[Seq[String]], after: Option[ItemRecScore])(implicit app: App, algo: Algo, offlineEval: Option[OfflineEval] = None): Iterator[ItemRecScore]
 
   /** Delete by Algo ID. */
   def deleteByAlgoid(algoid: Int)