Initial commit

Author: mateiz

.gitignore

@@ -0,0 +1,4 @@
+*~
+*.swp
+build
+work

Makefile

@@ -0,0 +1,65 @@
+EMPTY =
+SPACE = $(EMPTY) $(EMPTY)
+
+# Build up classpath by concatenating some strings
+JARS = third_party/nexus.jar
+JARS += third_party/asm-3.2/lib/all/asm-all-3.2.jar
+JARS += third_party/colt.jar
+JARS += third_party/google-collect-1.0-rc5/google-collect-1.0-rc5.jar
+JARS += third_party/hadoop-0.20.0/hadoop-0.20.0-core.jar
+JARS += third_party/hadoop-0.20.0/lib/commons-logging-1.0.4.jar
+JARS += third_party/scalatest-1.0/scalatest-1.0.jar
+JARS += third_party/ScalaCheck-1.5.jar
+CLASSPATH = $(subst $(SPACE),:,$(JARS))
+
+SCALA_SOURCES =  src/examples/*.scala src/scala/spark/*.scala src/scala/spark/repl/*.scala
+SCALA_SOURCES += src/test/spark/*.scala src/test/spark/repl/*.scala
+
+JAVA_SOURCES = $(wildcard src/java/spark/compress/lzf/*.java)
+
+ifeq ($(USE_FSC),1)
+  COMPILER_NAME = fsc
+else
+  COMPILER_NAME = scalac
+endif
+
+ifeq ($(SCALA_HOME),)
+  COMPILER = $(COMPILER_NAME)
+else
+  COMPILER = $(SCALA_HOME)/bin/$(COMPILER_NAME)
+endif
+
+all: scala java
+
+build/classes:
+	mkdir -p build/classes
+
+scala: build/classes java
+	$(COMPILER) -unchecked -d build/classes -classpath $(CLASSPATH) $(SCALA_SOURCES)
+
+java: $(JAVA_SOURCES) build/classes
+	javac -d build/classes $(JAVA_SOURCES)
+
+native: java
+	$(MAKE) -C src/native
+
+jar: build/spark.jar build/spark-dep.jar
+
+build/spark.jar: scala java
+	jar cf build/spark.jar -C build/classes spark
+
+build/spark-dep.jar:
+	mkdir -p build/dep
+	cd build/dep &&	for i in $(JARS); do jar xf ../../$$i; done
+	jar cf build/spark-dep.jar -C build/dep .
+
+test: all
+	./alltests
+
+default: all
+
+clean:
+	$(MAKE) -C src/native clean
+	rm -rf build
+
+.phony: default all clean scala java native jar

README

@@ -0,0 +1,16 @@
+Spark requires Scala 2.7.7. It will currently not work with 2.8, or with
+earlier versions of the 2.7 branch.
+
+To build and run Spark, you will need to have Scala's bin in your $PATH,
+or you will need to set the SCALA_HOME environment variable to point
+to where you've installed Scala. Scala must be accessible through one
+of these methods on Nexus slave nodes as well as on the master.
+
+To build Spark and the example programs, run make.
+
+To run one of the examples, use ./run <class> <params>. For example,
+./run SparkLR will run the Logistic Regression example. Each of the
+example programs prints usage help if no params are given.
+
+Tip: If you are building Spark and examples repeatedly, export USE_FSC=1
+to have the Makefile use the fsc compiler daemon instead of scalac.

alltests

@@ -0,0 +1,3 @@
+#!/bin/bash
+FWDIR=`dirname $0`
+$FWDIR/run org.scalatest.tools.Runner -p $FWDIR/build/classes -o $@

lr_data.txt

@@ -0,0 +1,33 @@
+#!/bin/bash
+
+# Figure out where the Scala framework is installed
+FWDIR=`dirname $0`
+
+# Set JAVA_OPTS to be able to load libnexus.so and set various other misc options
+JAVA_OPTS="-Djava.library.path=$FWDIR/third_party:$FWDIR/src/native -Xmx750m"
+if [ -e $FWDIR/conf/java-opts ] ; then
+  JAVA_OPTS+=" `cat $FWDIR/conf/java-opts`"
+fi
+export JAVA_OPTS
+
+# Build up classpath
+CLASSPATH=$FWDIR/build/classes
+CLASSPATH+=:$FWDIR/third_party/nexus.jar
+CLASSPATH+=:$FWDIR/third_party/asm-3.2/lib/all/asm-all-3.2.jar
+CLASSPATH+=:$FWDIR/third_party/colt.jar
+CLASSPATH+=:$FWDIR/third_party/google-collect-1.0-rc5/google-collect-1.0-rc5.jar
+CLASSPATH+=:$FWDIR/third_party/hadoop-0.20.0/hadoop-0.20.0-core.jar
+CLASSPATH+=:third_party/scalatest-1.0/scalatest-1.0.jar
+CLASSPATH+=:third_party/ScalaCheck-1.5.jar
+for jar in $FWDIR/third_party/hadoop-0.20.0/lib/*.jar; do
+  CLASSPATH+=:$jar
+done
+export CLASSPATH
+
+if [ -n "$SCALA_HOME" ]; then
+  SCALA=${SCALA_HOME}/bin/scala
+else
+  SCALA=scala
+fi
+
+exec $SCALA -cp $CLASSPATH $@

run

@@ -0,0 +1,5 @@
+#!/bin/sh
+echo "In spark-executor"
+FWDIR="`dirname $0`"
+echo Framework dir: $FWDIR
+exec $FWDIR/run spark.Executor

spark-executor

@@ -0,0 +1,3 @@
+#!/bin/sh
+FWDIR="`dirname $0`"
+exec $FWDIR/run spark.repl.Main $@

spark-shell

@@ -0,0 +1,24 @@
+import spark._
+
+object CpuHog {
+  def main(args: Array[String]) {
+    if (args.length != 3) {
+      System.err.println("Usage: CpuHog <master> <tasks> <threads_per_task>");
+      System.exit(1)
+    }
+    val sc = new SparkContext(args(0), "CPU hog")
+    val tasks = args(1).toInt
+    val threads = args(2).toInt
+    def task {
+      for (i <- 0 until threads-1) {
+        new Thread() {
+          override def run {
+            while(true) {}
+          }
+        }.start()
+      }
+      while(true) {}
+    }
+    sc.runTasks(Array.make(tasks, () => task))
+  }
+}

src/examples/CpuHog.scala

@@ -0,0 +1,16 @@
+import spark._
+
+object HdfsTest {
+  def main(args: Array[String]) {
+    val sc = new SparkContext(args(0), "HdfsTest")
+    val file = sc.textFile(args(1))
+    val mapped = file.map(s => s.length).cache()
+    for (iter <- 1 to 10) {
+      val start = System.currentTimeMillis()
+      for (x <- mapped) { x + 2 }
+      //  println("Processing: " + x)
+      val end = System.currentTimeMillis()
+      println("Iteration " + iter + " took " + (end-start) + " ms")
+    }
+  }
+}