Further improvements -- build native stuff in target directory and add a

test-report target for XML test reports

Makefile

-EMPTY =
-SPACE = $(EMPTY) $(EMPTY)
-
-# Build up classpath by concatenating some strings
-JARS = third_party/mesos.jar
-JARS += third_party/asm-3.2/lib/all/asm-all-3.2.jar
-JARS += third_party/colt.jar
-JARS += third_party/guava-r07/guava-r07.jar
-JARS += third_party/hadoop-0.20.0/hadoop-0.20.0-core.jar
-JARS += third_party/hadoop-0.20.0/lib/commons-logging-1.0.4.jar
-JARS += third_party/scalatest-1.2/scalatest-1.2.jar
-JARS += third_party/scalacheck_2.8.0-1.7.jar
-JARS += third_party/jetty-7.1.6.v20100715/jetty-server-7.1.6.v20100715.jar
-JARS += third_party/jetty-7.1.6.v20100715/servlet-api-2.5.jar
-JARS += third_party/apache-log4j-1.2.16/log4j-1.2.16.jar
-JARS += third_party/slf4j-1.6.1/slf4j-api-1.6.1.jar
-JARS += third_party/slf4j-1.6.1/slf4j-log4j12-1.6.1.jar
-CLASSPATH = $(subst $(SPACE),:,$(JARS))
-
-SCALA_SOURCES =  src/examples/*.scala src/scala/spark/*.scala src/scala/spark/repl/*.scala
-SCALA_SOURCES += src/test/spark/*.scala src/test/spark/repl/*.scala
-
-JAVA_SOURCES = $(wildcard src/java/spark/compress/lzf/*.java)
-
-ifeq ($(USE_FSC),1)
-  COMPILER_NAME = fsc
-else
-  COMPILER_NAME = scalac
-endif
-
-ifeq ($(SCALA_HOME),)
-  COMPILER = $(COMPILER_NAME)
-else
-  COMPILER = $(SCALA_HOME)/bin/$(COMPILER_NAME)
-endif
-
-CONF_FILES = conf/spark-env.sh conf/log4j.properties conf/java-opts
-
-all: scala java conf-files
-
-build/classes:
-	mkdir -p build/classes
-
-scala: build/classes java
-	$(COMPILER) -unchecked -d build/classes -classpath build/classes:$(CLASSPATH) $(SCALA_SOURCES)
-
-java: $(JAVA_SOURCES) build/classes
-	javac -d build/classes $(JAVA_SOURCES)
-
-native: java
-	$(MAKE) -C src/native
-
-jar: build/spark.jar build/spark-dep.jar
-
-dep-jar: build/spark-dep.jar
-
-build/spark.jar: scala java
-	jar cf build/spark.jar -C build/classes spark
-
-build/spark-dep.jar:
-	mkdir -p build/dep
-	cd build/dep &&	for i in $(JARS); do jar xf ../../$$i; done
-	jar cf build/spark-dep.jar -C build/dep .
-
-conf-files: $(CONF_FILES)
-
-$(CONF_FILES): %: | %.template
-	cp $@.template $@
-
-test: all
-	./alltests
-
-default: all
-
-clean:
-	$(MAKE) -C src/native clean
-	rm -rf build
-
-.phony: default all clean scala java native jar dep-jar conf-files

alltests

-#!/bin/bash
-FWDIR="`dirname $0`"
-if [ "x$SPARK_MEM" == "x" ]; then
-  export SPARK_MEM=500m
-fi
-RESULTS_DIR="$FWDIR/build/test_results"
-if [ -d $RESULTS_DIR ]; then
-  rm -r $RESULTS_DIR
-fi
-mkdir -p $RESULTS_DIR
-$FWDIR/run org.scalatest.tools.Runner -p $FWDIR/target/test/classes -u $RESULTS_DIR -o $@

project/build/SparkProject.scala

 import sbt._
 import de.element34.sbteclipsify._
 
+import Process._
+
 class SparkProject(info: ProjectInfo)
-extends DefaultProject(info) with Eclipsify {
+extends DefaultProject(info) with Eclipsify
+{
+  val TARGET = path("target") / "scala_2.8.1"
+
+  val NATIVE_DIR = path("src") / "main" / "native"
+
+  val NATIVE_SOURCES = NATIVE_DIR * "*.c"
+
+  val NATIVE_LIB = {
+    if (System.getProperty("os.name") == "Mac OS X")
+      "libspark_native.dylib"
+    else
+      "libspark_native.so"
+  }
+
+  lazy val native = fileTask(TARGET / NATIVE_LIB from NATIVE_SOURCES) {
+    val makeTarget = " ../../../target/scala_2.8.1/native/" + NATIVE_LIB
+    (("make -C " + NATIVE_DIR + " " + makeTarget) ! log)
+    None
+  } dependsOn(compile) describedAs("Compiles native library.")
+
+  val TEST_REPORT_DIR = TARGET / "test-report"
+
+  lazy val testReport = task {
+    log.info("Creating " + TEST_REPORT_DIR + "...")
+    if (!TEST_REPORT_DIR.exists) {
+      TEST_REPORT_DIR.asFile.mkdirs()
+    }
+
+    log.info("Executing org.scalatest.tools.Runner...")
+    val command = ("scala -classpath " + testClasspath.absString + 
+                   " org.scalatest.tools.Runner -o " + 
+                   " -u " + TEST_REPORT_DIR.absolutePath +
+                   " -p " + (TARGET / "test-classes").absolutePath)
+    val process = Process(command, path("."), "JAVA_OPTS" -> "-Xmx500m")
+    process !
+
+    None
+  } dependsOn(compile, testCompile) describedAs("Generate XML test report.")
 }

src/main/native/Makefile

@@ -8,22 +8,32 @@ SPARK = ../../..
 
 LZF = $(SPARK)/lib/liblzf-3.5
 
-LIB = libspark_native.so
+TARGET = $(SPARK)/target/scala_2.8.1
+
+LIB = $(TARGET)/native/libspark_native.so
+
+OS_X_LIB = $(TARGET)/native/libspark_native.dylib
 
 all: $(LIB)
 
-spark_compress_lzf_LZF.h: $(SPARK)/target/scala_2.8.1/classes/spark/compress/lzf/LZF.class
+spark_compress_lzf_LZF.h: $(TARGET)/classes/spark/compress/lzf/LZF.class
 ifeq ($(JAVA_HOME),)
 	$(error JAVA_HOME is not set)
 else
 	$(JAVA_HOME)/bin/javah -classpath $(SPARK)/target/scala_2.8.1/classes spark.compress.lzf.LZF
 endif
 
-$(LIB): spark_compress_lzf_LZF.h spark_compress_lzf_LZF.c
+$(TARGET)/native:
+	mkdir -p $@
+
+$(LIB): spark_compress_lzf_LZF.h spark_compress_lzf_LZF.c | $(TARGET)/native
 	$(CC) $(CFLAGS) -shared -o $@ spark_compress_lzf_LZF.c \
 	      -I $(JAVA_HOME)/include -I $(JAVA_HOME)/include/$(OS_NAME) \
 	      -I $(LZF) $(LZF)/lzf_c.c $(LZF)/lzf_d.c
 
+$(OS_X_LIB): $(LIB)
+	cp $< $@
+
 clean:
 	rm -f spark_compress_lzf_LZF.h $(LIB)