diff --git a/.gitignore b/.gitignore
index 5abdec5d50d8b68b20a6fbd05bdc1a952ac55370..e6cdca77e6fb960ad96cdc5b5e1e9a52d5a05ef3 100644
--- a/.gitignore
+++ b/.gitignore
@@ -1,10 +1,26 @@
*~
*.swp
-build
-work
+*.iml
+.idea/
+/build/
+work/
+out/
.DS_Store
third_party/libmesos.so
third_party/libmesos.dylib
conf/java-opts
conf/spark-env.sh
conf/log4j.properties
+target/
+reports/
+.project
+.classpath
+.scala_dependencies
+lib_managed/
+src_managed/
+project/boot/
+project/plugins/project/build.properties
+project/build/target/
+project/plugins/target/
+project/plugins/lib_managed/
+project/plugins/src_managed/
diff --git a/Makefile b/Makefile
deleted file mode 100644
index 762d752fdab52a27c241053e3f43189b5542b6d8..0000000000000000000000000000000000000000
--- a/Makefile
+++ /dev/null
@@ -1,73 +0,0 @@
-EMPTY =
-SPACE = $(EMPTY) $(EMPTY)
-
-# Build up classpath by concatenating some strings
-JARS = third_party/mesos.jar
-JARS += third_party/asm-3.2/lib/all/asm-all-3.2.jar
-JARS += third_party/colt.jar
-JARS += third_party/guava-r07/guava-r07.jar
-JARS += third_party/hadoop-0.20.0/hadoop-0.20.0-core.jar
-JARS += third_party/hadoop-0.20.0/lib/commons-logging-1.0.4.jar
-JARS += third_party/scalatest-1.2/scalatest-1.2.jar
-JARS += third_party/scalacheck_2.8.0-1.7.jar
-JARS += third_party/jetty-7.1.6.v20100715/jetty-server-7.1.6.v20100715.jar
-JARS += third_party/jetty-7.1.6.v20100715/servlet-api-2.5.jar
-JARS += third_party/apache-log4j-1.2.16/log4j-1.2.16.jar
-JARS += third_party/slf4j-1.6.1/slf4j-api-1.6.1.jar
-JARS += third_party/slf4j-1.6.1/slf4j-log4j12-1.6.1.jar
-JARS += third_party/compress-lzf-0.6.0/compress-lzf-0.6.0.jar
-
-CLASSPATH = $(subst $(SPACE),:,$(JARS))
-
-SCALA_SOURCES = src/examples/*.scala src/scala/spark/*.scala src/scala/spark/repl/*.scala
-SCALA_SOURCES += src/test/spark/*.scala src/test/spark/repl/*.scala
-
-
-ifeq ($(USE_FSC),1)
- COMPILER_NAME = fsc
-else
- COMPILER_NAME = scalac
-endif
-
-ifeq ($(SCALA_HOME),)
- COMPILER = $(COMPILER_NAME)
-else
- COMPILER = $(SCALA_HOME)/bin/$(COMPILER_NAME)
-endif
-
-CONF_FILES = conf/spark-env.sh conf/log4j.properties conf/java-opts
-
-all: scala conf-files
-
-build/classes:
- mkdir -p build/classes
-
-scala: build/classes
- $(COMPILER) -d build/classes -classpath build/classes:$(CLASSPATH) $(SCALA_SOURCES)
-
-jar: build/spark.jar build/spark-dep.jar
-
-dep-jar: build/spark-dep.jar
-
-build/spark.jar: scala
- jar cf build/spark.jar -C build/classes spark
-
-build/spark-dep.jar:
- mkdir -p build/dep
- cd build/dep && for i in $(JARS); do jar xf ../../$$i; done
- jar cf build/spark-dep.jar -C build/dep .
-
-conf-files: $(CONF_FILES)
-
-$(CONF_FILES): %: | %.template
- cp $@.template $@
-
-test: all
- ./alltests
-
-default: all
-
-clean:
- rm -rf build
-
-.phony: default all clean scala jar dep-jar conf-files
diff --git a/README b/README
index f084f22a1f9dc5891bc4340c0ceb14ca9152a745..3fbf8c98de4e6cb7e2616efc9ce5ca10b5eb9b28 100644
--- a/README
+++ b/README
@@ -1,24 +1,32 @@
+ONLINE DOCUMENTATION
+
+You can find the latest Spark documentation, including a programming guide,
+on the project wiki at http://github.com/mesos/spark/wiki. This file only
+contains basic setup instructions.
+
+
+
BUILDING
-Spark requires Scala 2.8. This version has been tested with 2.8.0.final.
+Spark requires Scala 2.8. This version has been tested with 2.8.1.final.
-To build and run Spark, you will need to have Scala's bin in your $PATH,
-or you will need to set the SCALA_HOME environment variable to point
-to where you've installed Scala. Scala must be accessible through one
-of these methods on Mesos slave nodes as well as on the master.
+The project is built using Simple Build Tool (SBT), which is packaged with it.
+To build Spark and its example programs, run sbt/sbt compile.
-To build Spark and the example programs, run make.
+To run Spark, you will need to have Scala's bin in your $PATH, or you
+will need to set the SCALA_HOME environment variable to point to where
+you've installed Scala. Scala must be accessible through one of these
+methods on Mesos slave nodes as well as on the master.
To run one of the examples, use ./run <class> <params>. For example,
-./run SparkLR will run the Logistic Regression example. Each of the
-example programs prints usage help if no params are given.
+./run spark.examples.SparkLR will run the Logistic Regression example.
+Each of the example programs prints usage help if no params are given.
All of the Spark samples take a <host> parameter that is the Mesos master
to connect to. This can be a Mesos URL, or "local" to run locally with one
thread, or "local[N]" to run locally with N threads.
-Tip: If you are building Spark and examples repeatedly, export USE_FSC=1
-to have the Makefile use the fsc compiler daemon instead of scalac.
+
CONFIGURATION
diff --git a/alltests b/alltests
deleted file mode 100755
index cd11604855b549d2ac054edf806a984a8a5ef252..0000000000000000000000000000000000000000
--- a/alltests
+++ /dev/null
@@ -1,11 +0,0 @@
-#!/bin/bash
-FWDIR="`dirname $0`"
-if [ "x$SPARK_MEM" == "x" ]; then
- export SPARK_MEM=500m
-fi
-RESULTS_DIR="$FWDIR/build/test_results"
-if [ -d $RESULTS_DIR ]; then
- rm -r $RESULTS_DIR
-fi
-mkdir -p $RESULTS_DIR
-$FWDIR/run org.scalatest.tools.Runner -p $FWDIR/build/classes -u $RESULTS_DIR -o $@
diff --git a/third_party/apache-log4j-1.2.16/log4j-1.2.16.jar b/core/lib/apache-log4j-1.2.16/log4j-1.2.16.jar
similarity index 100%
rename from third_party/apache-log4j-1.2.16/log4j-1.2.16.jar
rename to core/lib/apache-log4j-1.2.16/log4j-1.2.16.jar
diff --git a/third_party/asm-3.2/.DS_Store b/core/lib/asm-3.2/.DS_Store
similarity index 100%
rename from third_party/asm-3.2/.DS_Store
rename to core/lib/asm-3.2/.DS_Store
diff --git a/third_party/asm-3.2/lib/all/README.txt b/core/lib/asm-3.2/lib/all/README.txt
similarity index 100%
rename from third_party/asm-3.2/lib/all/README.txt
rename to core/lib/asm-3.2/lib/all/README.txt
diff --git a/third_party/asm-3.2/lib/all/asm-all-3.2.jar b/core/lib/asm-3.2/lib/all/asm-all-3.2.jar
similarity index 100%
rename from third_party/asm-3.2/lib/all/asm-all-3.2.jar
rename to core/lib/asm-3.2/lib/all/asm-all-3.2.jar
diff --git a/third_party/asm-3.2/lib/all/asm-all-3.2.pom b/core/lib/asm-3.2/lib/all/asm-all-3.2.pom
similarity index 100%
rename from third_party/asm-3.2/lib/all/asm-all-3.2.pom
rename to core/lib/asm-3.2/lib/all/asm-all-3.2.pom
diff --git a/third_party/asm-3.2/lib/all/asm-debug-all-3.2.jar b/core/lib/asm-3.2/lib/all/asm-debug-all-3.2.jar
similarity index 100%
rename from third_party/asm-3.2/lib/all/asm-debug-all-3.2.jar
rename to core/lib/asm-3.2/lib/all/asm-debug-all-3.2.jar
diff --git a/third_party/asm-3.2/lib/all/asm-debug-all-3.2.pom b/core/lib/asm-3.2/lib/all/asm-debug-all-3.2.pom
similarity index 100%
rename from third_party/asm-3.2/lib/all/asm-debug-all-3.2.pom
rename to core/lib/asm-3.2/lib/all/asm-debug-all-3.2.pom
diff --git a/third_party/asm-3.2/lib/asm-3.2.jar b/core/lib/asm-3.2/lib/asm-3.2.jar
similarity index 100%
rename from third_party/asm-3.2/lib/asm-3.2.jar
rename to core/lib/asm-3.2/lib/asm-3.2.jar
diff --git a/third_party/asm-3.2/lib/asm-3.2.pom b/core/lib/asm-3.2/lib/asm-3.2.pom
similarity index 100%
rename from third_party/asm-3.2/lib/asm-3.2.pom
rename to core/lib/asm-3.2/lib/asm-3.2.pom
diff --git a/third_party/asm-3.2/lib/asm-analysis-3.2.jar b/core/lib/asm-3.2/lib/asm-analysis-3.2.jar
similarity index 100%
rename from third_party/asm-3.2/lib/asm-analysis-3.2.jar
rename to core/lib/asm-3.2/lib/asm-analysis-3.2.jar
diff --git a/third_party/asm-3.2/lib/asm-analysis-3.2.pom b/core/lib/asm-3.2/lib/asm-analysis-3.2.pom
similarity index 100%
rename from third_party/asm-3.2/lib/asm-analysis-3.2.pom
rename to core/lib/asm-3.2/lib/asm-analysis-3.2.pom
diff --git a/third_party/asm-3.2/lib/asm-commons-3.2.jar b/core/lib/asm-3.2/lib/asm-commons-3.2.jar
similarity index 100%
rename from third_party/asm-3.2/lib/asm-commons-3.2.jar
rename to core/lib/asm-3.2/lib/asm-commons-3.2.jar
diff --git a/third_party/asm-3.2/lib/asm-commons-3.2.pom b/core/lib/asm-3.2/lib/asm-commons-3.2.pom
similarity index 100%
rename from third_party/asm-3.2/lib/asm-commons-3.2.pom
rename to core/lib/asm-3.2/lib/asm-commons-3.2.pom
diff --git a/third_party/asm-3.2/lib/asm-parent-3.2.pom b/core/lib/asm-3.2/lib/asm-parent-3.2.pom
similarity index 100%
rename from third_party/asm-3.2/lib/asm-parent-3.2.pom
rename to core/lib/asm-3.2/lib/asm-parent-3.2.pom
diff --git a/third_party/asm-3.2/lib/asm-tree-3.2.jar b/core/lib/asm-3.2/lib/asm-tree-3.2.jar
similarity index 100%
rename from third_party/asm-3.2/lib/asm-tree-3.2.jar
rename to core/lib/asm-3.2/lib/asm-tree-3.2.jar
diff --git a/third_party/asm-3.2/lib/asm-tree-3.2.pom b/core/lib/asm-3.2/lib/asm-tree-3.2.pom
similarity index 100%
rename from third_party/asm-3.2/lib/asm-tree-3.2.pom
rename to core/lib/asm-3.2/lib/asm-tree-3.2.pom
diff --git a/third_party/asm-3.2/lib/asm-util-3.2.jar b/core/lib/asm-3.2/lib/asm-util-3.2.jar
similarity index 100%
rename from third_party/asm-3.2/lib/asm-util-3.2.jar
rename to core/lib/asm-3.2/lib/asm-util-3.2.jar
diff --git a/third_party/asm-3.2/lib/asm-util-3.2.pom b/core/lib/asm-3.2/lib/asm-util-3.2.pom
similarity index 100%
rename from third_party/asm-3.2/lib/asm-util-3.2.pom
rename to core/lib/asm-3.2/lib/asm-util-3.2.pom
diff --git a/third_party/asm-3.2/lib/asm-xml-3.2.jar b/core/lib/asm-3.2/lib/asm-xml-3.2.jar
similarity index 100%
rename from third_party/asm-3.2/lib/asm-xml-3.2.jar
rename to core/lib/asm-3.2/lib/asm-xml-3.2.jar
diff --git a/third_party/asm-3.2/lib/asm-xml-3.2.pom b/core/lib/asm-3.2/lib/asm-xml-3.2.pom
similarity index 100%
rename from third_party/asm-3.2/lib/asm-xml-3.2.pom
rename to core/lib/asm-3.2/lib/asm-xml-3.2.pom
diff --git a/third_party/colt.jar b/core/lib/colt.jar
similarity index 100%
rename from third_party/colt.jar
rename to core/lib/colt.jar
diff --git a/third_party/compress-lzf-0.6.0/LICENSE b/core/lib/compress-lzf-0.6.0/LICENSE
similarity index 100%
rename from third_party/compress-lzf-0.6.0/LICENSE
rename to core/lib/compress-lzf-0.6.0/LICENSE
diff --git a/third_party/compress-lzf-0.6.0/compress-lzf-0.6.0.jar b/core/lib/compress-lzf-0.6.0/compress-lzf-0.6.0.jar
similarity index 100%
rename from third_party/compress-lzf-0.6.0/compress-lzf-0.6.0.jar
rename to core/lib/compress-lzf-0.6.0/compress-lzf-0.6.0.jar
diff --git a/third_party/guava-r07/COPYING b/core/lib/guava-r07/COPYING
similarity index 100%
rename from third_party/guava-r07/COPYING
rename to core/lib/guava-r07/COPYING
diff --git a/third_party/guava-r07/README b/core/lib/guava-r07/README
similarity index 100%
rename from third_party/guava-r07/README
rename to core/lib/guava-r07/README
diff --git a/third_party/guava-r07/guava-r07.jar b/core/lib/guava-r07/guava-r07.jar
similarity index 100%
rename from third_party/guava-r07/guava-r07.jar
rename to core/lib/guava-r07/guava-r07.jar
diff --git a/third_party/hadoop-0.20.0/.DS_Store b/core/lib/hadoop-0.20.0/.DS_Store
similarity index 100%
rename from third_party/hadoop-0.20.0/.DS_Store
rename to core/lib/hadoop-0.20.0/.DS_Store
diff --git a/third_party/hadoop-0.20.0/CHANGES.txt b/core/lib/hadoop-0.20.0/CHANGES.txt
similarity index 100%
rename from third_party/hadoop-0.20.0/CHANGES.txt
rename to core/lib/hadoop-0.20.0/CHANGES.txt
diff --git a/third_party/hadoop-0.20.0/LICENSE.txt b/core/lib/hadoop-0.20.0/LICENSE.txt
similarity index 100%
rename from third_party/hadoop-0.20.0/LICENSE.txt
rename to core/lib/hadoop-0.20.0/LICENSE.txt
diff --git a/third_party/hadoop-0.20.0/NOTICE.txt b/core/lib/hadoop-0.20.0/NOTICE.txt
similarity index 100%
rename from third_party/hadoop-0.20.0/NOTICE.txt
rename to core/lib/hadoop-0.20.0/NOTICE.txt
diff --git a/third_party/hadoop-0.20.0/README.txt b/core/lib/hadoop-0.20.0/README.txt
similarity index 100%
rename from third_party/hadoop-0.20.0/README.txt
rename to core/lib/hadoop-0.20.0/README.txt
diff --git a/third_party/hadoop-0.20.0/bin/hadoop b/core/lib/hadoop-0.20.0/bin/hadoop
similarity index 100%
rename from third_party/hadoop-0.20.0/bin/hadoop
rename to core/lib/hadoop-0.20.0/bin/hadoop
diff --git a/third_party/hadoop-0.20.0/bin/hadoop-config.sh b/core/lib/hadoop-0.20.0/bin/hadoop-config.sh
similarity index 100%
rename from third_party/hadoop-0.20.0/bin/hadoop-config.sh
rename to core/lib/hadoop-0.20.0/bin/hadoop-config.sh
diff --git a/third_party/hadoop-0.20.0/bin/hadoop-daemon.sh b/core/lib/hadoop-0.20.0/bin/hadoop-daemon.sh
similarity index 100%
rename from third_party/hadoop-0.20.0/bin/hadoop-daemon.sh
rename to core/lib/hadoop-0.20.0/bin/hadoop-daemon.sh
diff --git a/third_party/hadoop-0.20.0/bin/hadoop-daemons.sh b/core/lib/hadoop-0.20.0/bin/hadoop-daemons.sh
similarity index 100%
rename from third_party/hadoop-0.20.0/bin/hadoop-daemons.sh
rename to core/lib/hadoop-0.20.0/bin/hadoop-daemons.sh
diff --git a/third_party/hadoop-0.20.0/bin/rcc b/core/lib/hadoop-0.20.0/bin/rcc
similarity index 100%
rename from third_party/hadoop-0.20.0/bin/rcc
rename to core/lib/hadoop-0.20.0/bin/rcc
diff --git a/third_party/hadoop-0.20.0/bin/slaves.sh b/core/lib/hadoop-0.20.0/bin/slaves.sh
similarity index 100%
rename from third_party/hadoop-0.20.0/bin/slaves.sh
rename to core/lib/hadoop-0.20.0/bin/slaves.sh
diff --git a/third_party/hadoop-0.20.0/bin/start-all.sh b/core/lib/hadoop-0.20.0/bin/start-all.sh
similarity index 100%
rename from third_party/hadoop-0.20.0/bin/start-all.sh
rename to core/lib/hadoop-0.20.0/bin/start-all.sh
diff --git a/third_party/hadoop-0.20.0/bin/start-balancer.sh b/core/lib/hadoop-0.20.0/bin/start-balancer.sh
similarity index 100%
rename from third_party/hadoop-0.20.0/bin/start-balancer.sh
rename to core/lib/hadoop-0.20.0/bin/start-balancer.sh
diff --git a/third_party/hadoop-0.20.0/bin/start-dfs.sh b/core/lib/hadoop-0.20.0/bin/start-dfs.sh
similarity index 100%
rename from third_party/hadoop-0.20.0/bin/start-dfs.sh
rename to core/lib/hadoop-0.20.0/bin/start-dfs.sh
diff --git a/third_party/hadoop-0.20.0/bin/start-mapred.sh b/core/lib/hadoop-0.20.0/bin/start-mapred.sh
similarity index 100%
rename from third_party/hadoop-0.20.0/bin/start-mapred.sh
rename to core/lib/hadoop-0.20.0/bin/start-mapred.sh
diff --git a/third_party/hadoop-0.20.0/bin/stop-all.sh b/core/lib/hadoop-0.20.0/bin/stop-all.sh
similarity index 100%
rename from third_party/hadoop-0.20.0/bin/stop-all.sh
rename to core/lib/hadoop-0.20.0/bin/stop-all.sh
diff --git a/third_party/hadoop-0.20.0/bin/stop-balancer.sh b/core/lib/hadoop-0.20.0/bin/stop-balancer.sh
similarity index 100%
rename from third_party/hadoop-0.20.0/bin/stop-balancer.sh
rename to core/lib/hadoop-0.20.0/bin/stop-balancer.sh
diff --git a/third_party/hadoop-0.20.0/bin/stop-dfs.sh b/core/lib/hadoop-0.20.0/bin/stop-dfs.sh
similarity index 100%
rename from third_party/hadoop-0.20.0/bin/stop-dfs.sh
rename to core/lib/hadoop-0.20.0/bin/stop-dfs.sh
diff --git a/third_party/hadoop-0.20.0/bin/stop-mapred.sh b/core/lib/hadoop-0.20.0/bin/stop-mapred.sh
similarity index 100%
rename from third_party/hadoop-0.20.0/bin/stop-mapred.sh
rename to core/lib/hadoop-0.20.0/bin/stop-mapred.sh
diff --git a/third_party/hadoop-0.20.0/build.xml b/core/lib/hadoop-0.20.0/build.xml
similarity index 100%
rename from third_party/hadoop-0.20.0/build.xml
rename to core/lib/hadoop-0.20.0/build.xml
diff --git a/third_party/hadoop-0.20.0/c++/Linux-amd64-64/include/hadoop/Pipes.hh b/core/lib/hadoop-0.20.0/c++/Linux-amd64-64/include/hadoop/Pipes.hh
similarity index 100%
rename from third_party/hadoop-0.20.0/c++/Linux-amd64-64/include/hadoop/Pipes.hh
rename to core/lib/hadoop-0.20.0/c++/Linux-amd64-64/include/hadoop/Pipes.hh
diff --git a/third_party/hadoop-0.20.0/c++/Linux-amd64-64/include/hadoop/SerialUtils.hh b/core/lib/hadoop-0.20.0/c++/Linux-amd64-64/include/hadoop/SerialUtils.hh
similarity index 100%
rename from third_party/hadoop-0.20.0/c++/Linux-amd64-64/include/hadoop/SerialUtils.hh
rename to core/lib/hadoop-0.20.0/c++/Linux-amd64-64/include/hadoop/SerialUtils.hh
diff --git a/third_party/hadoop-0.20.0/c++/Linux-amd64-64/include/hadoop/StringUtils.hh b/core/lib/hadoop-0.20.0/c++/Linux-amd64-64/include/hadoop/StringUtils.hh
similarity index 100%
rename from third_party/hadoop-0.20.0/c++/Linux-amd64-64/include/hadoop/StringUtils.hh
rename to core/lib/hadoop-0.20.0/c++/Linux-amd64-64/include/hadoop/StringUtils.hh
diff --git a/third_party/hadoop-0.20.0/c++/Linux-amd64-64/include/hadoop/TemplateFactory.hh b/core/lib/hadoop-0.20.0/c++/Linux-amd64-64/include/hadoop/TemplateFactory.hh
similarity index 100%
rename from third_party/hadoop-0.20.0/c++/Linux-amd64-64/include/hadoop/TemplateFactory.hh
rename to core/lib/hadoop-0.20.0/c++/Linux-amd64-64/include/hadoop/TemplateFactory.hh
diff --git a/third_party/hadoop-0.20.0/c++/Linux-amd64-64/lib/libhadooppipes.a b/core/lib/hadoop-0.20.0/c++/Linux-amd64-64/lib/libhadooppipes.a
similarity index 100%
rename from third_party/hadoop-0.20.0/c++/Linux-amd64-64/lib/libhadooppipes.a
rename to core/lib/hadoop-0.20.0/c++/Linux-amd64-64/lib/libhadooppipes.a
diff --git a/third_party/hadoop-0.20.0/c++/Linux-amd64-64/lib/libhadooputils.a b/core/lib/hadoop-0.20.0/c++/Linux-amd64-64/lib/libhadooputils.a
similarity index 100%
rename from third_party/hadoop-0.20.0/c++/Linux-amd64-64/lib/libhadooputils.a
rename to core/lib/hadoop-0.20.0/c++/Linux-amd64-64/lib/libhadooputils.a
diff --git a/third_party/hadoop-0.20.0/c++/Linux-i386-32/include/hadoop/Pipes.hh b/core/lib/hadoop-0.20.0/c++/Linux-i386-32/include/hadoop/Pipes.hh
similarity index 100%
rename from third_party/hadoop-0.20.0/c++/Linux-i386-32/include/hadoop/Pipes.hh
rename to core/lib/hadoop-0.20.0/c++/Linux-i386-32/include/hadoop/Pipes.hh
diff --git a/third_party/hadoop-0.20.0/c++/Linux-i386-32/include/hadoop/SerialUtils.hh b/core/lib/hadoop-0.20.0/c++/Linux-i386-32/include/hadoop/SerialUtils.hh
similarity index 100%
rename from third_party/hadoop-0.20.0/c++/Linux-i386-32/include/hadoop/SerialUtils.hh
rename to core/lib/hadoop-0.20.0/c++/Linux-i386-32/include/hadoop/SerialUtils.hh
diff --git a/third_party/hadoop-0.20.0/c++/Linux-i386-32/include/hadoop/StringUtils.hh b/core/lib/hadoop-0.20.0/c++/Linux-i386-32/include/hadoop/StringUtils.hh
similarity index 100%
rename from third_party/hadoop-0.20.0/c++/Linux-i386-32/include/hadoop/StringUtils.hh
rename to core/lib/hadoop-0.20.0/c++/Linux-i386-32/include/hadoop/StringUtils.hh
diff --git a/third_party/hadoop-0.20.0/c++/Linux-i386-32/include/hadoop/TemplateFactory.hh b/core/lib/hadoop-0.20.0/c++/Linux-i386-32/include/hadoop/TemplateFactory.hh
similarity index 100%
rename from third_party/hadoop-0.20.0/c++/Linux-i386-32/include/hadoop/TemplateFactory.hh
rename to core/lib/hadoop-0.20.0/c++/Linux-i386-32/include/hadoop/TemplateFactory.hh
diff --git a/third_party/hadoop-0.20.0/c++/Linux-i386-32/lib/libhadooppipes.a b/core/lib/hadoop-0.20.0/c++/Linux-i386-32/lib/libhadooppipes.a
similarity index 100%
rename from third_party/hadoop-0.20.0/c++/Linux-i386-32/lib/libhadooppipes.a
rename to core/lib/hadoop-0.20.0/c++/Linux-i386-32/lib/libhadooppipes.a
diff --git a/third_party/hadoop-0.20.0/c++/Linux-i386-32/lib/libhadooputils.a b/core/lib/hadoop-0.20.0/c++/Linux-i386-32/lib/libhadooputils.a
similarity index 100%
rename from third_party/hadoop-0.20.0/c++/Linux-i386-32/lib/libhadooputils.a
rename to core/lib/hadoop-0.20.0/c++/Linux-i386-32/lib/libhadooputils.a
diff --git a/third_party/hadoop-0.20.0/c++/Linux-i386-32/lib/libhdfs.la b/core/lib/hadoop-0.20.0/c++/Linux-i386-32/lib/libhdfs.la
similarity index 100%
rename from third_party/hadoop-0.20.0/c++/Linux-i386-32/lib/libhdfs.la
rename to core/lib/hadoop-0.20.0/c++/Linux-i386-32/lib/libhdfs.la
diff --git a/third_party/hadoop-0.20.0/c++/Linux-i386-32/lib/libhdfs.so b/core/lib/hadoop-0.20.0/c++/Linux-i386-32/lib/libhdfs.so
similarity index 100%
rename from third_party/hadoop-0.20.0/c++/Linux-i386-32/lib/libhdfs.so
rename to core/lib/hadoop-0.20.0/c++/Linux-i386-32/lib/libhdfs.so
diff --git a/third_party/hadoop-0.20.0/c++/Linux-i386-32/lib/libhdfs.so.0 b/core/lib/hadoop-0.20.0/c++/Linux-i386-32/lib/libhdfs.so.0
similarity index 100%
rename from third_party/hadoop-0.20.0/c++/Linux-i386-32/lib/libhdfs.so.0
rename to core/lib/hadoop-0.20.0/c++/Linux-i386-32/lib/libhdfs.so.0
diff --git a/third_party/hadoop-0.20.0/c++/Linux-i386-32/lib/libhdfs.so.0.0.0 b/core/lib/hadoop-0.20.0/c++/Linux-i386-32/lib/libhdfs.so.0.0.0
similarity index 100%
rename from third_party/hadoop-0.20.0/c++/Linux-i386-32/lib/libhdfs.so.0.0.0
rename to core/lib/hadoop-0.20.0/c++/Linux-i386-32/lib/libhdfs.so.0.0.0
diff --git a/third_party/hadoop-0.20.0/conf/capacity-scheduler.xml b/core/lib/hadoop-0.20.0/conf/capacity-scheduler.xml
similarity index 100%
rename from third_party/hadoop-0.20.0/conf/capacity-scheduler.xml
rename to core/lib/hadoop-0.20.0/conf/capacity-scheduler.xml
diff --git a/third_party/hadoop-0.20.0/conf/configuration.xsl b/core/lib/hadoop-0.20.0/conf/configuration.xsl
similarity index 100%
rename from third_party/hadoop-0.20.0/conf/configuration.xsl
rename to core/lib/hadoop-0.20.0/conf/configuration.xsl
diff --git a/third_party/hadoop-0.20.0/conf/core-site.xml b/core/lib/hadoop-0.20.0/conf/core-site.xml
similarity index 100%
rename from third_party/hadoop-0.20.0/conf/core-site.xml
rename to core/lib/hadoop-0.20.0/conf/core-site.xml
diff --git a/third_party/hadoop-0.20.0/conf/hadoop-env.sh b/core/lib/hadoop-0.20.0/conf/hadoop-env.sh
similarity index 100%
rename from third_party/hadoop-0.20.0/conf/hadoop-env.sh
rename to core/lib/hadoop-0.20.0/conf/hadoop-env.sh
diff --git a/third_party/hadoop-0.20.0/conf/hadoop-metrics.properties b/core/lib/hadoop-0.20.0/conf/hadoop-metrics.properties
similarity index 100%
rename from third_party/hadoop-0.20.0/conf/hadoop-metrics.properties
rename to core/lib/hadoop-0.20.0/conf/hadoop-metrics.properties
diff --git a/third_party/hadoop-0.20.0/conf/hadoop-policy.xml b/core/lib/hadoop-0.20.0/conf/hadoop-policy.xml
similarity index 100%
rename from third_party/hadoop-0.20.0/conf/hadoop-policy.xml
rename to core/lib/hadoop-0.20.0/conf/hadoop-policy.xml
diff --git a/third_party/hadoop-0.20.0/conf/hdfs-site.xml b/core/lib/hadoop-0.20.0/conf/hdfs-site.xml
similarity index 100%
rename from third_party/hadoop-0.20.0/conf/hdfs-site.xml
rename to core/lib/hadoop-0.20.0/conf/hdfs-site.xml
diff --git a/third_party/hadoop-0.20.0/conf/log4j.properties b/core/lib/hadoop-0.20.0/conf/log4j.properties
similarity index 100%
rename from third_party/hadoop-0.20.0/conf/log4j.properties
rename to core/lib/hadoop-0.20.0/conf/log4j.properties
diff --git a/third_party/hadoop-0.20.0/conf/mapred-site.xml b/core/lib/hadoop-0.20.0/conf/mapred-site.xml
similarity index 100%
rename from third_party/hadoop-0.20.0/conf/mapred-site.xml
rename to core/lib/hadoop-0.20.0/conf/mapred-site.xml
diff --git a/third_party/hadoop-0.20.0/conf/masters b/core/lib/hadoop-0.20.0/conf/masters
similarity index 100%
rename from third_party/hadoop-0.20.0/conf/masters
rename to core/lib/hadoop-0.20.0/conf/masters
diff --git a/third_party/hadoop-0.20.0/conf/slaves b/core/lib/hadoop-0.20.0/conf/slaves
similarity index 100%
rename from third_party/hadoop-0.20.0/conf/slaves
rename to core/lib/hadoop-0.20.0/conf/slaves
diff --git a/third_party/hadoop-0.20.0/conf/ssl-client.xml.example b/core/lib/hadoop-0.20.0/conf/ssl-client.xml.example
similarity index 100%
rename from third_party/hadoop-0.20.0/conf/ssl-client.xml.example
rename to core/lib/hadoop-0.20.0/conf/ssl-client.xml.example
diff --git a/third_party/hadoop-0.20.0/conf/ssl-server.xml.example b/core/lib/hadoop-0.20.0/conf/ssl-server.xml.example
similarity index 100%
rename from third_party/hadoop-0.20.0/conf/ssl-server.xml.example
rename to core/lib/hadoop-0.20.0/conf/ssl-server.xml.example
diff --git a/third_party/hadoop-0.20.0/contrib/capacity-scheduler/hadoop-0.20.0-capacity-scheduler.jar b/core/lib/hadoop-0.20.0/contrib/capacity-scheduler/hadoop-0.20.0-capacity-scheduler.jar
similarity index 100%
rename from third_party/hadoop-0.20.0/contrib/capacity-scheduler/hadoop-0.20.0-capacity-scheduler.jar
rename to core/lib/hadoop-0.20.0/contrib/capacity-scheduler/hadoop-0.20.0-capacity-scheduler.jar
diff --git a/third_party/hadoop-0.20.0/contrib/datajoin/hadoop-0.20.0-datajoin.jar b/core/lib/hadoop-0.20.0/contrib/datajoin/hadoop-0.20.0-datajoin.jar
similarity index 100%
rename from third_party/hadoop-0.20.0/contrib/datajoin/hadoop-0.20.0-datajoin.jar
rename to core/lib/hadoop-0.20.0/contrib/datajoin/hadoop-0.20.0-datajoin.jar
diff --git a/third_party/hadoop-0.20.0/contrib/eclipse-plugin/hadoop-0.20.0-eclipse-plugin.jar b/core/lib/hadoop-0.20.0/contrib/eclipse-plugin/hadoop-0.20.0-eclipse-plugin.jar
similarity index 100%
rename from third_party/hadoop-0.20.0/contrib/eclipse-plugin/hadoop-0.20.0-eclipse-plugin.jar
rename to core/lib/hadoop-0.20.0/contrib/eclipse-plugin/hadoop-0.20.0-eclipse-plugin.jar
diff --git a/third_party/hadoop-0.20.0/contrib/fairscheduler/hadoop-0.20.0-fairscheduler.jar b/core/lib/hadoop-0.20.0/contrib/fairscheduler/hadoop-0.20.0-fairscheduler.jar
similarity index 100%
rename from third_party/hadoop-0.20.0/contrib/fairscheduler/hadoop-0.20.0-fairscheduler.jar
rename to core/lib/hadoop-0.20.0/contrib/fairscheduler/hadoop-0.20.0-fairscheduler.jar
diff --git a/third_party/hadoop-0.20.0/contrib/hdfsproxy/README b/core/lib/hadoop-0.20.0/contrib/hdfsproxy/README
similarity index 100%
rename from third_party/hadoop-0.20.0/contrib/hdfsproxy/README
rename to core/lib/hadoop-0.20.0/contrib/hdfsproxy/README
diff --git a/third_party/hadoop-0.20.0/contrib/hdfsproxy/bin/hdfsproxy b/core/lib/hadoop-0.20.0/contrib/hdfsproxy/bin/hdfsproxy
similarity index 100%
rename from third_party/hadoop-0.20.0/contrib/hdfsproxy/bin/hdfsproxy
rename to core/lib/hadoop-0.20.0/contrib/hdfsproxy/bin/hdfsproxy
diff --git a/third_party/hadoop-0.20.0/contrib/hdfsproxy/bin/hdfsproxy-config.sh b/core/lib/hadoop-0.20.0/contrib/hdfsproxy/bin/hdfsproxy-config.sh
similarity index 100%
rename from third_party/hadoop-0.20.0/contrib/hdfsproxy/bin/hdfsproxy-config.sh
rename to core/lib/hadoop-0.20.0/contrib/hdfsproxy/bin/hdfsproxy-config.sh
diff --git a/third_party/hadoop-0.20.0/contrib/hdfsproxy/bin/hdfsproxy-daemon.sh b/core/lib/hadoop-0.20.0/contrib/hdfsproxy/bin/hdfsproxy-daemon.sh
similarity index 100%
rename from third_party/hadoop-0.20.0/contrib/hdfsproxy/bin/hdfsproxy-daemon.sh
rename to core/lib/hadoop-0.20.0/contrib/hdfsproxy/bin/hdfsproxy-daemon.sh
diff --git a/third_party/hadoop-0.20.0/contrib/hdfsproxy/bin/hdfsproxy-daemons.sh b/core/lib/hadoop-0.20.0/contrib/hdfsproxy/bin/hdfsproxy-daemons.sh
similarity index 100%
rename from third_party/hadoop-0.20.0/contrib/hdfsproxy/bin/hdfsproxy-daemons.sh
rename to core/lib/hadoop-0.20.0/contrib/hdfsproxy/bin/hdfsproxy-daemons.sh
diff --git a/third_party/hadoop-0.20.0/contrib/hdfsproxy/bin/hdfsproxy-slaves.sh b/core/lib/hadoop-0.20.0/contrib/hdfsproxy/bin/hdfsproxy-slaves.sh
similarity index 100%
rename from third_party/hadoop-0.20.0/contrib/hdfsproxy/bin/hdfsproxy-slaves.sh
rename to core/lib/hadoop-0.20.0/contrib/hdfsproxy/bin/hdfsproxy-slaves.sh
diff --git a/third_party/hadoop-0.20.0/contrib/hdfsproxy/bin/start-hdfsproxy.sh b/core/lib/hadoop-0.20.0/contrib/hdfsproxy/bin/start-hdfsproxy.sh
similarity index 100%
rename from third_party/hadoop-0.20.0/contrib/hdfsproxy/bin/start-hdfsproxy.sh
rename to core/lib/hadoop-0.20.0/contrib/hdfsproxy/bin/start-hdfsproxy.sh
diff --git a/third_party/hadoop-0.20.0/contrib/hdfsproxy/bin/stop-hdfsproxy.sh b/core/lib/hadoop-0.20.0/contrib/hdfsproxy/bin/stop-hdfsproxy.sh
similarity index 100%
rename from third_party/hadoop-0.20.0/contrib/hdfsproxy/bin/stop-hdfsproxy.sh
rename to core/lib/hadoop-0.20.0/contrib/hdfsproxy/bin/stop-hdfsproxy.sh
diff --git a/third_party/hadoop-0.20.0/contrib/hdfsproxy/build.xml b/core/lib/hadoop-0.20.0/contrib/hdfsproxy/build.xml
similarity index 100%
rename from third_party/hadoop-0.20.0/contrib/hdfsproxy/build.xml
rename to core/lib/hadoop-0.20.0/contrib/hdfsproxy/build.xml
diff --git a/third_party/hadoop-0.20.0/contrib/hdfsproxy/conf/configuration.xsl b/core/lib/hadoop-0.20.0/contrib/hdfsproxy/conf/configuration.xsl
similarity index 100%
rename from third_party/hadoop-0.20.0/contrib/hdfsproxy/conf/configuration.xsl
rename to core/lib/hadoop-0.20.0/contrib/hdfsproxy/conf/configuration.xsl
diff --git a/third_party/hadoop-0.20.0/contrib/hdfsproxy/conf/hdfsproxy-default.xml b/core/lib/hadoop-0.20.0/contrib/hdfsproxy/conf/hdfsproxy-default.xml
similarity index 100%
rename from third_party/hadoop-0.20.0/contrib/hdfsproxy/conf/hdfsproxy-default.xml
rename to core/lib/hadoop-0.20.0/contrib/hdfsproxy/conf/hdfsproxy-default.xml
diff --git a/third_party/hadoop-0.20.0/contrib/hdfsproxy/conf/hdfsproxy-env.sh b/core/lib/hadoop-0.20.0/contrib/hdfsproxy/conf/hdfsproxy-env.sh
similarity index 100%
rename from third_party/hadoop-0.20.0/contrib/hdfsproxy/conf/hdfsproxy-env.sh
rename to core/lib/hadoop-0.20.0/contrib/hdfsproxy/conf/hdfsproxy-env.sh
diff --git a/third_party/hadoop-0.20.0/contrib/hdfsproxy/conf/hdfsproxy-env.sh.template b/core/lib/hadoop-0.20.0/contrib/hdfsproxy/conf/hdfsproxy-env.sh.template
similarity index 100%
rename from third_party/hadoop-0.20.0/contrib/hdfsproxy/conf/hdfsproxy-env.sh.template
rename to core/lib/hadoop-0.20.0/contrib/hdfsproxy/conf/hdfsproxy-env.sh.template
diff --git a/third_party/hadoop-0.20.0/contrib/hdfsproxy/conf/hdfsproxy-hosts b/core/lib/hadoop-0.20.0/contrib/hdfsproxy/conf/hdfsproxy-hosts
similarity index 100%
rename from third_party/hadoop-0.20.0/contrib/hdfsproxy/conf/hdfsproxy-hosts
rename to core/lib/hadoop-0.20.0/contrib/hdfsproxy/conf/hdfsproxy-hosts
diff --git a/third_party/hadoop-0.20.0/contrib/hdfsproxy/conf/log4j.properties b/core/lib/hadoop-0.20.0/contrib/hdfsproxy/conf/log4j.properties
similarity index 100%
rename from third_party/hadoop-0.20.0/contrib/hdfsproxy/conf/log4j.properties
rename to core/lib/hadoop-0.20.0/contrib/hdfsproxy/conf/log4j.properties
diff --git a/third_party/hadoop-0.20.0/contrib/hdfsproxy/conf/user-certs.xml b/core/lib/hadoop-0.20.0/contrib/hdfsproxy/conf/user-certs.xml
similarity index 100%
rename from third_party/hadoop-0.20.0/contrib/hdfsproxy/conf/user-certs.xml
rename to core/lib/hadoop-0.20.0/contrib/hdfsproxy/conf/user-certs.xml
diff --git a/third_party/hadoop-0.20.0/contrib/hdfsproxy/conf/user-permissions.xml b/core/lib/hadoop-0.20.0/contrib/hdfsproxy/conf/user-permissions.xml
similarity index 100%
rename from third_party/hadoop-0.20.0/contrib/hdfsproxy/conf/user-permissions.xml
rename to core/lib/hadoop-0.20.0/contrib/hdfsproxy/conf/user-permissions.xml
diff --git a/third_party/hadoop-0.20.0/contrib/hdfsproxy/hdfsproxy-1.0.jar b/core/lib/hadoop-0.20.0/contrib/hdfsproxy/hdfsproxy-1.0.jar
similarity index 100%
rename from third_party/hadoop-0.20.0/contrib/hdfsproxy/hdfsproxy-1.0.jar
rename to core/lib/hadoop-0.20.0/contrib/hdfsproxy/hdfsproxy-1.0.jar
diff --git a/third_party/hadoop-0.20.0/contrib/index/hadoop-0.20.0-index.jar b/core/lib/hadoop-0.20.0/contrib/index/hadoop-0.20.0-index.jar
similarity index 100%
rename from third_party/hadoop-0.20.0/contrib/index/hadoop-0.20.0-index.jar
rename to core/lib/hadoop-0.20.0/contrib/index/hadoop-0.20.0-index.jar
diff --git a/third_party/hadoop-0.20.0/contrib/streaming/hadoop-0.20.0-streaming.jar b/core/lib/hadoop-0.20.0/contrib/streaming/hadoop-0.20.0-streaming.jar
similarity index 100%
rename from third_party/hadoop-0.20.0/contrib/streaming/hadoop-0.20.0-streaming.jar
rename to core/lib/hadoop-0.20.0/contrib/streaming/hadoop-0.20.0-streaming.jar
diff --git a/third_party/hadoop-0.20.0/contrib/thriftfs/hadoop-0.20.0-thriftfs.jar b/core/lib/hadoop-0.20.0/contrib/thriftfs/hadoop-0.20.0-thriftfs.jar
similarity index 100%
rename from third_party/hadoop-0.20.0/contrib/thriftfs/hadoop-0.20.0-thriftfs.jar
rename to core/lib/hadoop-0.20.0/contrib/thriftfs/hadoop-0.20.0-thriftfs.jar
diff --git a/third_party/hadoop-0.20.0/contrib/vaidya/bin/vaidya.sh b/core/lib/hadoop-0.20.0/contrib/vaidya/bin/vaidya.sh
similarity index 100%
rename from third_party/hadoop-0.20.0/contrib/vaidya/bin/vaidya.sh
rename to core/lib/hadoop-0.20.0/contrib/vaidya/bin/vaidya.sh
diff --git a/third_party/hadoop-0.20.0/contrib/vaidya/conf/postex_diagnosis_tests.xml b/core/lib/hadoop-0.20.0/contrib/vaidya/conf/postex_diagnosis_tests.xml
similarity index 100%
rename from third_party/hadoop-0.20.0/contrib/vaidya/conf/postex_diagnosis_tests.xml
rename to core/lib/hadoop-0.20.0/contrib/vaidya/conf/postex_diagnosis_tests.xml
diff --git a/third_party/hadoop-0.20.0/contrib/vaidya/hadoop-0.20.0-vaidya.jar b/core/lib/hadoop-0.20.0/contrib/vaidya/hadoop-0.20.0-vaidya.jar
similarity index 100%
rename from third_party/hadoop-0.20.0/contrib/vaidya/hadoop-0.20.0-vaidya.jar
rename to core/lib/hadoop-0.20.0/contrib/vaidya/hadoop-0.20.0-vaidya.jar
diff --git a/third_party/hadoop-0.20.0/hadoop-0.20.0-ant.jar b/core/lib/hadoop-0.20.0/hadoop-0.20.0-ant.jar
similarity index 100%
rename from third_party/hadoop-0.20.0/hadoop-0.20.0-ant.jar
rename to core/lib/hadoop-0.20.0/hadoop-0.20.0-ant.jar
diff --git a/third_party/hadoop-0.20.0/hadoop-0.20.0-core.jar b/core/lib/hadoop-0.20.0/hadoop-0.20.0-core.jar
similarity index 100%
rename from third_party/hadoop-0.20.0/hadoop-0.20.0-core.jar
rename to core/lib/hadoop-0.20.0/hadoop-0.20.0-core.jar
diff --git a/third_party/hadoop-0.20.0/hadoop-0.20.0-examples.jar b/core/lib/hadoop-0.20.0/hadoop-0.20.0-examples.jar
similarity index 100%
rename from third_party/hadoop-0.20.0/hadoop-0.20.0-examples.jar
rename to core/lib/hadoop-0.20.0/hadoop-0.20.0-examples.jar
diff --git a/third_party/hadoop-0.20.0/hadoop-0.20.0-test.jar b/core/lib/hadoop-0.20.0/hadoop-0.20.0-test.jar
similarity index 100%
rename from third_party/hadoop-0.20.0/hadoop-0.20.0-test.jar
rename to core/lib/hadoop-0.20.0/hadoop-0.20.0-test.jar
diff --git a/third_party/hadoop-0.20.0/hadoop-0.20.0-tools.jar b/core/lib/hadoop-0.20.0/hadoop-0.20.0-tools.jar
similarity index 100%
rename from third_party/hadoop-0.20.0/hadoop-0.20.0-tools.jar
rename to core/lib/hadoop-0.20.0/hadoop-0.20.0-tools.jar
diff --git a/third_party/hadoop-0.20.0/ivy.xml b/core/lib/hadoop-0.20.0/ivy.xml
similarity index 100%
rename from third_party/hadoop-0.20.0/ivy.xml
rename to core/lib/hadoop-0.20.0/ivy.xml
diff --git a/third_party/hadoop-0.20.0/ivy/hadoop-core.pom b/core/lib/hadoop-0.20.0/ivy/hadoop-core.pom
similarity index 100%
rename from third_party/hadoop-0.20.0/ivy/hadoop-core.pom
rename to core/lib/hadoop-0.20.0/ivy/hadoop-core.pom
diff --git a/third_party/hadoop-0.20.0/ivy/ivy-2.0.0-rc2.jar b/core/lib/hadoop-0.20.0/ivy/ivy-2.0.0-rc2.jar
similarity index 100%
rename from third_party/hadoop-0.20.0/ivy/ivy-2.0.0-rc2.jar
rename to core/lib/hadoop-0.20.0/ivy/ivy-2.0.0-rc2.jar
diff --git a/third_party/hadoop-0.20.0/ivy/ivysettings.xml b/core/lib/hadoop-0.20.0/ivy/ivysettings.xml
similarity index 100%
rename from third_party/hadoop-0.20.0/ivy/ivysettings.xml
rename to core/lib/hadoop-0.20.0/ivy/ivysettings.xml
diff --git a/third_party/hadoop-0.20.0/ivy/libraries.properties b/core/lib/hadoop-0.20.0/ivy/libraries.properties
similarity index 100%
rename from third_party/hadoop-0.20.0/ivy/libraries.properties
rename to core/lib/hadoop-0.20.0/ivy/libraries.properties
diff --git a/third_party/hadoop-0.20.0/lib/.DS_Store b/core/lib/hadoop-0.20.0/lib/.DS_Store
similarity index 100%
rename from third_party/hadoop-0.20.0/lib/.DS_Store
rename to core/lib/hadoop-0.20.0/lib/.DS_Store
diff --git a/third_party/hadoop-0.20.0/lib/commons-cli-2.0-SNAPSHOT.jar b/core/lib/hadoop-0.20.0/lib/commons-cli-2.0-SNAPSHOT.jar
similarity index 100%
rename from third_party/hadoop-0.20.0/lib/commons-cli-2.0-SNAPSHOT.jar
rename to core/lib/hadoop-0.20.0/lib/commons-cli-2.0-SNAPSHOT.jar
diff --git a/third_party/hadoop-0.20.0/lib/commons-codec-1.3.jar b/core/lib/hadoop-0.20.0/lib/commons-codec-1.3.jar
similarity index 100%
rename from third_party/hadoop-0.20.0/lib/commons-codec-1.3.jar
rename to core/lib/hadoop-0.20.0/lib/commons-codec-1.3.jar
diff --git a/third_party/hadoop-0.20.0/lib/commons-el-1.0.jar b/core/lib/hadoop-0.20.0/lib/commons-el-1.0.jar
similarity index 100%
rename from third_party/hadoop-0.20.0/lib/commons-el-1.0.jar
rename to core/lib/hadoop-0.20.0/lib/commons-el-1.0.jar
diff --git a/third_party/hadoop-0.20.0/lib/commons-httpclient-3.0.1.jar b/core/lib/hadoop-0.20.0/lib/commons-httpclient-3.0.1.jar
similarity index 100%
rename from third_party/hadoop-0.20.0/lib/commons-httpclient-3.0.1.jar
rename to core/lib/hadoop-0.20.0/lib/commons-httpclient-3.0.1.jar
diff --git a/third_party/hadoop-0.20.0/lib/commons-logging-1.0.4.jar b/core/lib/hadoop-0.20.0/lib/commons-logging-1.0.4.jar
similarity index 100%
rename from third_party/hadoop-0.20.0/lib/commons-logging-1.0.4.jar
rename to core/lib/hadoop-0.20.0/lib/commons-logging-1.0.4.jar
diff --git a/third_party/hadoop-0.20.0/lib/commons-logging-api-1.0.4.jar b/core/lib/hadoop-0.20.0/lib/commons-logging-api-1.0.4.jar
similarity index 100%
rename from third_party/hadoop-0.20.0/lib/commons-logging-api-1.0.4.jar
rename to core/lib/hadoop-0.20.0/lib/commons-logging-api-1.0.4.jar
diff --git a/third_party/hadoop-0.20.0/lib/commons-net-1.4.1.jar b/core/lib/hadoop-0.20.0/lib/commons-net-1.4.1.jar
similarity index 100%
rename from third_party/hadoop-0.20.0/lib/commons-net-1.4.1.jar
rename to core/lib/hadoop-0.20.0/lib/commons-net-1.4.1.jar
diff --git a/third_party/hadoop-0.20.0/lib/core-3.1.1.jar b/core/lib/hadoop-0.20.0/lib/core-3.1.1.jar
similarity index 100%
rename from third_party/hadoop-0.20.0/lib/core-3.1.1.jar
rename to core/lib/hadoop-0.20.0/lib/core-3.1.1.jar
diff --git a/third_party/hadoop-0.20.0/lib/hsqldb-1.8.0.10.LICENSE.txt b/core/lib/hadoop-0.20.0/lib/hsqldb-1.8.0.10.LICENSE.txt
similarity index 100%
rename from third_party/hadoop-0.20.0/lib/hsqldb-1.8.0.10.LICENSE.txt
rename to core/lib/hadoop-0.20.0/lib/hsqldb-1.8.0.10.LICENSE.txt
diff --git a/third_party/hadoop-0.20.0/lib/hsqldb-1.8.0.10.jar b/core/lib/hadoop-0.20.0/lib/hsqldb-1.8.0.10.jar
similarity index 100%
rename from third_party/hadoop-0.20.0/lib/hsqldb-1.8.0.10.jar
rename to core/lib/hadoop-0.20.0/lib/hsqldb-1.8.0.10.jar
diff --git a/third_party/hadoop-0.20.0/lib/jasper-compiler-5.5.12.jar b/core/lib/hadoop-0.20.0/lib/jasper-compiler-5.5.12.jar
similarity index 100%
rename from third_party/hadoop-0.20.0/lib/jasper-compiler-5.5.12.jar
rename to core/lib/hadoop-0.20.0/lib/jasper-compiler-5.5.12.jar
diff --git a/third_party/hadoop-0.20.0/lib/jasper-runtime-5.5.12.jar b/core/lib/hadoop-0.20.0/lib/jasper-runtime-5.5.12.jar
similarity index 100%
rename from third_party/hadoop-0.20.0/lib/jasper-runtime-5.5.12.jar
rename to core/lib/hadoop-0.20.0/lib/jasper-runtime-5.5.12.jar
diff --git a/third_party/hadoop-0.20.0/lib/jdiff/hadoop_0.17.0.xml b/core/lib/hadoop-0.20.0/lib/jdiff/hadoop_0.17.0.xml
similarity index 100%
rename from third_party/hadoop-0.20.0/lib/jdiff/hadoop_0.17.0.xml
rename to core/lib/hadoop-0.20.0/lib/jdiff/hadoop_0.17.0.xml
diff --git a/third_party/hadoop-0.20.0/lib/jdiff/hadoop_0.18.1.xml b/core/lib/hadoop-0.20.0/lib/jdiff/hadoop_0.18.1.xml
similarity index 100%
rename from third_party/hadoop-0.20.0/lib/jdiff/hadoop_0.18.1.xml
rename to core/lib/hadoop-0.20.0/lib/jdiff/hadoop_0.18.1.xml
diff --git a/third_party/hadoop-0.20.0/lib/jdiff/hadoop_0.18.2.xml b/core/lib/hadoop-0.20.0/lib/jdiff/hadoop_0.18.2.xml
similarity index 100%
rename from third_party/hadoop-0.20.0/lib/jdiff/hadoop_0.18.2.xml
rename to core/lib/hadoop-0.20.0/lib/jdiff/hadoop_0.18.2.xml
diff --git a/third_party/hadoop-0.20.0/lib/jdiff/hadoop_0.18.3.xml b/core/lib/hadoop-0.20.0/lib/jdiff/hadoop_0.18.3.xml
similarity index 100%
rename from third_party/hadoop-0.20.0/lib/jdiff/hadoop_0.18.3.xml
rename to core/lib/hadoop-0.20.0/lib/jdiff/hadoop_0.18.3.xml
diff --git a/third_party/hadoop-0.20.0/lib/jdiff/hadoop_0.19.0.xml b/core/lib/hadoop-0.20.0/lib/jdiff/hadoop_0.19.0.xml
similarity index 100%
rename from third_party/hadoop-0.20.0/lib/jdiff/hadoop_0.19.0.xml
rename to core/lib/hadoop-0.20.0/lib/jdiff/hadoop_0.19.0.xml
diff --git a/third_party/hadoop-0.20.0/lib/jdiff/hadoop_0.19.1.xml b/core/lib/hadoop-0.20.0/lib/jdiff/hadoop_0.19.1.xml
similarity index 100%
rename from third_party/hadoop-0.20.0/lib/jdiff/hadoop_0.19.1.xml
rename to core/lib/hadoop-0.20.0/lib/jdiff/hadoop_0.19.1.xml
diff --git a/third_party/hadoop-0.20.0/lib/jdiff/hadoop_0.20.0.xml b/core/lib/hadoop-0.20.0/lib/jdiff/hadoop_0.20.0.xml
similarity index 100%
rename from third_party/hadoop-0.20.0/lib/jdiff/hadoop_0.20.0.xml
rename to core/lib/hadoop-0.20.0/lib/jdiff/hadoop_0.20.0.xml
diff --git a/third_party/hadoop-0.20.0/lib/jets3t-0.6.1.jar b/core/lib/hadoop-0.20.0/lib/jets3t-0.6.1.jar
similarity index 100%
rename from third_party/hadoop-0.20.0/lib/jets3t-0.6.1.jar
rename to core/lib/hadoop-0.20.0/lib/jets3t-0.6.1.jar
diff --git a/third_party/hadoop-0.20.0/lib/jetty-6.1.14.jar b/core/lib/hadoop-0.20.0/lib/jetty-6.1.14.jar
similarity index 100%
rename from third_party/hadoop-0.20.0/lib/jetty-6.1.14.jar
rename to core/lib/hadoop-0.20.0/lib/jetty-6.1.14.jar
diff --git a/third_party/hadoop-0.20.0/lib/jetty-util-6.1.14.jar b/core/lib/hadoop-0.20.0/lib/jetty-util-6.1.14.jar
similarity index 100%
rename from third_party/hadoop-0.20.0/lib/jetty-util-6.1.14.jar
rename to core/lib/hadoop-0.20.0/lib/jetty-util-6.1.14.jar
diff --git a/third_party/hadoop-0.20.0/lib/jsp-2.1/jsp-2.1.jar b/core/lib/hadoop-0.20.0/lib/jsp-2.1/jsp-2.1.jar
similarity index 100%
rename from third_party/hadoop-0.20.0/lib/jsp-2.1/jsp-2.1.jar
rename to core/lib/hadoop-0.20.0/lib/jsp-2.1/jsp-2.1.jar
diff --git a/third_party/hadoop-0.20.0/lib/jsp-2.1/jsp-api-2.1.jar b/core/lib/hadoop-0.20.0/lib/jsp-2.1/jsp-api-2.1.jar
similarity index 100%
rename from third_party/hadoop-0.20.0/lib/jsp-2.1/jsp-api-2.1.jar
rename to core/lib/hadoop-0.20.0/lib/jsp-2.1/jsp-api-2.1.jar
diff --git a/third_party/hadoop-0.20.0/lib/junit-3.8.1.jar b/core/lib/hadoop-0.20.0/lib/junit-3.8.1.jar
similarity index 100%
rename from third_party/hadoop-0.20.0/lib/junit-3.8.1.jar
rename to core/lib/hadoop-0.20.0/lib/junit-3.8.1.jar
diff --git a/third_party/hadoop-0.20.0/lib/kfs-0.2.2.jar b/core/lib/hadoop-0.20.0/lib/kfs-0.2.2.jar
similarity index 100%
rename from third_party/hadoop-0.20.0/lib/kfs-0.2.2.jar
rename to core/lib/hadoop-0.20.0/lib/kfs-0.2.2.jar
diff --git a/third_party/hadoop-0.20.0/lib/kfs-0.2.LICENSE.txt b/core/lib/hadoop-0.20.0/lib/kfs-0.2.LICENSE.txt
similarity index 100%
rename from third_party/hadoop-0.20.0/lib/kfs-0.2.LICENSE.txt
rename to core/lib/hadoop-0.20.0/lib/kfs-0.2.LICENSE.txt
diff --git a/third_party/hadoop-0.20.0/lib/log4j-1.2.15.jar b/core/lib/hadoop-0.20.0/lib/log4j-1.2.15.jar
similarity index 100%
rename from third_party/hadoop-0.20.0/lib/log4j-1.2.15.jar
rename to core/lib/hadoop-0.20.0/lib/log4j-1.2.15.jar
diff --git a/third_party/hadoop-0.20.0/lib/native/.DS_Store b/core/lib/hadoop-0.20.0/lib/native/.DS_Store
similarity index 100%
rename from third_party/hadoop-0.20.0/lib/native/.DS_Store
rename to core/lib/hadoop-0.20.0/lib/native/.DS_Store
diff --git a/third_party/hadoop-0.20.0/lib/native/Linux-amd64-64/libhadoop.a b/core/lib/hadoop-0.20.0/lib/native/Linux-amd64-64/libhadoop.a
similarity index 100%
rename from third_party/hadoop-0.20.0/lib/native/Linux-amd64-64/libhadoop.a
rename to core/lib/hadoop-0.20.0/lib/native/Linux-amd64-64/libhadoop.a
diff --git a/third_party/hadoop-0.20.0/lib/native/Linux-amd64-64/libhadoop.la b/core/lib/hadoop-0.20.0/lib/native/Linux-amd64-64/libhadoop.la
similarity index 100%
rename from third_party/hadoop-0.20.0/lib/native/Linux-amd64-64/libhadoop.la
rename to core/lib/hadoop-0.20.0/lib/native/Linux-amd64-64/libhadoop.la
diff --git a/third_party/hadoop-0.20.0/lib/native/Linux-amd64-64/libhadoop.so b/core/lib/hadoop-0.20.0/lib/native/Linux-amd64-64/libhadoop.so
similarity index 100%
rename from third_party/hadoop-0.20.0/lib/native/Linux-amd64-64/libhadoop.so
rename to core/lib/hadoop-0.20.0/lib/native/Linux-amd64-64/libhadoop.so
diff --git a/third_party/hadoop-0.20.0/lib/native/Linux-amd64-64/libhadoop.so.1 b/core/lib/hadoop-0.20.0/lib/native/Linux-amd64-64/libhadoop.so.1
similarity index 100%
rename from third_party/hadoop-0.20.0/lib/native/Linux-amd64-64/libhadoop.so.1
rename to core/lib/hadoop-0.20.0/lib/native/Linux-amd64-64/libhadoop.so.1
diff --git a/third_party/hadoop-0.20.0/lib/native/Linux-amd64-64/libhadoop.so.1.0.0 b/core/lib/hadoop-0.20.0/lib/native/Linux-amd64-64/libhadoop.so.1.0.0
similarity index 100%
rename from third_party/hadoop-0.20.0/lib/native/Linux-amd64-64/libhadoop.so.1.0.0
rename to core/lib/hadoop-0.20.0/lib/native/Linux-amd64-64/libhadoop.so.1.0.0
diff --git a/third_party/hadoop-0.20.0/lib/native/Linux-i386-32/libhadoop.a b/core/lib/hadoop-0.20.0/lib/native/Linux-i386-32/libhadoop.a
similarity index 100%
rename from third_party/hadoop-0.20.0/lib/native/Linux-i386-32/libhadoop.a
rename to core/lib/hadoop-0.20.0/lib/native/Linux-i386-32/libhadoop.a
diff --git a/third_party/hadoop-0.20.0/lib/native/Linux-i386-32/libhadoop.la b/core/lib/hadoop-0.20.0/lib/native/Linux-i386-32/libhadoop.la
similarity index 100%
rename from third_party/hadoop-0.20.0/lib/native/Linux-i386-32/libhadoop.la
rename to core/lib/hadoop-0.20.0/lib/native/Linux-i386-32/libhadoop.la
diff --git a/third_party/hadoop-0.20.0/lib/native/Linux-i386-32/libhadoop.so b/core/lib/hadoop-0.20.0/lib/native/Linux-i386-32/libhadoop.so
similarity index 100%
rename from third_party/hadoop-0.20.0/lib/native/Linux-i386-32/libhadoop.so
rename to core/lib/hadoop-0.20.0/lib/native/Linux-i386-32/libhadoop.so
diff --git a/third_party/hadoop-0.20.0/lib/native/Linux-i386-32/libhadoop.so.1 b/core/lib/hadoop-0.20.0/lib/native/Linux-i386-32/libhadoop.so.1
similarity index 100%
rename from third_party/hadoop-0.20.0/lib/native/Linux-i386-32/libhadoop.so.1
rename to core/lib/hadoop-0.20.0/lib/native/Linux-i386-32/libhadoop.so.1
diff --git a/third_party/hadoop-0.20.0/lib/native/Linux-i386-32/libhadoop.so.1.0.0 b/core/lib/hadoop-0.20.0/lib/native/Linux-i386-32/libhadoop.so.1.0.0
similarity index 100%
rename from third_party/hadoop-0.20.0/lib/native/Linux-i386-32/libhadoop.so.1.0.0
rename to core/lib/hadoop-0.20.0/lib/native/Linux-i386-32/libhadoop.so.1.0.0
diff --git a/third_party/hadoop-0.20.0/lib/oro-2.0.8.jar b/core/lib/hadoop-0.20.0/lib/oro-2.0.8.jar
similarity index 100%
rename from third_party/hadoop-0.20.0/lib/oro-2.0.8.jar
rename to core/lib/hadoop-0.20.0/lib/oro-2.0.8.jar
diff --git a/third_party/hadoop-0.20.0/lib/servlet-api-2.5-6.1.14.jar b/core/lib/hadoop-0.20.0/lib/servlet-api-2.5-6.1.14.jar
similarity index 100%
rename from third_party/hadoop-0.20.0/lib/servlet-api-2.5-6.1.14.jar
rename to core/lib/hadoop-0.20.0/lib/servlet-api-2.5-6.1.14.jar
diff --git a/third_party/hadoop-0.20.0/lib/xmlenc-0.52.jar b/core/lib/hadoop-0.20.0/lib/xmlenc-0.52.jar
similarity index 100%
rename from third_party/hadoop-0.20.0/lib/xmlenc-0.52.jar
rename to core/lib/hadoop-0.20.0/lib/xmlenc-0.52.jar
diff --git a/third_party/hadoop-0.20.0/librecordio/librecordio.a b/core/lib/hadoop-0.20.0/librecordio/librecordio.a
similarity index 100%
rename from third_party/hadoop-0.20.0/librecordio/librecordio.a
rename to core/lib/hadoop-0.20.0/librecordio/librecordio.a
diff --git a/third_party/hadoop-0.20.0/webapps/datanode/WEB-INF/web.xml b/core/lib/hadoop-0.20.0/webapps/datanode/WEB-INF/web.xml
similarity index 100%
rename from third_party/hadoop-0.20.0/webapps/datanode/WEB-INF/web.xml
rename to core/lib/hadoop-0.20.0/webapps/datanode/WEB-INF/web.xml
diff --git a/third_party/hadoop-0.20.0/webapps/hdfs/WEB-INF/web.xml b/core/lib/hadoop-0.20.0/webapps/hdfs/WEB-INF/web.xml
similarity index 100%
rename from third_party/hadoop-0.20.0/webapps/hdfs/WEB-INF/web.xml
rename to core/lib/hadoop-0.20.0/webapps/hdfs/WEB-INF/web.xml
diff --git a/third_party/hadoop-0.20.0/webapps/hdfs/index.html b/core/lib/hadoop-0.20.0/webapps/hdfs/index.html
similarity index 100%
rename from third_party/hadoop-0.20.0/webapps/hdfs/index.html
rename to core/lib/hadoop-0.20.0/webapps/hdfs/index.html
diff --git a/third_party/hadoop-0.20.0/webapps/job/WEB-INF/web.xml b/core/lib/hadoop-0.20.0/webapps/job/WEB-INF/web.xml
similarity index 100%
rename from third_party/hadoop-0.20.0/webapps/job/WEB-INF/web.xml
rename to core/lib/hadoop-0.20.0/webapps/job/WEB-INF/web.xml
diff --git a/third_party/hadoop-0.20.0/webapps/job/index.html b/core/lib/hadoop-0.20.0/webapps/job/index.html
similarity index 100%
rename from third_party/hadoop-0.20.0/webapps/job/index.html
rename to core/lib/hadoop-0.20.0/webapps/job/index.html
diff --git a/third_party/hadoop-0.20.0/webapps/static/hadoop-logo.jpg b/core/lib/hadoop-0.20.0/webapps/static/hadoop-logo.jpg
similarity index 100%
rename from third_party/hadoop-0.20.0/webapps/static/hadoop-logo.jpg
rename to core/lib/hadoop-0.20.0/webapps/static/hadoop-logo.jpg
diff --git a/third_party/hadoop-0.20.0/webapps/static/hadoop.css b/core/lib/hadoop-0.20.0/webapps/static/hadoop.css
similarity index 100%
rename from third_party/hadoop-0.20.0/webapps/static/hadoop.css
rename to core/lib/hadoop-0.20.0/webapps/static/hadoop.css
diff --git a/third_party/hadoop-0.20.0/webapps/static/jobconf.xsl b/core/lib/hadoop-0.20.0/webapps/static/jobconf.xsl
similarity index 100%
rename from third_party/hadoop-0.20.0/webapps/static/jobconf.xsl
rename to core/lib/hadoop-0.20.0/webapps/static/jobconf.xsl
diff --git a/third_party/hadoop-0.20.0/webapps/static/jobtracker.js b/core/lib/hadoop-0.20.0/webapps/static/jobtracker.js
similarity index 100%
rename from third_party/hadoop-0.20.0/webapps/static/jobtracker.js
rename to core/lib/hadoop-0.20.0/webapps/static/jobtracker.js
diff --git a/third_party/hadoop-0.20.0/webapps/task/WEB-INF/web.xml b/core/lib/hadoop-0.20.0/webapps/task/WEB-INF/web.xml
similarity index 100%
rename from third_party/hadoop-0.20.0/webapps/task/WEB-INF/web.xml
rename to core/lib/hadoop-0.20.0/webapps/task/WEB-INF/web.xml
diff --git a/third_party/hadoop-0.20.0/webapps/task/index.html b/core/lib/hadoop-0.20.0/webapps/task/index.html
similarity index 100%
rename from third_party/hadoop-0.20.0/webapps/task/index.html
rename to core/lib/hadoop-0.20.0/webapps/task/index.html
diff --git a/third_party/jetty-7.1.6.v20100715/jetty-server-7.1.6.v20100715.jar b/core/lib/jetty-7.1.6.v20100715/jetty-server-7.1.6.v20100715.jar
similarity index 100%
rename from third_party/jetty-7.1.6.v20100715/jetty-server-7.1.6.v20100715.jar
rename to core/lib/jetty-7.1.6.v20100715/jetty-server-7.1.6.v20100715.jar
diff --git a/third_party/jetty-7.1.6.v20100715/servlet-api-2.5.jar b/core/lib/jetty-7.1.6.v20100715/servlet-api-2.5.jar
similarity index 100%
rename from third_party/jetty-7.1.6.v20100715/servlet-api-2.5.jar
rename to core/lib/jetty-7.1.6.v20100715/servlet-api-2.5.jar
diff --git a/core/lib/jline.jar b/core/lib/jline.jar
new file mode 100644
index 0000000000000000000000000000000000000000..6ed67faab68b4ba09e264bdaaaa610971479438b
Binary files /dev/null and b/core/lib/jline.jar differ
diff --git a/third_party/liblzf-3.5/Changes b/core/lib/liblzf-3.5/Changes
similarity index 100%
rename from third_party/liblzf-3.5/Changes
rename to core/lib/liblzf-3.5/Changes
diff --git a/third_party/liblzf-3.5/LICENSE b/core/lib/liblzf-3.5/LICENSE
similarity index 100%
rename from third_party/liblzf-3.5/LICENSE
rename to core/lib/liblzf-3.5/LICENSE
diff --git a/third_party/liblzf-3.5/Makefile b/core/lib/liblzf-3.5/Makefile
similarity index 100%
rename from third_party/liblzf-3.5/Makefile
rename to core/lib/liblzf-3.5/Makefile
diff --git a/third_party/liblzf-3.5/Makefile.in b/core/lib/liblzf-3.5/Makefile.in
similarity index 100%
rename from third_party/liblzf-3.5/Makefile.in
rename to core/lib/liblzf-3.5/Makefile.in
diff --git a/third_party/liblzf-3.5/README b/core/lib/liblzf-3.5/README
similarity index 100%
rename from third_party/liblzf-3.5/README
rename to core/lib/liblzf-3.5/README
diff --git a/third_party/liblzf-3.5/config.h b/core/lib/liblzf-3.5/config.h
similarity index 100%
rename from third_party/liblzf-3.5/config.h
rename to core/lib/liblzf-3.5/config.h
diff --git a/third_party/liblzf-3.5/config.h.in b/core/lib/liblzf-3.5/config.h.in
similarity index 100%
rename from third_party/liblzf-3.5/config.h.in
rename to core/lib/liblzf-3.5/config.h.in
diff --git a/third_party/liblzf-3.5/config.log b/core/lib/liblzf-3.5/config.log
similarity index 100%
rename from third_party/liblzf-3.5/config.log
rename to core/lib/liblzf-3.5/config.log
diff --git a/third_party/liblzf-3.5/config.status b/core/lib/liblzf-3.5/config.status
similarity index 100%
rename from third_party/liblzf-3.5/config.status
rename to core/lib/liblzf-3.5/config.status
diff --git a/third_party/liblzf-3.5/configure b/core/lib/liblzf-3.5/configure
similarity index 100%
rename from third_party/liblzf-3.5/configure
rename to core/lib/liblzf-3.5/configure
diff --git a/third_party/liblzf-3.5/configure.ac b/core/lib/liblzf-3.5/configure.ac
similarity index 100%
rename from third_party/liblzf-3.5/configure.ac
rename to core/lib/liblzf-3.5/configure.ac
diff --git a/third_party/liblzf-3.5/crc32.h b/core/lib/liblzf-3.5/crc32.h
similarity index 100%
rename from third_party/liblzf-3.5/crc32.h
rename to core/lib/liblzf-3.5/crc32.h
diff --git a/third_party/liblzf-3.5/cs/CLZF.cs b/core/lib/liblzf-3.5/cs/CLZF.cs
similarity index 100%
rename from third_party/liblzf-3.5/cs/CLZF.cs
rename to core/lib/liblzf-3.5/cs/CLZF.cs
diff --git a/third_party/liblzf-3.5/cs/README b/core/lib/liblzf-3.5/cs/README
similarity index 100%
rename from third_party/liblzf-3.5/cs/README
rename to core/lib/liblzf-3.5/cs/README
diff --git a/third_party/liblzf-3.5/install-sh b/core/lib/liblzf-3.5/install-sh
similarity index 100%
rename from third_party/liblzf-3.5/install-sh
rename to core/lib/liblzf-3.5/install-sh
diff --git a/third_party/liblzf-3.5/lzf.c b/core/lib/liblzf-3.5/lzf.c
similarity index 100%
rename from third_party/liblzf-3.5/lzf.c
rename to core/lib/liblzf-3.5/lzf.c
diff --git a/third_party/liblzf-3.5/lzf.h b/core/lib/liblzf-3.5/lzf.h
similarity index 100%
rename from third_party/liblzf-3.5/lzf.h
rename to core/lib/liblzf-3.5/lzf.h
diff --git a/third_party/liblzf-3.5/lzfP.h b/core/lib/liblzf-3.5/lzfP.h
similarity index 100%
rename from third_party/liblzf-3.5/lzfP.h
rename to core/lib/liblzf-3.5/lzfP.h
diff --git a/third_party/liblzf-3.5/lzf_c.c b/core/lib/liblzf-3.5/lzf_c.c
similarity index 100%
rename from third_party/liblzf-3.5/lzf_c.c
rename to core/lib/liblzf-3.5/lzf_c.c
diff --git a/third_party/liblzf-3.5/lzf_d.c b/core/lib/liblzf-3.5/lzf_d.c
similarity index 100%
rename from third_party/liblzf-3.5/lzf_d.c
rename to core/lib/liblzf-3.5/lzf_d.c
diff --git a/third_party/mesos.jar b/core/lib/mesos.jar
similarity index 100%
rename from third_party/mesos.jar
rename to core/lib/mesos.jar
diff --git a/third_party/scalacheck_2.8.0-1.7.jar b/core/lib/scalacheck_2.8.0-1.7.jar
similarity index 100%
rename from third_party/scalacheck_2.8.0-1.7.jar
rename to core/lib/scalacheck_2.8.0-1.7.jar
diff --git a/third_party/scalatest-1.2/LICENSE b/core/lib/scalatest-1.2/LICENSE
similarity index 100%
rename from third_party/scalatest-1.2/LICENSE
rename to core/lib/scalatest-1.2/LICENSE
diff --git a/third_party/scalatest-1.2/NOTICE b/core/lib/scalatest-1.2/NOTICE
similarity index 100%
rename from third_party/scalatest-1.2/NOTICE
rename to core/lib/scalatest-1.2/NOTICE
diff --git a/third_party/scalatest-1.2/README.txt b/core/lib/scalatest-1.2/README.txt
similarity index 100%
rename from third_party/scalatest-1.2/README.txt
rename to core/lib/scalatest-1.2/README.txt
diff --git a/third_party/scalatest-1.2/scalatest-1.2.jar b/core/lib/scalatest-1.2/scalatest-1.2.jar
similarity index 100%
rename from third_party/scalatest-1.2/scalatest-1.2.jar
rename to core/lib/scalatest-1.2/scalatest-1.2.jar
diff --git a/third_party/slf4j-1.6.1/slf4j-api-1.6.1.jar b/core/lib/slf4j-1.6.1/slf4j-api-1.6.1.jar
similarity index 100%
rename from third_party/slf4j-1.6.1/slf4j-api-1.6.1.jar
rename to core/lib/slf4j-1.6.1/slf4j-api-1.6.1.jar
diff --git a/third_party/slf4j-1.6.1/slf4j-log4j12-1.6.1.jar b/core/lib/slf4j-1.6.1/slf4j-log4j12-1.6.1.jar
similarity index 100%
rename from third_party/slf4j-1.6.1/slf4j-log4j12-1.6.1.jar
rename to core/lib/slf4j-1.6.1/slf4j-log4j12-1.6.1.jar
diff --git a/src/scala/spark/Accumulators.scala b/core/src/main/scala/spark/Accumulators.scala
similarity index 100%
rename from src/scala/spark/Accumulators.scala
rename to core/src/main/scala/spark/Accumulators.scala
diff --git a/src/scala/spark/BitTorrentBroadcast.scala b/core/src/main/scala/spark/BitTorrentBroadcast.scala
similarity index 100%
rename from src/scala/spark/BitTorrentBroadcast.scala
rename to core/src/main/scala/spark/BitTorrentBroadcast.scala
diff --git a/src/scala/spark/BoundedMemoryCache.scala b/core/src/main/scala/spark/BoundedMemoryCache.scala
similarity index 100%
rename from src/scala/spark/BoundedMemoryCache.scala
rename to core/src/main/scala/spark/BoundedMemoryCache.scala
diff --git a/src/scala/spark/Broadcast.scala b/core/src/main/scala/spark/Broadcast.scala
similarity index 100%
rename from src/scala/spark/Broadcast.scala
rename to core/src/main/scala/spark/Broadcast.scala
diff --git a/src/scala/spark/Cache.scala b/core/src/main/scala/spark/Cache.scala
similarity index 100%
rename from src/scala/spark/Cache.scala
rename to core/src/main/scala/spark/Cache.scala
diff --git a/src/scala/spark/ChainedBroadcast.scala b/core/src/main/scala/spark/ChainedBroadcast.scala
similarity index 100%
rename from src/scala/spark/ChainedBroadcast.scala
rename to core/src/main/scala/spark/ChainedBroadcast.scala
diff --git a/src/scala/spark/ClosureCleaner.scala b/core/src/main/scala/spark/ClosureCleaner.scala
similarity index 100%
rename from src/scala/spark/ClosureCleaner.scala
rename to core/src/main/scala/spark/ClosureCleaner.scala
diff --git a/src/scala/spark/DfsBroadcast.scala b/core/src/main/scala/spark/DfsBroadcast.scala
similarity index 100%
rename from src/scala/spark/DfsBroadcast.scala
rename to core/src/main/scala/spark/DfsBroadcast.scala
diff --git a/src/scala/spark/DfsShuffle.scala b/core/src/main/scala/spark/DfsShuffle.scala
similarity index 100%
rename from src/scala/spark/DfsShuffle.scala
rename to core/src/main/scala/spark/DfsShuffle.scala
diff --git a/src/scala/spark/Executor.scala b/core/src/main/scala/spark/Executor.scala
similarity index 100%
rename from src/scala/spark/Executor.scala
rename to core/src/main/scala/spark/Executor.scala
diff --git a/src/scala/spark/HadoopFile.scala b/core/src/main/scala/spark/HadoopFile.scala
similarity index 100%
rename from src/scala/spark/HadoopFile.scala
rename to core/src/main/scala/spark/HadoopFile.scala
diff --git a/src/scala/spark/HttpServer.scala b/core/src/main/scala/spark/HttpServer.scala
similarity index 100%
rename from src/scala/spark/HttpServer.scala
rename to core/src/main/scala/spark/HttpServer.scala
diff --git a/src/scala/spark/Job.scala b/core/src/main/scala/spark/Job.scala
similarity index 100%
rename from src/scala/spark/Job.scala
rename to core/src/main/scala/spark/Job.scala
diff --git a/src/scala/spark/LocalFileShuffle.scala b/core/src/main/scala/spark/LocalFileShuffle.scala
similarity index 100%
rename from src/scala/spark/LocalFileShuffle.scala
rename to core/src/main/scala/spark/LocalFileShuffle.scala
diff --git a/src/scala/spark/LocalScheduler.scala b/core/src/main/scala/spark/LocalScheduler.scala
similarity index 100%
rename from src/scala/spark/LocalScheduler.scala
rename to core/src/main/scala/spark/LocalScheduler.scala
diff --git a/src/scala/spark/Logging.scala b/core/src/main/scala/spark/Logging.scala
similarity index 100%
rename from src/scala/spark/Logging.scala
rename to core/src/main/scala/spark/Logging.scala
diff --git a/src/scala/spark/MesosScheduler.scala b/core/src/main/scala/spark/MesosScheduler.scala
similarity index 100%
rename from src/scala/spark/MesosScheduler.scala
rename to core/src/main/scala/spark/MesosScheduler.scala
diff --git a/src/scala/spark/NumberedSplitRDD.scala b/core/src/main/scala/spark/NumberedSplitRDD.scala
similarity index 100%
rename from src/scala/spark/NumberedSplitRDD.scala
rename to core/src/main/scala/spark/NumberedSplitRDD.scala
diff --git a/src/scala/spark/ParallelArray.scala b/core/src/main/scala/spark/ParallelArray.scala
similarity index 100%
rename from src/scala/spark/ParallelArray.scala
rename to core/src/main/scala/spark/ParallelArray.scala
diff --git a/src/scala/spark/RDD.scala b/core/src/main/scala/spark/RDD.scala
similarity index 99%
rename from src/scala/spark/RDD.scala
rename to core/src/main/scala/spark/RDD.scala
index bac59319a0f4fa17e1eb00ce1601d07347e04351..39f2dc44581ef68f06f283a3f5f8f7eee3e6952d 100644
--- a/src/scala/spark/RDD.scala
+++ b/core/src/main/scala/spark/RDD.scala
@@ -359,7 +359,7 @@ extends RDD[Pair[T, U]](sc) {
: RDD[(K, C)] =
{
val shufClass = Class.forName(System.getProperty(
- "spark.shuffle.class", "spark.DfsShuffle"))
+ "spark.shuffle.class", "spark.LocalFileShuffle"))
val shuf = shufClass.newInstance().asInstanceOf[Shuffle[K, V, C]]
shuf.compute(self, numSplits, createCombiner, mergeValue, mergeCombiners)
}
diff --git a/src/scala/spark/Scheduler.scala b/core/src/main/scala/spark/Scheduler.scala
similarity index 100%
rename from src/scala/spark/Scheduler.scala
rename to core/src/main/scala/spark/Scheduler.scala
diff --git a/src/scala/spark/SerializableWritable.scala b/core/src/main/scala/spark/SerializableWritable.scala
similarity index 100%
rename from src/scala/spark/SerializableWritable.scala
rename to core/src/main/scala/spark/SerializableWritable.scala
diff --git a/src/scala/spark/Shuffle.scala b/core/src/main/scala/spark/Shuffle.scala
similarity index 100%
rename from src/scala/spark/Shuffle.scala
rename to core/src/main/scala/spark/Shuffle.scala
diff --git a/src/scala/spark/SimpleJob.scala b/core/src/main/scala/spark/SimpleJob.scala
similarity index 100%
rename from src/scala/spark/SimpleJob.scala
rename to core/src/main/scala/spark/SimpleJob.scala
diff --git a/src/scala/spark/SizeEstimator.scala b/core/src/main/scala/spark/SizeEstimator.scala
similarity index 100%
rename from src/scala/spark/SizeEstimator.scala
rename to core/src/main/scala/spark/SizeEstimator.scala
diff --git a/src/scala/spark/SoftReferenceCache.scala b/core/src/main/scala/spark/SoftReferenceCache.scala
similarity index 100%
rename from src/scala/spark/SoftReferenceCache.scala
rename to core/src/main/scala/spark/SoftReferenceCache.scala
diff --git a/src/scala/spark/SparkContext.scala b/core/src/main/scala/spark/SparkContext.scala
similarity index 100%
rename from src/scala/spark/SparkContext.scala
rename to core/src/main/scala/spark/SparkContext.scala
diff --git a/src/scala/spark/SparkException.scala b/core/src/main/scala/spark/SparkException.scala
similarity index 100%
rename from src/scala/spark/SparkException.scala
rename to core/src/main/scala/spark/SparkException.scala
diff --git a/src/scala/spark/Split.scala b/core/src/main/scala/spark/Split.scala
similarity index 100%
rename from src/scala/spark/Split.scala
rename to core/src/main/scala/spark/Split.scala
diff --git a/src/scala/spark/Task.scala b/core/src/main/scala/spark/Task.scala
similarity index 100%
rename from src/scala/spark/Task.scala
rename to core/src/main/scala/spark/Task.scala
diff --git a/src/scala/spark/TaskResult.scala b/core/src/main/scala/spark/TaskResult.scala
similarity index 100%
rename from src/scala/spark/TaskResult.scala
rename to core/src/main/scala/spark/TaskResult.scala
diff --git a/src/scala/spark/Utils.scala b/core/src/main/scala/spark/Utils.scala
similarity index 100%
rename from src/scala/spark/Utils.scala
rename to core/src/main/scala/spark/Utils.scala
diff --git a/src/scala/spark/WeakReferenceCache.scala b/core/src/main/scala/spark/WeakReferenceCache.scala
similarity index 100%
rename from src/scala/spark/WeakReferenceCache.scala
rename to core/src/main/scala/spark/WeakReferenceCache.scala
diff --git a/src/scala/spark/repl/ExecutorClassLoader.scala b/core/src/main/scala/spark/repl/ExecutorClassLoader.scala
similarity index 100%
rename from src/scala/spark/repl/ExecutorClassLoader.scala
rename to core/src/main/scala/spark/repl/ExecutorClassLoader.scala
diff --git a/src/scala/spark/repl/Main.scala b/core/src/main/scala/spark/repl/Main.scala
similarity index 100%
rename from src/scala/spark/repl/Main.scala
rename to core/src/main/scala/spark/repl/Main.scala
diff --git a/src/scala/spark/repl/SparkCompletion.scala b/core/src/main/scala/spark/repl/SparkCompletion.scala
similarity index 99%
rename from src/scala/spark/repl/SparkCompletion.scala
rename to core/src/main/scala/spark/repl/SparkCompletion.scala
index d67438445b0f301719cf760271d9bc1fd03a09af..9fa41736f3c281cc3c9e70ade81b587932298e1e 100644
--- a/src/scala/spark/repl/SparkCompletion.scala
+++ b/core/src/main/scala/spark/repl/SparkCompletion.scala
@@ -310,7 +310,7 @@ class SparkCompletion(val repl: SparkInterpreter) extends SparkCompletionOutput
else xs.reduceLeft(_ zip _ takeWhile (x => x._1 == x._2) map (_._1) mkString)
// This is jline's entry point for completion.
- override def complete(_buf: String, cursor: Int, candidates: JList[String]): Int = {
+ override def complete(_buf: String, cursor: Int, candidates: java.util.List[java.lang.String]): Int = {
val buf = onull(_buf)
verbosity = if (isConsecutiveTabs(buf, cursor)) verbosity + 1 else 0
DBG("complete(%s, %d) last = (%s, %d), verbosity: %s".format(buf, cursor, lastBuf, lastCursor, verbosity))
@@ -321,7 +321,7 @@ class SparkCompletion(val repl: SparkInterpreter) extends SparkCompletionOutput
case Nil => None
case xs =>
// modify in place and return the position
- xs foreach (candidates add _)
+ xs.foreach(x => candidates.add(x))
// update the last buffer unless this is an alternatives list
if (xs contains "") Some(p.cursor)
diff --git a/src/scala/spark/repl/SparkCompletionOutput.scala b/core/src/main/scala/spark/repl/SparkCompletionOutput.scala
similarity index 100%
rename from src/scala/spark/repl/SparkCompletionOutput.scala
rename to core/src/main/scala/spark/repl/SparkCompletionOutput.scala
diff --git a/src/scala/spark/repl/SparkInteractiveReader.scala b/core/src/main/scala/spark/repl/SparkInteractiveReader.scala
similarity index 100%
rename from src/scala/spark/repl/SparkInteractiveReader.scala
rename to core/src/main/scala/spark/repl/SparkInteractiveReader.scala
diff --git a/src/scala/spark/repl/SparkInterpreter.scala b/core/src/main/scala/spark/repl/SparkInterpreter.scala
similarity index 100%
rename from src/scala/spark/repl/SparkInterpreter.scala
rename to core/src/main/scala/spark/repl/SparkInterpreter.scala
diff --git a/src/scala/spark/repl/SparkInterpreterLoop.scala b/core/src/main/scala/spark/repl/SparkInterpreterLoop.scala
similarity index 99%
rename from src/scala/spark/repl/SparkInterpreterLoop.scala
rename to core/src/main/scala/spark/repl/SparkInterpreterLoop.scala
index 5bad0a37daaee8c7ac87d66870f4fe166cd2e816..d4974009ce055afc5d682fe952f5c78f7055d61f 100644
--- a/src/scala/spark/repl/SparkInterpreterLoop.scala
+++ b/core/src/main/scala/spark/repl/SparkInterpreterLoop.scala
@@ -129,7 +129,8 @@ extends InterpreterControl {
settings.classpath append addedClasspath
interpreter = new SparkInterpreter(settings, out) {
- override protected def parentClassLoader = classOf[SparkInterpreterLoop].getClassLoader
+ override protected def parentClassLoader =
+ classOf[SparkInterpreterLoop].getClassLoader
}
interpreter.setContextClassLoader()
// interpreter.quietBind("settings", "spark.repl.SparkInterpreterSettings", interpreter.isettings)
diff --git a/src/scala/spark/repl/SparkInterpreterSettings.scala b/core/src/main/scala/spark/repl/SparkInterpreterSettings.scala
similarity index 100%
rename from src/scala/spark/repl/SparkInterpreterSettings.scala
rename to core/src/main/scala/spark/repl/SparkInterpreterSettings.scala
diff --git a/src/scala/spark/repl/SparkJLineReader.scala b/core/src/main/scala/spark/repl/SparkJLineReader.scala
similarity index 100%
rename from src/scala/spark/repl/SparkJLineReader.scala
rename to core/src/main/scala/spark/repl/SparkJLineReader.scala
diff --git a/src/scala/spark/repl/SparkSimpleReader.scala b/core/src/main/scala/spark/repl/SparkSimpleReader.scala
similarity index 100%
rename from src/scala/spark/repl/SparkSimpleReader.scala
rename to core/src/main/scala/spark/repl/SparkSimpleReader.scala
diff --git a/src/test/spark/ParallelArraySplitSuite.scala b/core/src/test/scala/spark/ParallelArraySplitSuite.scala
similarity index 100%
rename from src/test/spark/ParallelArraySplitSuite.scala
rename to core/src/test/scala/spark/ParallelArraySplitSuite.scala
diff --git a/src/test/spark/ShuffleSuite.scala b/core/src/test/scala/spark/ShuffleSuite.scala
similarity index 100%
rename from src/test/spark/ShuffleSuite.scala
rename to core/src/test/scala/spark/ShuffleSuite.scala
diff --git a/src/test/spark/repl/ReplSuite.scala b/core/src/test/scala/spark/repl/ReplSuite.scala
similarity index 89%
rename from src/test/spark/repl/ReplSuite.scala
rename to core/src/test/scala/spark/repl/ReplSuite.scala
index 8b38cde85f8ba0371dad67efe6f3bd3742220a7c..225e766c7114494bb7b67c757145c87a7c83a39e 100644
--- a/src/test/spark/repl/ReplSuite.scala
+++ b/core/src/test/scala/spark/repl/ReplSuite.scala
@@ -1,6 +1,10 @@
package spark.repl
import java.io._
+import java.net.URLClassLoader
+
+import scala.collection.mutable.ArrayBuffer
+import scala.collection.JavaConversions._
import org.scalatest.FunSuite
@@ -8,9 +12,20 @@ class ReplSuite extends FunSuite {
def runInterpreter(master: String, input: String): String = {
val in = new BufferedReader(new StringReader(input + "\n"))
val out = new StringWriter()
+ val cl = getClass.getClassLoader
+ var paths = new ArrayBuffer[String]
+ if (cl.isInstanceOf[URLClassLoader]) {
+ val urlLoader = cl.asInstanceOf[URLClassLoader]
+ for (url <- urlLoader.getURLs) {
+ if (url.getProtocol == "file") {
+ paths += url.getFile
+ }
+ }
+ }
val interp = new SparkInterpreterLoop(in, new PrintWriter(out), master)
spark.repl.Main.interp = interp
- interp.main(new Array[String](0))
+ val separator = System.getProperty("path.separator")
+ interp.main(Array("-classpath", paths.mkString(separator)))
spark.repl.Main.interp = null
return out.toString
}
diff --git a/src/examples/BroadcastTest.scala b/examples/src/main/scala/spark/examples/BroadcastTest.scala
similarity index 96%
rename from src/examples/BroadcastTest.scala
rename to examples/src/main/scala/spark/examples/BroadcastTest.scala
index 1d89b3ed13833e950e08d6f722d1be731fe20c66..f3a173b1830dabcf15992c527b4a2a1696492413 100644
--- a/src/examples/BroadcastTest.scala
+++ b/examples/src/main/scala/spark/examples/BroadcastTest.scala
@@ -1,3 +1,5 @@
+package spark.examples
+
import spark.SparkContext
object BroadcastTest {
diff --git a/src/examples/CpuHog.scala b/examples/src/main/scala/spark/examples/CpuHog.scala
similarity index 95%
rename from src/examples/CpuHog.scala
rename to examples/src/main/scala/spark/examples/CpuHog.scala
index f37c6f7824f36d54f2eed6605119dc1349c92ae4..94b370985031db78bc4b34a46d225894b8fb3b8c 100644
--- a/src/examples/CpuHog.scala
+++ b/examples/src/main/scala/spark/examples/CpuHog.scala
@@ -1,3 +1,5 @@
+package spark.examples
+
import spark._
object CpuHog {
diff --git a/src/examples/HdfsTest.scala b/examples/src/main/scala/spark/examples/HdfsTest.scala
similarity index 95%
rename from src/examples/HdfsTest.scala
rename to examples/src/main/scala/spark/examples/HdfsTest.scala
index e678154aaba1538751f334f612ae98a6806efc9b..072b4ce417f6e29cab02020deda7c83840e32a9b 100644
--- a/src/examples/HdfsTest.scala
+++ b/examples/src/main/scala/spark/examples/HdfsTest.scala
@@ -1,3 +1,5 @@
+package spark.examples
+
import spark._
object HdfsTest {
diff --git a/src/examples/LocalALS.scala b/examples/src/main/scala/spark/examples/LocalALS.scala
similarity index 99%
rename from src/examples/LocalALS.scala
rename to examples/src/main/scala/spark/examples/LocalALS.scala
index a976a5e1c516fbe88d83a3cfdd7b509053defb70..10360dab3dfe3b1298d9f1bc1563a87f208f1b52 100644
--- a/src/examples/LocalALS.scala
+++ b/examples/src/main/scala/spark/examples/LocalALS.scala
@@ -1,3 +1,5 @@
+package spark.examples
+
import java.util.Random
import scala.math.sqrt
import cern.jet.math._
diff --git a/src/examples/LocalFileLR.scala b/examples/src/main/scala/spark/examples/LocalFileLR.scala
similarity index 97%
rename from src/examples/LocalFileLR.scala
rename to examples/src/main/scala/spark/examples/LocalFileLR.scala
index 3d3bb60677abc26cc289d30e29399ea2e0e0d1dd..cc14aa7090308f060cfaa3edbc744b3871a2206c 100644
--- a/src/examples/LocalFileLR.scala
+++ b/examples/src/main/scala/spark/examples/LocalFileLR.scala
@@ -1,3 +1,5 @@
+package spark.examples
+
import java.util.Random
import Vector._
diff --git a/src/examples/LocalLR.scala b/examples/src/main/scala/spark/examples/LocalLR.scala
similarity index 97%
rename from src/examples/LocalLR.scala
rename to examples/src/main/scala/spark/examples/LocalLR.scala
index 175907e551bb00c39267c0ef46b0e808478d1f35..3fd3f88fa8384a0d1efde0d2401b88225e0ce331 100644
--- a/src/examples/LocalLR.scala
+++ b/examples/src/main/scala/spark/examples/LocalLR.scala
@@ -1,3 +1,5 @@
+package spark.examples
+
import java.util.Random
import Vector._
diff --git a/src/examples/LocalPi.scala b/examples/src/main/scala/spark/examples/LocalPi.scala
similarity index 93%
rename from src/examples/LocalPi.scala
rename to examples/src/main/scala/spark/examples/LocalPi.scala
index c61b3e53d44510d20acea7e9394a5c16d3fde9b8..9457472f2d5bea973e560bc0ab9564dc5c940656 100644
--- a/src/examples/LocalPi.scala
+++ b/examples/src/main/scala/spark/examples/LocalPi.scala
@@ -1,3 +1,5 @@
+package spark.examples
+
import scala.math.random
import spark._
import SparkContext._
diff --git a/src/examples/SleepJob.scala b/examples/src/main/scala/spark/examples/SleepJob.scala
similarity index 95%
rename from src/examples/SleepJob.scala
rename to examples/src/main/scala/spark/examples/SleepJob.scala
index a5e0ea0dc20c346e9c1344016a19e945b4667ea7..02673a5f884f5adf202ac95a96664ef2cb57e7b7 100644
--- a/src/examples/SleepJob.scala
+++ b/examples/src/main/scala/spark/examples/SleepJob.scala
@@ -1,3 +1,5 @@
+package spark.examples
+
import spark._
object SleepJob {
diff --git a/src/examples/SparkALS.scala b/examples/src/main/scala/spark/examples/SparkALS.scala
similarity index 99%
rename from src/examples/SparkALS.scala
rename to examples/src/main/scala/spark/examples/SparkALS.scala
index 6fae3c0940cf5a147c4b3b2e32a5131d67175b0b..08e042037168f8f6059e1471a32e2fbb32f9b7f2 100644
--- a/src/examples/SparkALS.scala
+++ b/examples/src/main/scala/spark/examples/SparkALS.scala
@@ -1,3 +1,5 @@
+package spark.examples
+
import java.io.Serializable
import java.util.Random
import scala.math.sqrt
diff --git a/src/examples/SparkHdfsLR.scala b/examples/src/main/scala/spark/examples/SparkHdfsLR.scala
similarity index 98%
rename from src/examples/SparkHdfsLR.scala
rename to examples/src/main/scala/spark/examples/SparkHdfsLR.scala
index f14d48b17c49e9fb261d8285627d9a45fad2595f..4c71fd084532c671a723d0730e97baa068054a3a 100644
--- a/src/examples/SparkHdfsLR.scala
+++ b/examples/src/main/scala/spark/examples/SparkHdfsLR.scala
@@ -1,3 +1,5 @@
+package spark.examples
+
import java.util.Random
import scala.math.exp
import Vector._
diff --git a/src/examples/SparkLR.scala b/examples/src/main/scala/spark/examples/SparkLR.scala
similarity index 98%
rename from src/examples/SparkLR.scala
rename to examples/src/main/scala/spark/examples/SparkLR.scala
index 71f9aea6241df284c7b38cf304e2f3dc1cd3e78a..d08f5d3f015d538eaa0a3b67215e1298334052fc 100644
--- a/src/examples/SparkLR.scala
+++ b/examples/src/main/scala/spark/examples/SparkLR.scala
@@ -1,3 +1,5 @@
+package spark.examples
+
import java.util.Random
import scala.math.exp
import Vector._
diff --git a/src/examples/SparkPi.scala b/examples/src/main/scala/spark/examples/SparkPi.scala
similarity index 96%
rename from src/examples/SparkPi.scala
rename to examples/src/main/scala/spark/examples/SparkPi.scala
index f055614125b102f6c2aa4e2fabca56b416224c1c..31c6c5b9b11d0c166c8bae6eef4e48f77da91388 100644
--- a/src/examples/SparkPi.scala
+++ b/examples/src/main/scala/spark/examples/SparkPi.scala
@@ -1,3 +1,5 @@
+package spark.examples
+
import scala.math.random
import spark._
import SparkContext._
diff --git a/src/examples/Vector.scala b/examples/src/main/scala/spark/examples/Vector.scala
similarity index 98%
rename from src/examples/Vector.scala
rename to examples/src/main/scala/spark/examples/Vector.scala
index e9fbdca7523a36e8067b1d34e69960b35039d410..ea70626e717d9b52c1e5e162120bf18513730592 100644
--- a/src/examples/Vector.scala
+++ b/examples/src/main/scala/spark/examples/Vector.scala
@@ -1,3 +1,5 @@
+package spark.examples
+
@serializable class Vector(val elements: Array[Double]) {
def length = elements.length
diff --git a/project/build.properties b/project/build.properties
new file mode 100644
index 0000000000000000000000000000000000000000..b048c01e2a7cbed0a84802af35b9381cd4ed89f0
--- /dev/null
+++ b/project/build.properties
@@ -0,0 +1,8 @@
+#Project properties
+#Sat Nov 13 21:57:32 PST 2010
+project.organization=UC Berkeley
+project.name=Spark
+sbt.version=0.7.5.RC0
+project.version=0.0.0
+build.scala.versions=2.8.1
+project.initialize=false
diff --git a/project/build/SparkProject.scala b/project/build/SparkProject.scala
new file mode 100644
index 0000000000000000000000000000000000000000..63bd3ff578f7112f54eec0b0b9462c07efeeef5b
--- /dev/null
+++ b/project/build/SparkProject.scala
@@ -0,0 +1,72 @@
+import sbt._
+import sbt.Process._
+
+import assembly._
+
+import de.element34.sbteclipsify._
+
+
+class SparkProject(info: ProjectInfo)
+extends ParentProject(info) with IdeaProject
+{
+ lazy val core = project("core", "Spark Core", new CoreProject(_))
+
+ lazy val examples =
+ project("examples", "Spark Examples", new ExamplesProject(_), core)
+
+ class CoreProject(info: ProjectInfo)
+ extends DefaultProject(info) with Eclipsify with IdeaProject with AssemblyBuilder
+ {
+ def testReportDir = outputPath / "test-report"
+
+ // Create an XML test report using ScalaTest's -u option. Unfortunately
+ // there is currently no way to call this directly from SBT without
+ // executing a subprocess.
+ lazy val testReport = task {
+ log.info("Creating " + testReportDir + "...")
+ if (!testReportDir.exists) {
+ testReportDir.asFile.mkdirs()
+ }
+
+ log.info("Executing org.scalatest.tools.Runner...")
+ val command = ("scala -classpath " + testClasspath.absString +
+ " org.scalatest.tools.Runner -o " +
+ " -u " + testReportDir.absolutePath +
+ " -p " + (outputPath / "test-classes").absolutePath)
+ val process = Process(command, path("."), "JAVA_OPTS" -> "-Xmx500m")
+ process !
+
+ None
+ }.dependsOn(compile, testCompile).describedAs("Generate XML test report.")
+
+ def singleJarExclude(base: PathFinder) = {
+ (base / "scala" ** "*") +++ ( // exclude scala library
+ (base / "META-INF" ** "*") --- // generally ignore the hell out of META-INF
+ (base / "META-INF" / "services" ** "*") --- // include all service providers
+ (base / "META-INF" / "maven" ** "*")) // include all Maven POMs and such
+ }
+
+ def singleJarTempDir = outputPath / "single-jar-classes"
+
+ def singleJarOutputPath =
+ outputPath / (name.toLowerCase.replace(" ", "-") + "-single-jar-" + version.toString + ".jar")
+
+ // Create a JAR with Spark Core and all its dependencies. We use some methods in
+ // the AssemblyBuilder plugin, but because this plugin attempts to package Scala
+ // too, we leave that out using our own exclude filter (singleJarExclude).
+ lazy val singleJar = {
+ packageTask(
+ Path.lazyPathFinder(assemblyPaths(singleJarTempDir,
+ assemblyClasspath,
+ assemblyExtraJars,
+ singleJarExclude)),
+ singleJarOutputPath,
+ packageOptions)
+ }.dependsOn(compile).describedAs("Build a single JAR with project and its dependencies")
+ }
+
+ class ExamplesProject(info: ProjectInfo)
+ extends DefaultProject(info) with Eclipsify with IdeaProject
+ {
+ }
+}
diff --git a/project/plugins/SparkProjectPlugins.scala b/project/plugins/SparkProjectPlugins.scala
new file mode 100644
index 0000000000000000000000000000000000000000..b07dfafcfb1a3b16db3b4627882974b507b28587
--- /dev/null
+++ b/project/plugins/SparkProjectPlugins.scala
@@ -0,0 +1,11 @@
+import sbt._
+
+class SparkProjectPlugins(info: ProjectInfo) extends PluginDefinition(info) {
+ val eclipse = "de.element34" % "sbt-eclipsify" % "0.7.0"
+
+ val sbtIdeaRepo = "sbt-idea-repo" at "http://mpeltonen.github.com/maven/"
+ val sbtIdea = "com.github.mpeltonen" % "sbt-idea-plugin" % "0.2.0"
+
+ val codaRepo = "Coda Hale's Repository" at "http://repo.codahale.com/"
+ val assemblySBT = "com.codahale" % "assembly-sbt" % "0.1.1"
+}
diff --git a/project/plugins/project/build.properties b/project/plugins/project/build.properties
new file mode 100644
index 0000000000000000000000000000000000000000..f834af277cc04cf017a9bd7d077c4967b0b3a00c
--- /dev/null
+++ b/project/plugins/project/build.properties
@@ -0,0 +1,3 @@
+#Project properties
+#Wed Feb 02 16:50:14 PST 2011
+plugin.uptodate=true
diff --git a/run b/run
index 5c8943c91b9aa4f153ea0e32a922092185fad3e9..c19b221ebc3711c3747486cf8326730311d44ec1 100755
--- a/run
+++ b/run
@@ -25,7 +25,7 @@ fi
# Set JAVA_OPTS to be able to load native libraries and to set heap size
JAVA_OPTS="$SPARK_JAVA_OPTS"
-JAVA_OPTS+=" -Djava.library.path=$SPARK_LIBRARY_PATH:$FWDIR/third_party:$FWDIR/src/native:$MESOS_LIBRARY_PATH"
+JAVA_OPTS+=" -Djava.library.path=$SPARK_LIBRARY_PATH:$FWDIR/lib:$FWDIR/src/main/native:$MESOS_LIBRARY_PATH"
JAVA_OPTS+=" -Xms$SPARK_MEM -Xmx$SPARK_MEM"
# Load extra JAVA_OPTS from conf/java-opts, if it exists
if [ -e $FWDIR/conf/java-opts ] ; then
@@ -33,23 +33,27 @@ if [ -e $FWDIR/conf/java-opts ] ; then
fi
export JAVA_OPTS
+CORE_DIR=$FWDIR/core
+EXAMPLES_DIR=$FWDIR/examples
+
# Build up classpath
-CLASSPATH="$SPARK_CLASSPATH:$FWDIR/build/classes:$MESOS_CLASSPATH"
+CLASSPATH="$SPARK_CLASSPATH:$CORE_DIR/target/scala_2.8.1/classes:$MESOS_CLASSPATH"
CLASSPATH+=:$FWDIR/conf
-CLASSPATH+=:$FWDIR/third_party/mesos.jar
-CLASSPATH+=:$FWDIR/third_party/asm-3.2/lib/all/asm-all-3.2.jar
-CLASSPATH+=:$FWDIR/third_party/colt.jar
-CLASSPATH+=:$FWDIR/third_party/guava-r07/guava-r07.jar
-CLASSPATH+=:$FWDIR/third_party/hadoop-0.20.0/hadoop-0.20.0-core.jar
-CLASSPATH+=:$FWDIR/third_party/scalatest-1.2/scalatest-1.2.jar
-CLASSPATH+=:$FWDIR/third_party/scalacheck_2.8.0-1.7.jar
-CLASSPATH+=:$FWDIR/third_party/jetty-7.1.6.v20100715/jetty-server-7.1.6.v20100715.jar
-CLASSPATH+=:$FWDIR/third_party/jetty-7.1.6.v20100715/servlet-api-2.5.jar
-CLASSPATH+=:$FWDIR/third_party/apache-log4j-1.2.16/log4j-1.2.16.jar
-CLASSPATH+=:$FWDIR/third_party/slf4j-1.6.1/slf4j-api-1.6.1.jar
-CLASSPATH+=:$FWDIR/third_party/slf4j-1.6.1/slf4j-log4j12-1.6.1.jar
-CLASSPATH+=:$FWDIR/third_party/compress-lzf-0.6.0/compress-lzf-0.6.0.jar
-for jar in $FWDIR/third_party/hadoop-0.20.0/lib/*.jar; do
+CLASSPATH+=:$CORE_DIR/lib/mesos.jar
+CLASSPATH+=:$CORE_DIR/lib/asm-3.2/lib/all/asm-all-3.2.jar
+CLASSPATH+=:$CORE_DIR/lib/colt.jar
+CLASSPATH+=:$CORE_DIR/lib/guava-r07/guava-r07.jar
+CLASSPATH+=:$CORE_DIR/lib/hadoop-0.20.0/hadoop-0.20.0-core.jar
+CLASSPATH+=:$CORE_DIR/lib/scalatest-1.2/scalatest-1.2.jar
+CLASSPATH+=:$CORE_DIR/lib/scalacheck_2.8.0-1.7.jar
+CLASSPATH+=:$CORE_DIR/lib/jetty-7.1.6.v20100715/jetty-server-7.1.6.v20100715.jar
+CLASSPATH+=:$CORE_DIR/lib/jetty-7.1.6.v20100715/servlet-api-2.5.jar
+CLASSPATH+=:$CORE_DIR/lib/apache-log4j-1.2.16/log4j-1.2.16.jar
+CLASSPATH+=:$CORE_DIR/lib/slf4j-1.6.1/slf4j-api-1.6.1.jar
+CLASSPATH+=:$CORE_DIR/lib/slf4j-1.6.1/slf4j-log4j12-1.6.1.jar
+CLASSPATH+=:$CORE_DIR/lib/compress-lzf-0.6.0/compress-lzf-0.6.0.jar
+CLASSPATH+=:$EXAMPLES_DIR/target/scala_2.8.1/classes
+for jar in $CORE_DIR/lib/hadoop-0.20.0/lib/*.jar; do
CLASSPATH+=:$jar
done
export CLASSPATH # Needed for spark-shell
diff --git a/sbt/sbt b/sbt/sbt
new file mode 100755
index 0000000000000000000000000000000000000000..87395fa28b07cbdfa2ca5f72ed2b5e4eb2375383
--- /dev/null
+++ b/sbt/sbt
@@ -0,0 +1,2 @@
+#!/bin/bash
+java -Xmx700M -jar `dirname $0`/sbt-launch-*.jar "$@"
diff --git a/sbt/sbt-launch-0.7.5.RC0.jar b/sbt/sbt-launch-0.7.5.RC0.jar
new file mode 100644
index 0000000000000000000000000000000000000000..e8a5c00f80134b075097e466ced7a91c97d92efe
Binary files /dev/null and b/sbt/sbt-launch-0.7.5.RC0.jar differ