diff --git a/bin/compute-classpath.sh b/bin/compute-classpath.sh index a8c344b1ca59495154ecbdb412c24ad7004de6ea..9e8d0b785194e170bd35b2cde39b034d48c3652c 100755 --- a/bin/compute-classpath.sh +++ b/bin/compute-classpath.sh @@ -50,8 +50,8 @@ fi if [ -n "$SPARK_PREPEND_CLASSES" ]; then echo "NOTE: SPARK_PREPEND_CLASSES is set, placing locally compiled Spark"\ "classes ahead of assembly." >&2 - # Spark classes CLASSPATH="$CLASSPATH:$FWDIR/core/target/scala-$SPARK_SCALA_VERSION/classes" + CLASSPATH="$CLASSPATH:$FWDIR/core/target/jars/*" CLASSPATH="$CLASSPATH:$FWDIR/repl/target/scala-$SPARK_SCALA_VERSION/classes" CLASSPATH="$CLASSPATH:$FWDIR/mllib/target/scala-$SPARK_SCALA_VERSION/classes" CLASSPATH="$CLASSPATH:$FWDIR/bagel/target/scala-$SPARK_SCALA_VERSION/classes" @@ -63,8 +63,6 @@ if [ -n "$SPARK_PREPEND_CLASSES" ]; then CLASSPATH="$CLASSPATH:$FWDIR/sql/hive/target/scala-$SPARK_SCALA_VERSION/classes" CLASSPATH="$CLASSPATH:$FWDIR/sql/hive-thriftserver/target/scala-$SPARK_SCALA_VERSION/classes" CLASSPATH="$CLASSPATH:$FWDIR/yarn/stable/target/scala-$SPARK_SCALA_VERSION/classes" - # Jars for shaded deps in their original form (copied here during build) - CLASSPATH="$CLASSPATH:$FWDIR/core/target/jars/*" fi # Use spark-assembly jar from either RELEASE or assembly directory diff --git a/core/pom.xml b/core/pom.xml index d91f4ee0241ac15a8ddf737ac1c9b757f7896bec..31e919a1c831a81f49af733e21eb1e83b7685e92 100644 --- a/core/pom.xml +++ b/core/pom.xml @@ -94,35 +94,22 @@ <groupId>org.apache.curator</groupId> <artifactId>curator-recipes</artifactId> </dependency> - - <!-- Jetty dependencies promoted to compile here so they are shaded - and inlined into spark-core jar --> <dependency> <groupId>org.eclipse.jetty</groupId> <artifactId>jetty-plus</artifactId> - <scope>compile</scope> </dependency> <dependency> <groupId>org.eclipse.jetty</groupId> <artifactId>jetty-security</artifactId> - <scope>compile</scope> </dependency> <dependency> <groupId>org.eclipse.jetty</groupId> <artifactId>jetty-util</artifactId> - <scope>compile</scope> </dependency> <dependency> <groupId>org.eclipse.jetty</groupId> <artifactId>jetty-server</artifactId> - <scope>compile</scope> </dependency> - <dependency> - <groupId>org.eclipse.jetty</groupId> - <artifactId>jetty-http</artifactId> - <scope>compile</scope> - </dependency> - <dependency> <groupId>org.apache.commons</groupId> <artifactId>commons-lang3</artifactId> @@ -361,24 +348,19 @@ <groupId>org.apache.maven.plugins</groupId> <artifactId>maven-dependency-plugin</artifactId> <executions> - <!-- When using SPARK_PREPEND_CLASSES Spark classes compiled locally don't use - shaded deps. So here we store jars in their original form which are added - when the classpath is computed. --> <execution> <id>copy-dependencies</id> <phase>package</phase> <goals> <goal>copy-dependencies</goal> </goals> - <configuration> + <configuration> <outputDirectory>${project.build.directory}</outputDirectory> <overWriteReleases>false</overWriteReleases> <overWriteSnapshots>false</overWriteSnapshots> <overWriteIfNewer>true</overWriteIfNewer> <useSubDirectoryPerType>true</useSubDirectoryPerType> - <includeArtifactIds> - guava,jetty-io,jetty-http,jetty-plus,jetty-util,jetty-server - </includeArtifactIds> + <includeArtifactIds>guava</includeArtifactIds> <silent>true</silent> </configuration> </execution> diff --git a/network/common/pom.xml b/network/common/pom.xml index 8f7c924d6b3a39848b203d585e52b7cb8dfba04b..5a9bbe105d9f110009d41f2b5f19e12215b54330 100644 --- a/network/common/pom.xml +++ b/network/common/pom.xml @@ -101,6 +101,18 @@ </execution> </executions> </plugin> + <plugin> + <groupId>org.apache.maven.plugins</groupId> + <artifactId>maven-shade-plugin</artifactId> + <configuration> + <shadedArtifactAttached>false</shadedArtifactAttached> + <artifactSet> + <includes> + <include>com.google.guava:guava</include> + </includes> + </artifactSet> + </configuration> + </plugin> </plugins> </build> </project> diff --git a/pom.xml b/pom.xml index 63c0a2af9e021488e28a9fafd11fa4d8061250ec..4adfdf3eb870239c6301473088f1232f7469eeb0 100644 --- a/pom.xml +++ b/pom.xml @@ -337,39 +337,25 @@ </exclusion> </exclusions> </dependency> - - <!-- Shaded deps marked as provided. These are promoted to compile scope - in the modules where we want the shaded classes to appear in the - associated jar. --> - <dependency> - <groupId>org.eclipse.jetty</groupId> - <artifactId>jetty-http</artifactId> - <version>${jetty.version}</version> - <scope>provided</scope> - </dependency> <dependency> <groupId>org.eclipse.jetty</groupId> <artifactId>jetty-util</artifactId> <version>${jetty.version}</version> - <scope>provided</scope> </dependency> <dependency> <groupId>org.eclipse.jetty</groupId> <artifactId>jetty-security</artifactId> <version>${jetty.version}</version> - <scope>provided</scope> </dependency> <dependency> <groupId>org.eclipse.jetty</groupId> <artifactId>jetty-plus</artifactId> <version>${jetty.version}</version> - <scope>provided</scope> </dependency> <dependency> <groupId>org.eclipse.jetty</groupId> <artifactId>jetty-server</artifactId> <version>${jetty.version}</version> - <scope>provided</scope> </dependency> <dependency> <groupId>com.google.guava</groupId> @@ -377,8 +363,6 @@ <version>14.0.1</version> <scope>provided</scope> </dependency> - <!-- End of shaded deps --> - <dependency> <groupId>org.apache.commons</groupId> <artifactId>commons-lang3</artifactId> @@ -1292,26 +1276,10 @@ <shadedArtifactAttached>false</shadedArtifactAttached> <artifactSet> <includes> - <!-- At a minimum we must include this to force effective pom generation --> <include>org.spark-project.spark:unused</include> - - <include>org.eclipse.jetty:jetty-io</include> - <include>org.eclipse.jetty:jetty-http</include> - <include>org.eclipse.jetty:jetty-plus</include> - <include>org.eclipse.jetty:jetty-security</include> - <include>org.eclipse.jetty:jetty-util</include> - <include>org.eclipse.jetty:jetty-server</include> - <include>com.google.guava:guava</include> </includes> </artifactSet> <relocations> - <relocation> - <pattern>org.eclipse.jetty</pattern> - <shadedPattern>org.spark-project.jetty</shadedPattern> - <includes> - <include>org.eclipse.jetty.**</include> - </includes> - </relocation> <relocation> <pattern>com.google.common</pattern> <shadedPattern>org.spark-project.guava</shadedPattern>