diff --git a/assembly/pom.xml b/assembly/pom.xml
index 594fa0c779e1bb74b4de28daf153afdd9d6efe69..1bb5a671f53903f514911dd5f89738b0aa9ff47e 100644
--- a/assembly/pom.xml
+++ b/assembly/pom.xml
@@ -43,12 +43,6 @@
</properties>
<dependencies>
- <!-- Promote Guava to compile scope in this module so it's included while shading. -->
- <dependency>
- <groupId>com.google.guava</groupId>
- <artifactId>guava</artifactId>
- <scope>compile</scope>
- </dependency>
<dependency>
<groupId>org.apache.spark</groupId>
<artifactId>spark-core_${scala.binary.version}</artifactId>
@@ -133,22 +127,6 @@
<goal>shade</goal>
</goals>
<configuration>
- <relocations>
- <relocation>
- <pattern>com.google</pattern>
- <shadedPattern>org.spark-project.guava</shadedPattern>
- <includes>
- <include>com.google.common.**</include>
- </includes>
- <excludes>
- <exclude>com/google/common/base/Absent*</exclude>
- <exclude>com/google/common/base/Function</exclude>
- <exclude>com/google/common/base/Optional*</exclude>
- <exclude>com/google/common/base/Present*</exclude>
- <exclude>com/google/common/base/Supplier</exclude>
- </excludes>
- </relocation>
- </relocations>
<transformers>
<transformer implementation="org.apache.maven.plugins.shade.resource.ServicesResourceTransformer" />
<transformer implementation="org.apache.maven.plugins.shade.resource.AppendingTransformer">
diff --git a/core/pom.xml b/core/pom.xml
index 1984682b9c0995bcbdb22dca7e11a55bae4c03ca..3c51b2d6b58f9fa6a59c3a423a02ddb00c1717f8 100644
--- a/core/pom.xml
+++ b/core/pom.xml
@@ -106,16 +106,6 @@
<groupId>org.eclipse.jetty</groupId>
<artifactId>jetty-server</artifactId>
</dependency>
- <!--
- Promote Guava to "compile" so that maven-shade-plugin picks it up (for packaging the Optional
- class exposed in the Java API). The plugin will then remove this dependency from the published
- pom, so that Guava does not pollute the client's compilation classpath.
- -->
- <dependency>
- <groupId>com.google.guava</groupId>
- <artifactId>guava</artifactId>
- <scope>compile</scope>
- </dependency>
<dependency>
<groupId>org.apache.commons</groupId>
<artifactId>commons-lang3</artifactId>
@@ -350,44 +340,6 @@
<verbose>true</verbose>
</configuration>
</plugin>
- <plugin>
- <groupId>org.apache.maven.plugins</groupId>
- <artifactId>maven-shade-plugin</artifactId>
- <executions>
- <execution>
- <phase>package</phase>
- <goals>
- <goal>shade</goal>
- </goals>
- <configuration>
- <shadedArtifactAttached>false</shadedArtifactAttached>
- <artifactSet>
- <includes>
- <include>com.google.guava:guava</include>
- </includes>
- </artifactSet>
- <filters>
- <!-- See comment in the guava dependency declaration above. -->
- <filter>
- <artifact>com.google.guava:guava</artifact>
- <includes>
- <include>com/google/common/base/Absent*</include>
- <include>com/google/common/base/Function</include>
- <include>com/google/common/base/Optional*</include>
- <include>com/google/common/base/Present*</include>
- <include>com/google/common/base/Supplier</include>
- </includes>
- </filter>
- </filters>
- </configuration>
- </execution>
- </executions>
- </plugin>
- <!--
- Copy guava to the build directory. This is needed to make the SPARK_PREPEND_CLASSES
- option work in compute-classpath.sh, since it would put the non-shaded Spark classes in
- the runtime classpath.
- -->
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-dependency-plugin</artifactId>
diff --git a/examples/pom.xml b/examples/pom.xml
index 4b92147725f6bfa12dd00eabe73e453eaa6defd1..8caad2bc2e27a7b3f802784a275f8752dd7a49d7 100644
--- a/examples/pom.xml
+++ b/examples/pom.xml
@@ -35,12 +35,6 @@
<url>http://spark.apache.org/</url>
<dependencies>
- <!-- Promote Guava to compile scope in this module so it's included while shading. -->
- <dependency>
- <groupId>com.google.guava</groupId>
- <artifactId>guava</artifactId>
- <scope>compile</scope>
- </dependency>
<dependency>
<groupId>org.apache.spark</groupId>
<artifactId>spark-core_${scala.binary.version}</artifactId>
@@ -310,69 +304,40 @@
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-shade-plugin</artifactId>
- <executions>
- <execution>
- <phase>package</phase>
- <goals>
- <goal>shade</goal>
- </goals>
- <configuration>
- <shadedArtifactAttached>false</shadedArtifactAttached>
- <outputFile>${project.build.directory}/scala-${scala.binary.version}/spark-examples-${project.version}-hadoop${hadoop.version}.jar</outputFile>
- <artifactSet>
- <includes>
- <include>*:*</include>
- </includes>
- </artifactSet>
- <filters>
- <filter>
- <artifact>com.google.guava:guava</artifact>
- <excludes>
- <!--
- Exclude all Guava classes so they're picked up from the main assembly. The
- dependency still needs to be compile-scoped so that the relocation below
- works.
- -->
- <exclude>**</exclude>
- </excludes>
- </filter>
- <filter>
- <artifact>*:*</artifact>
- <excludes>
- <exclude>META-INF/*.SF</exclude>
- <exclude>META-INF/*.DSA</exclude>
- <exclude>META-INF/*.RSA</exclude>
- </excludes>
- </filter>
- </filters>
- <relocations>
- <relocation>
- <pattern>com.google</pattern>
- <shadedPattern>org.spark-project.guava</shadedPattern>
- <includes>
- <include>com.google.common.**</include>
- </includes>
- <excludes>
- <exclude>com.google.common.base.Optional**</exclude>
- </excludes>
- </relocation>
- <relocation>
- <pattern>org.apache.commons.math3</pattern>
- <shadedPattern>org.spark-project.commons.math3</shadedPattern>
- </relocation>
- </relocations>
- <transformers>
- <transformer implementation="org.apache.maven.plugins.shade.resource.ServicesResourceTransformer" />
- <transformer implementation="org.apache.maven.plugins.shade.resource.AppendingTransformer">
- <resource>reference.conf</resource>
- </transformer>
- <transformer implementation="org.apache.maven.plugins.shade.resource.DontIncludeResourceTransformer">
- <resource>log4j.properties</resource>
- </transformer>
- </transformers>
- </configuration>
- </execution>
- </executions>
+ <configuration>
+ <shadedArtifactAttached>false</shadedArtifactAttached>
+ <outputFile>${project.build.directory}/scala-${scala.binary.version}/spark-examples-${project.version}-hadoop${hadoop.version}.jar</outputFile>
+ <artifactSet>
+ <includes>
+ <include>*:*</include>
+ </includes>
+ </artifactSet>
+ <filters>
+ <filter>
+ <artifact>*:*</artifact>
+ <excludes>
+ <exclude>META-INF/*.SF</exclude>
+ <exclude>META-INF/*.DSA</exclude>
+ <exclude>META-INF/*.RSA</exclude>
+ </excludes>
+ </filter>
+ </filters>
+ <relocations combine.children="append">
+ <relocation>
+ <pattern>org.apache.commons.math3</pattern>
+ <shadedPattern>org.spark-project.commons.math3</shadedPattern>
+ </relocation>
+ </relocations>
+ <transformers>
+ <transformer implementation="org.apache.maven.plugins.shade.resource.ServicesResourceTransformer" />
+ <transformer implementation="org.apache.maven.plugins.shade.resource.AppendingTransformer">
+ <resource>reference.conf</resource>
+ </transformer>
+ <transformer implementation="org.apache.maven.plugins.shade.resource.DontIncludeResourceTransformer">
+ <resource>log4j.properties</resource>
+ </transformer>
+ </transformers>
+ </configuration>
</plugin>
</plugins>
</build>
diff --git a/network/common/pom.xml b/network/common/pom.xml
index 245a96b8c4038fc754d2ebbd32bdd13f82fdc7c8..5a9bbe105d9f110009d41f2b5f19e12215b54330 100644
--- a/network/common/pom.xml
+++ b/network/common/pom.xml
@@ -48,10 +48,15 @@
<artifactId>slf4j-api</artifactId>
<scope>provided</scope>
</dependency>
+ <!--
+ Promote Guava to "compile" so that maven-shade-plugin picks it up (for packaging the Optional
+ class exposed in the Java API). The plugin will then remove this dependency from the published
+ pom, so that Guava does not pollute the client's compilation classpath.
+ -->
<dependency>
<groupId>com.google.guava</groupId>
<artifactId>guava</artifactId>
- <scope>provided</scope>
+ <scope>compile</scope>
</dependency>
<!-- Test dependencies -->
@@ -87,11 +92,6 @@
<artifactId>maven-jar-plugin</artifactId>
<version>2.2</version>
<executions>
- <execution>
- <goals>
- <goal>test-jar</goal>
- </goals>
- </execution>
<execution>
<id>test-jar-on-test-compile</id>
<phase>test-compile</phase>
@@ -101,6 +101,18 @@
</execution>
</executions>
</plugin>
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-shade-plugin</artifactId>
+ <configuration>
+ <shadedArtifactAttached>false</shadedArtifactAttached>
+ <artifactSet>
+ <includes>
+ <include>com.google.guava:guava</include>
+ </includes>
+ </artifactSet>
+ </configuration>
+ </plugin>
</plugins>
</build>
</project>
diff --git a/network/shuffle/pom.xml b/network/shuffle/pom.xml
index 5bfa1ac9c373ee7b74e8364eff190a8c9d253432..c2d0300ecd90430984771517408c73a5d9306244 100644
--- a/network/shuffle/pom.xml
+++ b/network/shuffle/pom.xml
@@ -52,7 +52,6 @@
<dependency>
<groupId>com.google.guava</groupId>
<artifactId>guava</artifactId>
- <scope>provided</scope>
</dependency>
<!-- Test dependencies -->
diff --git a/pom.xml b/pom.xml
index 05cb3797fc55b1d17cfe470400beeab1b2760dd9..4adfdf3eb870239c6301473088f1232f7469eeb0 100644
--- a/pom.xml
+++ b/pom.xml
@@ -1264,7 +1264,10 @@
</execution>
</executions>
</plugin>
- <!-- The shade plug-in is used here to create effective pom's (see SPARK-3812). -->
+ <!--
+ The shade plug-in is used here to create effective pom's (see SPARK-3812), and also
+ remove references from the shaded libraries from artifacts published by Spark.
+ -->
<plugin>
<groupId>org.apache.maven.plugins</groupId>
<artifactId>maven-shade-plugin</artifactId>
@@ -1276,6 +1279,23 @@
<include>org.spark-project.spark:unused</include>
</includes>
</artifactSet>
+ <relocations>
+ <relocation>
+ <pattern>com.google.common</pattern>
+ <shadedPattern>org.spark-project.guava</shadedPattern>
+ <excludes>
+ <!--
+ These classes cannot be relocated, because the Java API exposes the
+ "Optional" type; the others are referenced by the Optional class.
+ -->
+ <exclude>com/google/common/base/Absent*</exclude>
+ <exclude>com/google/common/base/Function</exclude>
+ <exclude>com/google/common/base/Optional*</exclude>
+ <exclude>com/google/common/base/Present*</exclude>
+ <exclude>com/google/common/base/Supplier</exclude>
+ </excludes>
+ </relocation>
+ </relocations>
</configuration>
<executions>
<execution>
diff --git a/streaming/pom.xml b/streaming/pom.xml
index 22b0d714b57f647277c27c9e80bf1871eb66304c..98f5b41de84a190214e87e5d5677a3164ed35c4e 100644
--- a/streaming/pom.xml
+++ b/streaming/pom.xml
@@ -95,6 +95,14 @@
</execution>
</executions>
</plugin>
+
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-shade-plugin</artifactId>
+ <configuration>
+ <shadeTestJar>true</shadeTestJar>
+ </configuration>
+ </plugin>
</plugins>
<resources>
<resource>