From 4af6cad37a256fe958e8da9e0937d359bdd5dec5 Mon Sep 17 00:00:00 2001
From: Thomas Dudziak <tomdzk@gmail.com>
Date: Tue, 18 Dec 2012 10:44:03 -0800
Subject: [PATCH] Fixed repl maven build to produce artifacts with the
appropriate hadoop classifier and extracted repl fat-jar and debian packaging
into a separate project to make Maven happy
---
.gitignore | 1 +
pom.xml | 5 +-
repl-bin/pom.xml | 231 ++++++++++++++++++
{repl => repl-bin}/src/deb/bin/run | 0
{repl => repl-bin}/src/deb/bin/spark-executor | 0
{repl => repl-bin}/src/deb/bin/spark-shell | 0
{repl => repl-bin}/src/deb/control/control | 2 +-
repl/pom.xml | 146 +----------
8 files changed, 243 insertions(+), 142 deletions(-)
create mode 100644 repl-bin/pom.xml
rename {repl => repl-bin}/src/deb/bin/run (100%)
rename {repl => repl-bin}/src/deb/bin/spark-executor (100%)
rename {repl => repl-bin}/src/deb/bin/spark-shell (100%)
rename {repl => repl-bin}/src/deb/control/control (88%)
diff --git a/.gitignore b/.gitignore
index f22248f40d..c207409e3c 100644
--- a/.gitignore
+++ b/.gitignore
@@ -31,3 +31,4 @@ project/plugins/src_managed/
logs/
log/
spark-tests.log
+dependency-reduced-pom.xml
diff --git a/pom.xml b/pom.xml
index 6eec7ad173..52a4e9d932 100644
--- a/pom.xml
+++ b/pom.xml
@@ -39,9 +39,10 @@
<modules>
<module>core</module>
- <module>repl</module>
- <module>examples</module>
<module>bagel</module>
+ <module>examples</module>
+ <module>repl</module>
+ <module>repl-bin</module>
</modules>
<properties>
diff --git a/repl-bin/pom.xml b/repl-bin/pom.xml
new file mode 100644
index 0000000000..72a946f3d7
--- /dev/null
+++ b/repl-bin/pom.xml
@@ -0,0 +1,231 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+ <modelVersion>4.0.0</modelVersion>
+ <parent>
+ <groupId>org.spark-project</groupId>
+ <artifactId>parent</artifactId>
+ <version>0.7.0-SNAPSHOT</version>
+ <relativePath>../pom.xml</relativePath>
+ </parent>
+
+ <groupId>org.spark-project</groupId>
+ <artifactId>spark-repl-bin</artifactId>
+ <packaging>pom</packaging>
+ <name>Spark Project REPL binary packaging</name>
+ <url>http://spark-project.org/</url>
+
+ <properties>
+ <deb.install.path>/usr/share/spark</deb.install.path>
+ <deb.user>root</deb.user>
+ </properties>
+
+ <build>
+ <plugins>
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-shade-plugin</artifactId>
+ <configuration>
+ <shadedArtifactAttached>false</shadedArtifactAttached>
+ <outputFile>${project.build.directory}/${project.artifactId}-${project.version}-shaded-${classifier}.jar</outputFile>
+ <artifactSet>
+ <includes>
+ <include>*:*</include>
+ </includes>
+ </artifactSet>
+ <filters>
+ <filter>
+ <artifact>*:*</artifact>
+ <excludes>
+ <exclude>META-INF/*.SF</exclude>
+ <exclude>META-INF/*.DSA</exclude>
+ <exclude>META-INF/*.RSA</exclude>
+ </excludes>
+ </filter>
+ </filters>
+ </configuration>
+ <executions>
+ <execution>
+ <phase>package</phase>
+ <goals>
+ <goal>shade</goal>
+ </goals>
+ <configuration>
+ <transformers>
+ <transformer implementation="org.apache.maven.plugins.shade.resource.ServicesResourceTransformer"/>
+ <transformer implementation="org.apache.maven.plugins.shade.resource.AppendingTransformer">
+ <resource>reference.conf</resource>
+ </transformer>
+ <transformer implementation="org.apache.maven.plugins.shade.resource.ManifestResourceTransformer">
+ <mainClass>spark.repl.Main</mainClass>
+ </transformer>
+ </transformers>
+ </configuration>
+ </execution>
+ </executions>
+ </plugin>
+ </plugins>
+ </build>
+
+ <profiles>
+ <profile>
+ <id>hadoop1</id>
+ <properties>
+ <classifier>hadoop1</classifier>
+ </properties>
+ <dependencies>
+ <dependency>
+ <groupId>org.spark-project</groupId>
+ <artifactId>spark-core</artifactId>
+ <version>${project.version}</version>
+ <classifier>hadoop1</classifier>
+ </dependency>
+ <dependency>
+ <groupId>org.spark-project</groupId>
+ <artifactId>spark-bagel</artifactId>
+ <version>${project.version}</version>
+ <classifier>hadoop1</classifier>
+ <scope>runtime</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.spark-project</groupId>
+ <artifactId>spark-examples</artifactId>
+ <version>${project.version}</version>
+ <classifier>hadoop1</classifier>
+ <scope>runtime</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.spark-project</groupId>
+ <artifactId>spark-repl</artifactId>
+ <version>${project.version}</version>
+ <classifier>hadoop1</classifier>
+ <scope>runtime</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-core</artifactId>
+ <scope>runtime</scope>
+ </dependency>
+ </dependencies>
+ </profile>
+ <profile>
+ <id>hadoop2</id>
+ <properties>
+ <classifier>hadoop2</classifier>
+ </properties>
+ <dependencies>
+ <dependency>
+ <groupId>org.spark-project</groupId>
+ <artifactId>spark-core</artifactId>
+ <version>${project.version}</version>
+ <classifier>hadoop2</classifier>
+ </dependency>
+ <dependency>
+ <groupId>org.spark-project</groupId>
+ <artifactId>spark-bagel</artifactId>
+ <version>${project.version}</version>
+ <classifier>hadoop2</classifier>
+ <scope>runtime</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.spark-project</groupId>
+ <artifactId>spark-examples</artifactId>
+ <version>${project.version}</version>
+ <classifier>hadoop2</classifier>
+ <scope>runtime</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.spark-project</groupId>
+ <artifactId>spark-repl</artifactId>
+ <version>${project.version}</version>
+ <classifier>hadoop2</classifier>
+ <scope>runtime</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-core</artifactId>
+ <scope>runtime</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>hadoop-client</artifactId>
+ <scope>runtime</scope>
+ </dependency>
+ </dependencies>
+ </profile>
+ <profile>
+ <id>deb</id>
+ <build>
+ <plugins>
+ <plugin>
+ <groupId>org.codehaus.mojo</groupId>
+ <artifactId>buildnumber-maven-plugin</artifactId>
+ <version>1.1</version>
+ <executions>
+ <execution>
+ <phase>validate</phase>
+ <goals>
+ <goal>create</goal>
+ </goals>
+ <configuration>
+ <shortRevisionLength>8</shortRevisionLength>
+ </configuration>
+ </execution>
+ </executions>
+ </plugin>
+ <plugin>
+ <groupId>org.vafer</groupId>
+ <artifactId>jdeb</artifactId>
+ <version>0.11</version>
+ <executions>
+ <execution>
+ <phase>package</phase>
+ <goals>
+ <goal>jdeb</goal>
+ </goals>
+ <configuration>
+ <deb>${project.build.directory}/${project.artifactId}-${classifier}_${project.version}-${buildNumber}_all.deb</deb>
+ <attach>false</attach>
+ <compression>gzip</compression>
+ <dataSet>
+ <data>
+ <src>${project.build.directory}/${project.artifactId}-${project.version}-shaded-${classifier}.jar</src>
+ <type>file</type>
+ <mapper>
+ <type>perm</type>
+ <user>${deb.user}</user>
+ <group>${deb.user}</group>
+ <prefix>${deb.install.path}</prefix>
+ </mapper>
+ </data>
+ <data>
+ <src>${basedir}/src/deb/bin</src>
+ <type>directory</type>
+ <mapper>
+ <type>perm</type>
+ <user>${deb.user}</user>
+ <group>${deb.user}</group>
+ <prefix>${deb.install.path}</prefix>
+ <filemode>744</filemode>
+ </mapper>
+ </data>
+ <data>
+ <src>${basedir}/../conf</src>
+ <type>directory</type>
+ <mapper>
+ <type>perm</type>
+ <user>${deb.user}</user>
+ <group>${deb.user}</group>
+ <prefix>${deb.install.path}/conf</prefix>
+ <filemode>744</filemode>
+ </mapper>
+ </data>
+ </dataSet>
+ </configuration>
+ </execution>
+ </executions>
+ </plugin>
+ </plugins>
+ </build>
+ </profile>
+ </profiles>
+</project>
diff --git a/repl/src/deb/bin/run b/repl-bin/src/deb/bin/run
similarity index 100%
rename from repl/src/deb/bin/run
rename to repl-bin/src/deb/bin/run
diff --git a/repl/src/deb/bin/spark-executor b/repl-bin/src/deb/bin/spark-executor
similarity index 100%
rename from repl/src/deb/bin/spark-executor
rename to repl-bin/src/deb/bin/spark-executor
diff --git a/repl/src/deb/bin/spark-shell b/repl-bin/src/deb/bin/spark-shell
similarity index 100%
rename from repl/src/deb/bin/spark-shell
rename to repl-bin/src/deb/bin/spark-shell
diff --git a/repl/src/deb/control/control b/repl-bin/src/deb/control/control
similarity index 88%
rename from repl/src/deb/control/control
rename to repl-bin/src/deb/control/control
index 6586986c76..afadb3fbfe 100644
--- a/repl/src/deb/control/control
+++ b/repl-bin/src/deb/control/control
@@ -1,4 +1,4 @@
-Package: spark-repl
+Package: [[artifactId]]
Version: [[version]]-[[buildNumber]]
Section: misc
Priority: extra
diff --git a/repl/pom.xml b/repl/pom.xml
index f6df4ba9f7..114e3e9932 100644
--- a/repl/pom.xml
+++ b/repl/pom.xml
@@ -99,46 +99,17 @@
<dependency>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-core</artifactId>
+ <scope>provided</scope>
</dependency>
</dependencies>
<build>
<plugins>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
- <artifactId>maven-shade-plugin</artifactId>
+ <artifactId>maven-jar-plugin</artifactId>
<configuration>
- <shadedArtifactAttached>true</shadedArtifactAttached>
- <shadedClassifierName>shaded-hadoop1</shadedClassifierName>
- <filters>
- <filter>
- <artifact>*:*</artifact>
- <excludes>
- <exclude>META-INF/*.SF</exclude>
- <exclude>META-INF/*.DSA</exclude>
- <exclude>META-INF/*.RSA</exclude>
- </excludes>
- </filter>
- </filters>
+ <classifier>hadoop1</classifier>
</configuration>
- <executions>
- <execution>
- <phase>package</phase>
- <goals>
- <goal>shade</goal>
- </goals>
- <configuration>
- <transformers>
- <transformer implementation="org.apache.maven.plugins.shade.resource.ServicesResourceTransformer"/>
- <transformer implementation="org.apache.maven.plugins.shade.resource.AppendingTransformer">
- <resource>reference.conf</resource>
- </transformer>
- <transformer implementation="org.apache.maven.plugins.shade.resource.ManifestResourceTransformer">
- <mainClass>spark.repl.Main</mainClass>
- </transformer>
- </transformers>
- </configuration>
- </execution>
- </executions>
</plugin>
</plugins>
</build>
@@ -172,125 +143,22 @@
<dependency>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-core</artifactId>
+ <scope>provided</scope>
</dependency>
<dependency>
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-client</artifactId>
+ <scope>provided</scope>
</dependency>
</dependencies>
<build>
<plugins>
<plugin>
<groupId>org.apache.maven.plugins</groupId>
- <artifactId>maven-shade-plugin</artifactId>
+ <artifactId>maven-jar-plugin</artifactId>
<configuration>
- <shadedArtifactAttached>true</shadedArtifactAttached>
- <shadedClassifierName>shaded-hadoop2</shadedClassifierName>
- <filters>
- <filter>
- <artifact>*:*</artifact>
- <excludes>
- <exclude>META-INF/*.SF</exclude>
- <exclude>META-INF/*.DSA</exclude>
- <exclude>META-INF/*.RSA</exclude>
- </excludes>
- </filter>
- </filters>
+ <classifier>hadoop2</classifier>
</configuration>
- <executions>
- <execution>
- <phase>package</phase>
- <goals>
- <goal>shade</goal>
- </goals>
- <configuration>
- <transformers>
- <transformer implementation="org.apache.maven.plugins.shade.resource.ServicesResourceTransformer"/>
- <transformer implementation="org.apache.maven.plugins.shade.resource.AppendingTransformer">
- <resource>reference.conf</resource>
- </transformer>
- <transformer implementation="org.apache.maven.plugins.shade.resource.ManifestResourceTransformer">
- <mainClass>spark.repl.Main</mainClass>
- </transformer>
- </transformers>
- </configuration>
- </execution>
- </executions>
- </plugin>
- </plugins>
- </build>
- </profile>
- <profile>
- <id>deb</id>
- <build>
- <plugins>
- <plugin>
- <groupId>org.codehaus.mojo</groupId>
- <artifactId>buildnumber-maven-plugin</artifactId>
- <version>1.1</version>
- <executions>
- <execution>
- <phase>validate</phase>
- <goals>
- <goal>create</goal>
- </goals>
- <configuration>
- <shortRevisionLength>8</shortRevisionLength>
- </configuration>
- </execution>
- </executions>
- </plugin>
- <plugin>
- <groupId>org.vafer</groupId>
- <artifactId>jdeb</artifactId>
- <version>0.11</version>
- <executions>
- <execution>
- <phase>package</phase>
- <goals>
- <goal>jdeb</goal>
- </goals>
- <configuration>
- <deb>${project.build.directory}/${project.artifactId}-${classifier}_${project.version}-${buildNumber}_all.deb</deb>
- <attach>false</attach>
- <compression>gzip</compression>
- <dataSet>
- <data>
- <src>${project.build.directory}/${project.artifactId}-${project.version}-shaded-${classifier}.jar</src>
- <type>file</type>
- <mapper>
- <type>perm</type>
- <user>${deb.user}</user>
- <group>${deb.user}</group>
- <prefix>${deb.install.path}</prefix>
- </mapper>
- </data>
- <data>
- <src>${basedir}/src/deb/bin</src>
- <type>directory</type>
- <mapper>
- <type>perm</type>
- <user>${deb.user}</user>
- <group>${deb.user}</group>
- <prefix>${deb.install.path}</prefix>
- <filemode>744</filemode>
- </mapper>
- </data>
- <data>
- <src>${basedir}/../conf</src>
- <type>directory</type>
- <mapper>
- <type>perm</type>
- <user>${deb.user}</user>
- <group>${deb.user}</group>
- <prefix>${deb.install.path}/conf</prefix>
- <filemode>744</filemode>
- </mapper>
- </data>
- </dataSet>
- </configuration>
- </execution>
- </executions>
</plugin>
</plugins>
</build>
--
GitLab