diff --git a/docs/building-with-maven.md b/docs/building-with-maven.md index b9ff0af76f6478f49fe37a0e6a360796eb1d967a..6a9a8d681742fb6245029cbea7af189e0092a1bb 100644 --- a/docs/building-with-maven.md +++ b/docs/building-with-maven.md @@ -71,8 +71,8 @@ This setup works fine in IntelliJ IDEA 11.1.4. After opening the project via the ## Building Spark Debian Packages ## -It includes support for building a Debian package containing a 'fat-jar' which includes the repl, the examples and bagel. This can be created by specifying the following profiles: +The maven build includes support for building a Debian package containing the assembly 'fat-jar', PySpark, and the necessary scripts and configuration files. This can be created by specifying the following: - $ mvn -Prepl-bin -Pdeb clean package + $ mvn -Pdeb -DskipTests clean package -The debian package can then be found under repl/target. We added the short commit hash to the file name so that we can distinguish individual packages build for SNAPSHOT versions. +The debian package can then be found under assembly/target. We added the short commit hash to the file name so that we can distinguish individual packages built for SNAPSHOT versions. diff --git a/pom.xml b/pom.xml index b25d9d7ef891dfcbc6d2c5d17586a363a5e9a198..54072b053cb5e8389eb8be5922206f7fbb2b9f16 100644 --- a/pom.xml +++ b/pom.xml @@ -754,15 +754,5 @@ </modules> </profile> - - <profile> - <id>repl-bin</id> - <activation> - <activeByDefault>false</activeByDefault> - </activation> - <modules> - <module>repl-bin</module> - </modules> - </profile> </profiles> </project> diff --git a/repl-bin/pom.xml b/repl-bin/pom.xml deleted file mode 100644 index 869dbdb9b095a170debd8794cb060f9572edf5e7..0000000000000000000000000000000000000000 --- a/repl-bin/pom.xml +++ /dev/null @@ -1,184 +0,0 @@ -<?xml version="1.0" encoding="UTF-8"?> -<!-- - ~ Licensed to the Apache Software Foundation (ASF) under one or more - ~ contributor license agreements. See the NOTICE file distributed with - ~ this work for additional information regarding copyright ownership. - ~ The ASF licenses this file to You under the Apache License, Version 2.0 - ~ (the "License"); you may not use this file except in compliance with - ~ the License. You may obtain a copy of the License at - ~ - ~ http://www.apache.org/licenses/LICENSE-2.0 - ~ - ~ Unless required by applicable law or agreed to in writing, software - ~ distributed under the License is distributed on an "AS IS" BASIS, - ~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - ~ See the License for the specific language governing permissions and - ~ limitations under the License. - --> - -<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd"> - <modelVersion>4.0.0</modelVersion> - <parent> - <groupId>org.apache.spark</groupId> - <artifactId>spark-parent</artifactId> - <version>0.9.0-incubating-SNAPSHOT</version> - <relativePath>../pom.xml</relativePath> - </parent> - - <groupId>org.apache.spark</groupId> - <artifactId>spark-repl-bin_2.10</artifactId> - <packaging>pom</packaging> - <name>Spark Project REPL binary packaging</name> - <url>http://spark.incubator.apache.org/</url> - - <properties> - <deb.pkg.name>spark</deb.pkg.name> - <deb.install.path>/usr/share/spark</deb.install.path> - <deb.user>root</deb.user> - </properties> - - <dependencies> - <dependency> - <groupId>org.apache.spark</groupId> - <artifactId>spark-core_${scala.binary.version}</artifactId> - <version>${project.version}</version> - </dependency> - <dependency> - <groupId>org.apache.spark</groupId> - <artifactId>spark-bagel_${scala.binary.version}</artifactId> - <version>${project.version}</version> - <scope>runtime</scope> - </dependency> - <dependency> - <groupId>org.apache.spark</groupId> - <artifactId>spark-repl_${scala.binary.version}</artifactId> - <version>${project.version}</version> - <scope>runtime</scope> - </dependency> - </dependencies> - - <build> - <plugins> - <plugin> - <groupId>org.apache.maven.plugins</groupId> - <artifactId>maven-shade-plugin</artifactId> - <configuration> - <shadedArtifactAttached>false</shadedArtifactAttached> - <outputFile>${project.build.directory}/${project.artifactId}-${project.version}-shaded.jar</outputFile> - <artifactSet> - <includes> - <include>*:*</include> - </includes> - </artifactSet> - <filters> - <filter> - <artifact>*:*</artifact> - <excludes> - <exclude>META-INF/*.SF</exclude> - <exclude>META-INF/*.DSA</exclude> - <exclude>META-INF/*.RSA</exclude> - </excludes> - </filter> - </filters> - </configuration> - <executions> - <execution> - <phase>package</phase> - <goals> - <goal>shade</goal> - </goals> - <configuration> - <transformers> - <transformer implementation="org.apache.maven.plugins.shade.resource.ServicesResourceTransformer" /> - <transformer implementation="org.apache.maven.plugins.shade.resource.AppendingTransformer"> - <resource>reference.conf</resource> - </transformer> - <transformer implementation="org.apache.maven.plugins.shade.resource.ManifestResourceTransformer"> - <mainClass>spark.repl.Main</mainClass> - </transformer> - </transformers> - </configuration> - </execution> - </executions> - </plugin> - </plugins> - </build> - - <profiles> - <profile> - <id>deb</id> - <build> - <plugins> - <plugin> - <groupId>org.codehaus.mojo</groupId> - <artifactId>buildnumber-maven-plugin</artifactId> - <version>1.1</version> - <executions> - <execution> - <phase>validate</phase> - <goals> - <goal>create</goal> - </goals> - <configuration> - <shortRevisionLength>8</shortRevisionLength> - </configuration> - </execution> - </executions> - </plugin> - <plugin> - <groupId>org.vafer</groupId> - <artifactId>jdeb</artifactId> - <version>0.11</version> - <executions> - <execution> - <phase>package</phase> - <goals> - <goal>jdeb</goal> - </goals> - <configuration> - <deb>${project.build.directory}/${deb.pkg.name}_${project.version}-${buildNumber}_all.deb</deb> - <attach>false</attach> - <compression>gzip</compression> - <dataSet> - <data> - <src>${project.build.directory}/${project.artifactId}-${project.version}-shaded.jar</src> - <type>file</type> - <mapper> - <type>perm</type> - <user>${deb.user}</user> - <group>${deb.user}</group> - <prefix>${deb.install.path}</prefix> - </mapper> - </data> - <data> - <src>${basedir}/src/deb/bin</src> - <type>directory</type> - <mapper> - <type>perm</type> - <user>${deb.user}</user> - <group>${deb.user}</group> - <prefix>${deb.install.path}</prefix> - <filemode>744</filemode> - </mapper> - </data> - <data> - <src>${basedir}/../conf</src> - <type>directory</type> - <mapper> - <type>perm</type> - <user>${deb.user}</user> - <group>${deb.user}</group> - <prefix>${deb.install.path}/conf</prefix> - <filemode>744</filemode> - </mapper> - </data> - </dataSet> - </configuration> - </execution> - </executions> - </plugin> - </plugins> - </build> - </profile> - </profiles> -</project> diff --git a/repl-bin/src/deb/bin/run b/repl-bin/src/deb/bin/run deleted file mode 100755 index 3a6f22f41fca5406a615f57b9d1c96187ce6d928..0000000000000000000000000000000000000000 --- a/repl-bin/src/deb/bin/run +++ /dev/null @@ -1,57 +0,0 @@ -#!/usr/bin/env bash - -# -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -SCALA_VERSION=2.10 - -# Figure out where the Scala framework is installed -FWDIR="$(cd `dirname $0`; pwd)" - -# Export this as SPARK_HOME -export SPARK_HOME="$FWDIR" - -# Load environment variables from conf/spark-env.sh, if it exists -if [ -e $FWDIR/conf/spark-env.sh ] ; then - . $FWDIR/conf/spark-env.sh -fi - -# Figure out how much memory to use per executor and set it as an environment -# variable so that our process sees it and can report it to Mesos -if [ -z "$SPARK_MEM" ] ; then - SPARK_MEM="512m" -fi -export SPARK_MEM - -# Set JAVA_OPTS to be able to load native libraries and to set heap size -JAVA_OPTS="$SPARK_JAVA_OPTS" -JAVA_OPTS+=" -Djava.library.path=$SPARK_LIBRARY_PATH" -JAVA_OPTS+=" -Xms$SPARK_MEM -Xmx$SPARK_MEM" -# Load extra JAVA_OPTS from conf/java-opts, if it exists -if [ -e $FWDIR/conf/java-opts ] ; then - JAVA_OPTS+=" `cat $FWDIR/conf/java-opts`" -fi -export JAVA_OPTS - -# Build up classpath -CLASSPATH=":$FWDIR/conf" -for jar in `find $FWDIR -name '*jar'`; do - CLASSPATH+=":$jar" -done -export CLASSPATH - -exec java -Dscala.usejavacp=true -Djline.shutdownhook=true -cp "$CLASSPATH" $JAVA_OPTS $EXTRA_ARGS "$@" diff --git a/repl-bin/src/deb/bin/spark-executor b/repl-bin/src/deb/bin/spark-executor deleted file mode 100755 index 052d76fb8d81cad0b27af8f0ac86b51b6acc8b7c..0000000000000000000000000000000000000000 --- a/repl-bin/src/deb/bin/spark-executor +++ /dev/null @@ -1,22 +0,0 @@ -#!/usr/bin/env bash - -# -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -FWDIR="$(cd `dirname $0`; pwd)" -echo "Running spark-executor with framework dir = $FWDIR" -exec $FWDIR/run org.apache.spark.executor.MesosExecutorBackend diff --git a/repl-bin/src/deb/bin/spark-shell b/repl-bin/src/deb/bin/spark-shell deleted file mode 100755 index 118349d7c30f26ec8770724fca797a3aed69cbd4..0000000000000000000000000000000000000000 --- a/repl-bin/src/deb/bin/spark-shell +++ /dev/null @@ -1,21 +0,0 @@ -#!/usr/bin/env bash - -# -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -# - -FWDIR="$(cd `dirname $0`; pwd)" -exec $FWDIR/run org.apache.spark.repl.Main "$@" diff --git a/repl-bin/src/deb/control/control b/repl-bin/src/deb/control/control deleted file mode 100644 index a6b4471d485f475358462b5f9437b104cce46d09..0000000000000000000000000000000000000000 --- a/repl-bin/src/deb/control/control +++ /dev/null @@ -1,8 +0,0 @@ -Package: [[deb.pkg.name]] -Version: [[version]]-[[buildNumber]] -Section: misc -Priority: extra -Architecture: all -Maintainer: Matei Zaharia <matei.zaharia@gmail.com> -Description: [[name]] -Distribution: development