Migrate Java code to Scala or move it to src/main/java

These classes can't be migrated:
  StorageLevels: impossible to create static fields in Scala
  JavaSparkContextVarargsWorkaround: incompatible varargs
  JavaAPISuite: should test Java APIs in pure Java (for sanity)

Author: Punya Biswal <pbiswal@palantir.com>

Closes #605 from punya/move-java-sources and squashes the following commits:

25b00b2 [Punya Biswal] Remove redundant type param; reformat
853da46 [Punya Biswal] Use factory method rather than constructor
e5d53d9 [Punya Biswal] Migrate Java code to Scala or move it to src/main/java

core/src/main/scala/org/apache/spark/api/java/StorageLevels.java → core/src/main/java/org/apache/spark/api/java/StorageLevels.java

@@ -23,17 +23,17 @@ import org.apache.spark.storage.StorageLevel;
  * Expose some commonly useful storage level constants.
  */
 public class StorageLevels {
-  public static final StorageLevel NONE = new StorageLevel(false, false, false, 1);
-  public static final StorageLevel DISK_ONLY = new StorageLevel(true, false, false, 1);
-  public static final StorageLevel DISK_ONLY_2 = new StorageLevel(true, false, false, 2);
-  public static final StorageLevel MEMORY_ONLY = new StorageLevel(false, true, true, 1);
-  public static final StorageLevel MEMORY_ONLY_2 = new StorageLevel(false, true, true, 2);
-  public static final StorageLevel MEMORY_ONLY_SER = new StorageLevel(false, true, false, 1);
-  public static final StorageLevel MEMORY_ONLY_SER_2 = new StorageLevel(false, true, false, 2);
-  public static final StorageLevel MEMORY_AND_DISK = new StorageLevel(true, true, true, 1);
-  public static final StorageLevel MEMORY_AND_DISK_2 = new StorageLevel(true, true, true, 2);
-  public static final StorageLevel MEMORY_AND_DISK_SER = new StorageLevel(true, true, false, 1);
-  public static final StorageLevel MEMORY_AND_DISK_SER_2 = new StorageLevel(true, true, false, 2);
+  public static final StorageLevel NONE = create(false, false, false, 1);
+  public static final StorageLevel DISK_ONLY = create(true, false, false, 1);
+  public static final StorageLevel DISK_ONLY_2 = create(true, false, false, 2);
+  public static final StorageLevel MEMORY_ONLY = create(false, true, true, 1);
+  public static final StorageLevel MEMORY_ONLY_2 = create(false, true, true, 2);
+  public static final StorageLevel MEMORY_ONLY_SER = create(false, true, false, 1);
+  public static final StorageLevel MEMORY_ONLY_SER_2 = create(false, true, false, 2);
+  public static final StorageLevel MEMORY_AND_DISK = create(true, true, true, 1);
+  public static final StorageLevel MEMORY_AND_DISK_2 = create(true, true, true, 2);
+  public static final StorageLevel MEMORY_AND_DISK_SER = create(true, true, false, 1);
+  public static final StorageLevel MEMORY_AND_DISK_SER_2 = create(true, true, false, 2);
 
   /**
    * Create a new StorageLevel object.

core/src/main/scala/org/apache/spark/SparkFiles.java → core/src/main/scala/org/apache/spark/SparkFiles.scala

@@ -15,28 +15,25 @@
  * limitations under the License.
  */
 
-package org.apache.spark;
+package org.apache.spark
 
-import java.io.File;
+import java.io.File
 
 /**
  * Resolves paths to files added through `SparkContext.addFile()`.
  */
-public class SparkFiles {
-
-  private SparkFiles() {}
+object SparkFiles {
 
   /**
    * Get the absolute path of a file added through `SparkContext.addFile()`.
    */
-  public static String get(String filename) {
-    return new File(getRootDirectory(), filename).getAbsolutePath();
-  }
+  def get(filename: String): String =
+    new File(getRootDirectory(), filename).getAbsolutePath()
 
   /**
    * Get the root directory that contains files added through `SparkContext.addFile()`.
    */
-  public static String getRootDirectory() {
-    return SparkEnv.get().sparkFilesDir();
-  }
+  def getRootDirectory(): String =
+    SparkEnv.get.sparkFilesDir
+
 }

core/src/main/scala/org/apache/spark/api/java/function/DoubleFlatMapFunction.java → core/src/main/scala/org/apache/spark/api/java/function/DoubleFlatMapFunction.scala

@@ -15,16 +15,16 @@
  * limitations under the License.
  */
 
-package org.apache.spark.api.java.function;
+package org.apache.spark.api.java.function
 
-import java.io.Serializable;
+import java.lang.{Double => JDouble, Iterable => JIterable}
 
 /**
  * A function that returns zero or more records of type Double from each input record.
  */
 // DoubleFlatMapFunction does not extend FlatMapFunction because flatMap is
 // overloaded for both FlatMapFunction and DoubleFlatMapFunction.
-public abstract class DoubleFlatMapFunction<T> extends WrappedFunction1<T, Iterable<Double>>
-  implements Serializable {
-    // Intentionally left blank
+abstract class DoubleFlatMapFunction[T] extends WrappedFunction1[T, JIterable[JDouble]]
+   with Serializable {
+   // Intentionally left blank
 }

core/src/main/scala/org/apache/spark/api/java/function/DoubleFunction.java → core/src/main/scala/org/apache/spark/api/java/function/DoubleFunction.scala

@@ -15,16 +15,15 @@
  * limitations under the License.
  */
 
-package org.apache.spark.api.java.function;
+package org.apache.spark.api.java.function
 
-import java.io.Serializable;
+import java.lang.{Double => JDouble}
 
 /**
  * A function that returns Doubles, and can be used to construct DoubleRDDs.
  */
 // DoubleFunction does not extend Function because some UDF functions, like map,
 // are overloaded for both Function and DoubleFunction.
-public abstract class DoubleFunction<T> extends WrappedFunction1<T, Double>
-  implements Serializable {
+abstract class DoubleFunction[T] extends WrappedFunction1[T, JDouble] with Serializable {
     // Intentionally left blank
 }

core/src/main/scala/org/apache/spark/api/java/function/Function.java → core/src/main/scala/org/apache/spark/api/java/function/Function.scala

@@ -15,21 +15,17 @@
  * limitations under the License.
  */
 
-package org.apache.spark.api.java.function;
+package org.apache.spark.api.java.function
 
-import java.io.Serializable;
-
-import scala.reflect.ClassTag;
-import scala.reflect.ClassTag$;
+import scala.reflect.ClassTag
+import org.apache.spark.api.java.JavaSparkContext
 
 /**
  * Base class for functions whose return types do not create special RDDs. PairFunction and
  * DoubleFunction are handled separately, to allow PairRDDs and DoubleRDDs to be constructed
  * when mapping RDDs of other types.
  */
-public abstract class Function<T, R> extends WrappedFunction1<T, R> implements Serializable {
-  public ClassTag<R> returnType() {
-    return ClassTag$.MODULE$.apply(Object.class);
-  }
+abstract class Function[T, R] extends WrappedFunction1[T, R] with Serializable {
+  def returnType(): ClassTag[R] = JavaSparkContext.fakeClassTag
 }
 

core/src/main/scala/org/apache/spark/api/java/function/Function2.java → core/src/main/scala/org/apache/spark/api/java/function/Function2.scala

@@ -15,21 +15,15 @@
  * limitations under the License.
  */
 
-package org.apache.spark.api.java.function;
+package org.apache.spark.api.java.function
 
-import java.io.Serializable;
-
-import scala.reflect.ClassTag;
-import scala.reflect.ClassTag$;
+import scala.reflect.ClassTag
+import org.apache.spark.api.java.JavaSparkContext
 
 /**
  * A two-argument function that takes arguments of type T1 and T2 and returns an R.
  */
-public abstract class Function2<T1, T2, R> extends WrappedFunction2<T1, T2, R>
-  implements Serializable {
-
-  public ClassTag<R> returnType() {
-    return (ClassTag<R>) ClassTag$.MODULE$.apply(Object.class);
-  }
+abstract class Function2[T1, T2, R] extends WrappedFunction2[T1, T2, R] with Serializable {
+  def returnType(): ClassTag[R] = JavaSparkContext.fakeClassTag
 }
 

core/src/main/scala/org/apache/spark/api/java/function/Function3.java → core/src/main/scala/org/apache/spark/api/java/function/Function3.scala

@@ -15,21 +15,14 @@
  * limitations under the License.
  */
 
-package org.apache.spark.api.java.function;
+package org.apache.spark.api.java.function
 
-import java.io.Serializable;
-
-import scala.reflect.ClassTag;
-import scala.reflect.ClassTag$;
+import org.apache.spark.api.java.JavaSparkContext
+import scala.reflect.ClassTag
 
 /**
  * A three-argument function that takes arguments of type T1, T2 and T3 and returns an R.
  */
-public abstract class Function3<T1, T2, T3, R> extends WrappedFunction3<T1, T2, T3, R>
-        implements Serializable {
-
-    public ClassTag<R> returnType() {
-        return (ClassTag<R>) ClassTag$.MODULE$.apply(Object.class);
-    }
+abstract class Function3[T1, T2, T3, R] extends WrappedFunction3[T1, T2, T3, R] with Serializable {
+  def returnType(): ClassTag[R] = JavaSparkContext.fakeClassTag
 }
-

core/src/main/scala/org/apache/spark/api/java/function/PairFlatMapFunction.java → core/src/main/scala/org/apache/spark/api/java/function/PairFlatMapFunction.scala

@@ -15,13 +15,11 @@
  * limitations under the License.
  */
 
-package org.apache.spark.api.java.function;
+package org.apache.spark.api.java.function
 
-import java.io.Serializable;
-
-import scala.Tuple2;
-import scala.reflect.ClassTag;
-import scala.reflect.ClassTag$;
+import java.lang.{Iterable => JIterable}
+import org.apache.spark.api.java.JavaSparkContext
+import scala.reflect.ClassTag
 
 /**
  * A function that returns zero or more key-value pair records from each input record. The
@@ -29,15 +27,10 @@ import scala.reflect.ClassTag$;
  */
 // PairFlatMapFunction does not extend FlatMapFunction because flatMap is
 // overloaded for both FlatMapFunction and PairFlatMapFunction.
-public abstract class PairFlatMapFunction<T, K, V>
-  extends WrappedFunction1<T, Iterable<Tuple2<K, V>>>
-  implements Serializable {
+abstract class PairFlatMapFunction[T, K, V] extends WrappedFunction1[T, JIterable[(K, V)]]
+  with Serializable {
 
-  public ClassTag<K> keyType() {
-    return (ClassTag<K>) ClassTag$.MODULE$.apply(Object.class);
-  }
+  def keyType(): ClassTag[K] = JavaSparkContext.fakeClassTag
 
-  public ClassTag<V> valueType() {
-    return (ClassTag<V>) ClassTag$.MODULE$.apply(Object.class);
-  }
+  def valueType(): ClassTag[V] = JavaSparkContext.fakeClassTag
 }

core/src/main/scala/org/apache/spark/api/java/function/PairFunction.java → core/src/main/scala/org/apache/spark/api/java/function/PairFunction.scala

@@ -15,27 +15,19 @@
  * limitations under the License.
  */
 
-package org.apache.spark.api.java.function;
+package org.apache.spark.api.java.function
 
-import java.io.Serializable;
-
-import scala.Tuple2;
-import scala.reflect.ClassTag;
-import scala.reflect.ClassTag$;
+import scala.reflect.ClassTag
+import org.apache.spark.api.java.JavaSparkContext
 
 /**
  * A function that returns key-value pairs (Tuple2<K, V>), and can be used to construct PairRDDs.
  */
 // PairFunction does not extend Function because some UDF functions, like map,
 // are overloaded for both Function and PairFunction.
-public abstract class PairFunction<T, K, V> extends WrappedFunction1<T, Tuple2<K, V>>
-  implements Serializable {
+abstract class PairFunction[T, K, V] extends WrappedFunction1[T, (K, V)] with Serializable {
 
-  public ClassTag<K> keyType() {
-    return (ClassTag<K>) ClassTag$.MODULE$.apply(Object.class);
-  }
+  def keyType(): ClassTag[K] = JavaSparkContext.fakeClassTag
 
-  public ClassTag<V> valueType() {
-    return (ClassTag<V>) ClassTag$.MODULE$.apply(Object.class);
-  }
+  def valueType(): ClassTag[V] = JavaSparkContext.fakeClassTag
 }