From f3201aeeb06aae3b11e8cf6ee9693182dd896b32 Mon Sep 17 00:00:00 2001
From: Dongjoon Hyun <dongjoon@apache.org>
Date: Wed, 9 Mar 2016 10:12:23 +0000
Subject: [PATCH] [SPARK-13692][CORE][SQL] Fix trivial Coverity/Checkstyle
defects
## What changes were proposed in this pull request?
This issue fixes the following potential bugs and Java coding style detected by Coverity and Checkstyle.
- Implement both null and type checking in equals functions.
- Fix wrong type casting logic in SimpleJavaBean2.equals.
- Add `implement Cloneable` to `UTF8String` and `SortedIterator`.
- Remove dereferencing before null check in `AbstractBytesToBytesMapSuite`.
- Fix coding style: Add '{}' to single `for` statement in mllib examples.
- Remove unused imports in `ColumnarBatch` and `JavaKinesisStreamSuite`.
- Remove unused fields in `ChunkFetchIntegrationSuite`.
- Add `stop()` to prevent resource leak.
Please note that the last two checkstyle errors exist on newly added commits after [SPARK-13583](https://issues.apache.org/jira/browse/SPARK-13583).
## How was this patch tested?
manual via `./dev/lint-java` and Coverity site.
Author: Dongjoon Hyun <dongjoon@apache.org>
Closes #11530 from dongjoon-hyun/SPARK-13692.
---
.../apache/spark/network/ChunkFetchIntegrationSuite.java | 2 --
.../spark/network/RequestTimeoutIntegrationSuite.java | 2 +-
.../apache/spark/network/TransportClientFactorySuite.java | 2 ++
.../java/org/apache/spark/unsafe/types/UTF8String.java | 3 ++-
.../util/collection/unsafe/sort/UnsafeInMemorySorter.java | 2 +-
.../spark/unsafe/map/AbstractBytesToBytesMapSuite.java | 4 ++--
.../spark/examples/ml/JavaCountVectorizerExample.java | 2 ++
.../ml/JavaDecisionTreeClassificationExample.java | 2 ++
.../examples/ml/JavaDecisionTreeRegressionExample.java | 2 ++
.../spark/examples/ml/JavaSQLTransformerExample.java | 2 ++
.../org/apache/spark/examples/ml/JavaWord2VecExample.java | 2 ++
.../spark/examples/mllib/JavaAssociationRulesExample.java | 2 ++
.../spark/examples/mllib/JavaGaussianMixtureExample.java | 3 ++-
.../mllib/JavaGradientBoostingClassificationExample.java | 2 ++
.../mllib/JavaGradientBoostingRegressionExample.java | 2 ++
.../examples/mllib/JavaIsotonicRegressionExample.java | 2 ++
.../apache/spark/examples/mllib/JavaKMeansExample.java | 3 ++-
.../mllib/JavaLatentDirichletAllocationExample.java | 3 ++-
.../mllib/JavaMultiLabelClassificationMetricsExample.java | 2 ++
.../spark/examples/mllib/JavaNaiveBayesExample.java | 2 ++
.../spark/examples/mllib/JavaPrefixSpanExample.java | 2 ++
.../mllib/JavaRandomForestClassificationExample.java | 2 ++
.../examples/mllib/JavaRandomForestRegressionExample.java | 2 ++
.../spark/examples/mllib/JavaRankingMetricsExample.java | 2 ++
.../spark/examples/mllib/JavaRecommendationExample.java | 2 ++
.../examples/mllib/JavaRegressionMetricsExample.java | 2 ++
.../org/apache/spark/examples/mllib/JavaSVDExample.java | 5 ++++-
.../apache/spark/examples/mllib/JavaSimpleFPGrowth.java | 2 ++
.../spark/streaming/kinesis/JavaKinesisStreamSuite.java | 2 --
.../spark/sql/execution/vectorized/ColumnarBatch.java | 1 -
.../java/test/org/apache/spark/sql/JavaDatasetSuite.java | 8 +++++++-
31 files changed, 61 insertions(+), 15 deletions(-)
diff --git a/common/network-common/src/test/java/org/apache/spark/network/ChunkFetchIntegrationSuite.java b/common/network-common/src/test/java/org/apache/spark/network/ChunkFetchIntegrationSuite.java
index d17e986e17..6d62eaf35d 100644
--- a/common/network-common/src/test/java/org/apache/spark/network/ChunkFetchIntegrationSuite.java
+++ b/common/network-common/src/test/java/org/apache/spark/network/ChunkFetchIntegrationSuite.java
@@ -64,8 +64,6 @@ public class ChunkFetchIntegrationSuite {
static ManagedBuffer bufferChunk;
static ManagedBuffer fileChunk;
- private TransportConf transportConf;
-
@BeforeClass
public static void setUp() throws Exception {
int bufSize = 100000;
diff --git a/common/network-common/src/test/java/org/apache/spark/network/RequestTimeoutIntegrationSuite.java b/common/network-common/src/test/java/org/apache/spark/network/RequestTimeoutIntegrationSuite.java
index e2d026c66f..c0ff9dc5f5 100644
--- a/common/network-common/src/test/java/org/apache/spark/network/RequestTimeoutIntegrationSuite.java
+++ b/common/network-common/src/test/java/org/apache/spark/network/RequestTimeoutIntegrationSuite.java
@@ -55,7 +55,7 @@ public class RequestTimeoutIntegrationSuite {
private TransportConf conf;
// A large timeout that "shouldn't happen", for the sake of faulty tests not hanging forever.
- private final int FOREVER = 60 * 1000;
+ private static final int FOREVER = 60 * 1000;
@Before
public void setUp() throws Exception {
diff --git a/common/network-common/src/test/java/org/apache/spark/network/TransportClientFactorySuite.java b/common/network-common/src/test/java/org/apache/spark/network/TransportClientFactorySuite.java
index 9a89dd114f..44d16d5422 100644
--- a/common/network-common/src/test/java/org/apache/spark/network/TransportClientFactorySuite.java
+++ b/common/network-common/src/test/java/org/apache/spark/network/TransportClientFactorySuite.java
@@ -122,6 +122,8 @@ public class TransportClientFactorySuite {
for (TransportClient client : clients) {
client.close();
}
+
+ factory.close();
}
@Test
diff --git a/common/unsafe/src/main/java/org/apache/spark/unsafe/types/UTF8String.java b/common/unsafe/src/main/java/org/apache/spark/unsafe/types/UTF8String.java
index 87706d0b68..427a8315e0 100644
--- a/common/unsafe/src/main/java/org/apache/spark/unsafe/types/UTF8String.java
+++ b/common/unsafe/src/main/java/org/apache/spark/unsafe/types/UTF8String.java
@@ -44,7 +44,8 @@ import static org.apache.spark.unsafe.Platform.*;
* <p>
* Note: This is not designed for general use cases, should not be used outside SQL.
*/
-public final class UTF8String implements Comparable<UTF8String>, Externalizable, KryoSerializable {
+public final class UTF8String implements Comparable<UTF8String>, Externalizable, KryoSerializable,
+ Cloneable {
// These are only updated by readExternal() or read()
@Nonnull
diff --git a/core/src/main/java/org/apache/spark/util/collection/unsafe/sort/UnsafeInMemorySorter.java b/core/src/main/java/org/apache/spark/util/collection/unsafe/sort/UnsafeInMemorySorter.java
index cea0f0a0c6..145c3a1950 100644
--- a/core/src/main/java/org/apache/spark/util/collection/unsafe/sort/UnsafeInMemorySorter.java
+++ b/core/src/main/java/org/apache/spark/util/collection/unsafe/sort/UnsafeInMemorySorter.java
@@ -172,7 +172,7 @@ public final class UnsafeInMemorySorter {
pos++;
}
- public final class SortedIterator extends UnsafeSorterIterator {
+ public final class SortedIterator extends UnsafeSorterIterator implements Cloneable {
private final int numRecords;
private int position;
diff --git a/core/src/test/java/org/apache/spark/unsafe/map/AbstractBytesToBytesMapSuite.java b/core/src/test/java/org/apache/spark/unsafe/map/AbstractBytesToBytesMapSuite.java
index d8af2b336d..61b94b736d 100644
--- a/core/src/test/java/org/apache/spark/unsafe/map/AbstractBytesToBytesMapSuite.java
+++ b/core/src/test/java/org/apache/spark/unsafe/map/AbstractBytesToBytesMapSuite.java
@@ -64,7 +64,7 @@ public abstract class AbstractBytesToBytesMapSuite {
private TestMemoryManager memoryManager;
private TaskMemoryManager taskMemoryManager;
- private final long PAGE_SIZE_BYTES = 1L << 26; // 64 megabytes
+ private static final long PAGE_SIZE_BYTES = 1L << 26; // 64 megabytes
final LinkedList<File> spillFilesCreated = new LinkedList<File>();
File tempDir;
@@ -131,8 +131,8 @@ public abstract class AbstractBytesToBytesMapSuite {
Utils.deleteRecursively(tempDir);
tempDir = null;
- Assert.assertEquals(0L, taskMemoryManager.cleanUpAllAllocatedMemory());
if (taskMemoryManager != null) {
+ Assert.assertEquals(0L, taskMemoryManager.cleanUpAllAllocatedMemory());
long leakedMemory = taskMemoryManager.getMemoryConsumptionForThisTask();
taskMemoryManager = null;
Assert.assertEquals(0L, leakedMemory);
diff --git a/examples/src/main/java/org/apache/spark/examples/ml/JavaCountVectorizerExample.java b/examples/src/main/java/org/apache/spark/examples/ml/JavaCountVectorizerExample.java
index ac33adb652..872e5a07d1 100644
--- a/examples/src/main/java/org/apache/spark/examples/ml/JavaCountVectorizerExample.java
+++ b/examples/src/main/java/org/apache/spark/examples/ml/JavaCountVectorizerExample.java
@@ -65,5 +65,7 @@ public class JavaCountVectorizerExample {
cvModel.transform(df).show();
// $example off$
+
+ jsc.stop();
}
}
diff --git a/examples/src/main/java/org/apache/spark/examples/ml/JavaDecisionTreeClassificationExample.java b/examples/src/main/java/org/apache/spark/examples/ml/JavaDecisionTreeClassificationExample.java
index 482225e585..b5347b7650 100644
--- a/examples/src/main/java/org/apache/spark/examples/ml/JavaDecisionTreeClassificationExample.java
+++ b/examples/src/main/java/org/apache/spark/examples/ml/JavaDecisionTreeClassificationExample.java
@@ -95,5 +95,7 @@ public class JavaDecisionTreeClassificationExample {
(DecisionTreeClassificationModel) (model.stages()[2]);
System.out.println("Learned classification tree model:\n" + treeModel.toDebugString());
// $example off$
+
+ jsc.stop();
}
}
diff --git a/examples/src/main/java/org/apache/spark/examples/ml/JavaDecisionTreeRegressionExample.java b/examples/src/main/java/org/apache/spark/examples/ml/JavaDecisionTreeRegressionExample.java
index c7f1868dd1..9cb67be04a 100644
--- a/examples/src/main/java/org/apache/spark/examples/ml/JavaDecisionTreeRegressionExample.java
+++ b/examples/src/main/java/org/apache/spark/examples/ml/JavaDecisionTreeRegressionExample.java
@@ -83,5 +83,7 @@ public class JavaDecisionTreeRegressionExample {
(DecisionTreeRegressionModel) (model.stages()[1]);
System.out.println("Learned regression tree model:\n" + treeModel.toDebugString());
// $example off$
+
+ jsc.stop();
}
}
diff --git a/examples/src/main/java/org/apache/spark/examples/ml/JavaSQLTransformerExample.java b/examples/src/main/java/org/apache/spark/examples/ml/JavaSQLTransformerExample.java
index d55c70796a..a9d64d5e3f 100644
--- a/examples/src/main/java/org/apache/spark/examples/ml/JavaSQLTransformerExample.java
+++ b/examples/src/main/java/org/apache/spark/examples/ml/JavaSQLTransformerExample.java
@@ -55,5 +55,7 @@ public class JavaSQLTransformerExample {
sqlTrans.transform(df).show();
// $example off$
+
+ jsc.stop();
}
}
diff --git a/examples/src/main/java/org/apache/spark/examples/ml/JavaWord2VecExample.java b/examples/src/main/java/org/apache/spark/examples/ml/JavaWord2VecExample.java
index d472375ca9..a4a05af7c6 100644
--- a/examples/src/main/java/org/apache/spark/examples/ml/JavaWord2VecExample.java
+++ b/examples/src/main/java/org/apache/spark/examples/ml/JavaWord2VecExample.java
@@ -63,5 +63,7 @@ public class JavaWord2VecExample {
System.out.println(r);
}
// $example off$
+
+ jsc.stop();
}
}
diff --git a/examples/src/main/java/org/apache/spark/examples/mllib/JavaAssociationRulesExample.java b/examples/src/main/java/org/apache/spark/examples/mllib/JavaAssociationRulesExample.java
index 4d0f989819..189560e3fe 100644
--- a/examples/src/main/java/org/apache/spark/examples/mllib/JavaAssociationRulesExample.java
+++ b/examples/src/main/java/org/apache/spark/examples/mllib/JavaAssociationRulesExample.java
@@ -52,5 +52,7 @@ public class JavaAssociationRulesExample {
rule.javaAntecedent() + " => " + rule.javaConsequent() + ", " + rule.confidence());
}
// $example off$
+
+ sc.stop();
}
}
diff --git a/examples/src/main/java/org/apache/spark/examples/mllib/JavaGaussianMixtureExample.java b/examples/src/main/java/org/apache/spark/examples/mllib/JavaGaussianMixtureExample.java
index 4d1c64aa3c..3124411c82 100644
--- a/examples/src/main/java/org/apache/spark/examples/mllib/JavaGaussianMixtureExample.java
+++ b/examples/src/main/java/org/apache/spark/examples/mllib/JavaGaussianMixtureExample.java
@@ -44,8 +44,9 @@ public class JavaGaussianMixtureExample {
public Vector call(String s) {
String[] sarray = s.trim().split(" ");
double[] values = new double[sarray.length];
- for (int i = 0; i < sarray.length; i++)
+ for (int i = 0; i < sarray.length; i++) {
values[i] = Double.parseDouble(sarray[i]);
+ }
return Vectors.dense(values);
}
}
diff --git a/examples/src/main/java/org/apache/spark/examples/mllib/JavaGradientBoostingClassificationExample.java b/examples/src/main/java/org/apache/spark/examples/mllib/JavaGradientBoostingClassificationExample.java
index 80faabd232..0c2e4c928b 100644
--- a/examples/src/main/java/org/apache/spark/examples/mllib/JavaGradientBoostingClassificationExample.java
+++ b/examples/src/main/java/org/apache/spark/examples/mllib/JavaGradientBoostingClassificationExample.java
@@ -87,6 +87,8 @@ public class JavaGradientBoostingClassificationExample {
GradientBoostedTreesModel sameModel = GradientBoostedTreesModel.load(jsc.sc(),
"target/tmp/myGradientBoostingClassificationModel");
// $example off$
+
+ jsc.stop();
}
}
diff --git a/examples/src/main/java/org/apache/spark/examples/mllib/JavaGradientBoostingRegressionExample.java b/examples/src/main/java/org/apache/spark/examples/mllib/JavaGradientBoostingRegressionExample.java
index 216895b368..c1bc2098dc 100644
--- a/examples/src/main/java/org/apache/spark/examples/mllib/JavaGradientBoostingRegressionExample.java
+++ b/examples/src/main/java/org/apache/spark/examples/mllib/JavaGradientBoostingRegressionExample.java
@@ -92,5 +92,7 @@ public class JavaGradientBoostingRegressionExample {
GradientBoostedTreesModel sameModel = GradientBoostedTreesModel.load(jsc.sc(),
"target/tmp/myGradientBoostingRegressionModel");
// $example off$
+
+ jsc.stop();
}
}
diff --git a/examples/src/main/java/org/apache/spark/examples/mllib/JavaIsotonicRegressionExample.java b/examples/src/main/java/org/apache/spark/examples/mllib/JavaIsotonicRegressionExample.java
index 37e709b4cb..e632e35110 100644
--- a/examples/src/main/java/org/apache/spark/examples/mllib/JavaIsotonicRegressionExample.java
+++ b/examples/src/main/java/org/apache/spark/examples/mllib/JavaIsotonicRegressionExample.java
@@ -82,5 +82,7 @@ public class JavaIsotonicRegressionExample {
model.save(jsc.sc(), "target/tmp/myIsotonicRegressionModel");
IsotonicRegressionModel sameModel = IsotonicRegressionModel.load(jsc.sc(), "target/tmp/myIsotonicRegressionModel");
// $example off$
+
+ jsc.stop();
}
}
diff --git a/examples/src/main/java/org/apache/spark/examples/mllib/JavaKMeansExample.java b/examples/src/main/java/org/apache/spark/examples/mllib/JavaKMeansExample.java
index a24606a2e9..006d96d111 100644
--- a/examples/src/main/java/org/apache/spark/examples/mllib/JavaKMeansExample.java
+++ b/examples/src/main/java/org/apache/spark/examples/mllib/JavaKMeansExample.java
@@ -44,8 +44,9 @@ public class JavaKMeansExample {
public Vector call(String s) {
String[] sarray = s.split(" ");
double[] values = new double[sarray.length];
- for (int i = 0; i < sarray.length; i++)
+ for (int i = 0; i < sarray.length; i++) {
values[i] = Double.parseDouble(sarray[i]);
+ }
return Vectors.dense(values);
}
}
diff --git a/examples/src/main/java/org/apache/spark/examples/mllib/JavaLatentDirichletAllocationExample.java b/examples/src/main/java/org/apache/spark/examples/mllib/JavaLatentDirichletAllocationExample.java
index 4d8b65c544..578564eeb2 100644
--- a/examples/src/main/java/org/apache/spark/examples/mllib/JavaLatentDirichletAllocationExample.java
+++ b/examples/src/main/java/org/apache/spark/examples/mllib/JavaLatentDirichletAllocationExample.java
@@ -49,8 +49,9 @@ public class JavaLatentDirichletAllocationExample {
public Vector call(String s) {
String[] sarray = s.trim().split(" ");
double[] values = new double[sarray.length];
- for (int i = 0; i < sarray.length; i++)
+ for (int i = 0; i < sarray.length; i++) {
values[i] = Double.parseDouble(sarray[i]);
+ }
return Vectors.dense(values);
}
}
diff --git a/examples/src/main/java/org/apache/spark/examples/mllib/JavaMultiLabelClassificationMetricsExample.java b/examples/src/main/java/org/apache/spark/examples/mllib/JavaMultiLabelClassificationMetricsExample.java
index 4717155438..5904260e2d 100644
--- a/examples/src/main/java/org/apache/spark/examples/mllib/JavaMultiLabelClassificationMetricsExample.java
+++ b/examples/src/main/java/org/apache/spark/examples/mllib/JavaMultiLabelClassificationMetricsExample.java
@@ -74,5 +74,7 @@ public class JavaMultiLabelClassificationMetricsExample {
// Subset accuracy
System.out.format("Subset accuracy = %f\n", metrics.subsetAccuracy());
// $example off$
+
+ sc.stop();
}
}
diff --git a/examples/src/main/java/org/apache/spark/examples/mllib/JavaNaiveBayesExample.java b/examples/src/main/java/org/apache/spark/examples/mllib/JavaNaiveBayesExample.java
index e6a5904bd7..478e615123 100644
--- a/examples/src/main/java/org/apache/spark/examples/mllib/JavaNaiveBayesExample.java
+++ b/examples/src/main/java/org/apache/spark/examples/mllib/JavaNaiveBayesExample.java
@@ -60,5 +60,7 @@ public class JavaNaiveBayesExample {
model.save(jsc.sc(), "target/tmp/myNaiveBayesModel");
NaiveBayesModel sameModel = NaiveBayesModel.load(jsc.sc(), "target/tmp/myNaiveBayesModel");
// $example off$
+
+ jsc.stop();
}
}
diff --git a/examples/src/main/java/org/apache/spark/examples/mllib/JavaPrefixSpanExample.java b/examples/src/main/java/org/apache/spark/examples/mllib/JavaPrefixSpanExample.java
index 68ec7c1e6e..1634075941 100644
--- a/examples/src/main/java/org/apache/spark/examples/mllib/JavaPrefixSpanExample.java
+++ b/examples/src/main/java/org/apache/spark/examples/mllib/JavaPrefixSpanExample.java
@@ -51,5 +51,7 @@ public class JavaPrefixSpanExample {
System.out.println(freqSeq.javaSequence() + ", " + freqSeq.freq());
}
// $example off$
+
+ sc.stop();
}
}
diff --git a/examples/src/main/java/org/apache/spark/examples/mllib/JavaRandomForestClassificationExample.java b/examples/src/main/java/org/apache/spark/examples/mllib/JavaRandomForestClassificationExample.java
index 9219eef1ad..f4c9d8a35d 100644
--- a/examples/src/main/java/org/apache/spark/examples/mllib/JavaRandomForestClassificationExample.java
+++ b/examples/src/main/java/org/apache/spark/examples/mllib/JavaRandomForestClassificationExample.java
@@ -85,5 +85,7 @@ public class JavaRandomForestClassificationExample {
RandomForestModel sameModel = RandomForestModel.load(jsc.sc(),
"target/tmp/myRandomForestClassificationModel");
// $example off$
+
+ jsc.stop();
}
}
diff --git a/examples/src/main/java/org/apache/spark/examples/mllib/JavaRandomForestRegressionExample.java b/examples/src/main/java/org/apache/spark/examples/mllib/JavaRandomForestRegressionExample.java
index 4db926a421..c71125ce41 100644
--- a/examples/src/main/java/org/apache/spark/examples/mllib/JavaRandomForestRegressionExample.java
+++ b/examples/src/main/java/org/apache/spark/examples/mllib/JavaRandomForestRegressionExample.java
@@ -91,5 +91,7 @@ public class JavaRandomForestRegressionExample {
RandomForestModel sameModel = RandomForestModel.load(jsc.sc(),
"target/tmp/myRandomForestRegressionModel");
// $example off$
+
+ jsc.stop();
}
}
diff --git a/examples/src/main/java/org/apache/spark/examples/mllib/JavaRankingMetricsExample.java b/examples/src/main/java/org/apache/spark/examples/mllib/JavaRankingMetricsExample.java
index 4ad2104763..54dfc404ca 100644
--- a/examples/src/main/java/org/apache/spark/examples/mllib/JavaRankingMetricsExample.java
+++ b/examples/src/main/java/org/apache/spark/examples/mllib/JavaRankingMetricsExample.java
@@ -181,5 +181,7 @@ public class JavaRankingMetricsExample {
// R-squared
System.out.format("R-squared = %f\n", regressionMetrics.r2());
// $example off$
+
+ sc.stop();
}
}
diff --git a/examples/src/main/java/org/apache/spark/examples/mllib/JavaRecommendationExample.java b/examples/src/main/java/org/apache/spark/examples/mllib/JavaRecommendationExample.java
index c179e7578c..5e643420ad 100644
--- a/examples/src/main/java/org/apache/spark/examples/mllib/JavaRecommendationExample.java
+++ b/examples/src/main/java/org/apache/spark/examples/mllib/JavaRecommendationExample.java
@@ -93,5 +93,7 @@ public class JavaRecommendationExample {
MatrixFactorizationModel sameModel = MatrixFactorizationModel.load(jsc.sc(),
"target/tmp/myCollaborativeFilter");
// $example off$
+
+ jsc.stop();
}
}
diff --git a/examples/src/main/java/org/apache/spark/examples/mllib/JavaRegressionMetricsExample.java b/examples/src/main/java/org/apache/spark/examples/mllib/JavaRegressionMetricsExample.java
index 4e89dd0c37..b3e5c04759 100644
--- a/examples/src/main/java/org/apache/spark/examples/mllib/JavaRegressionMetricsExample.java
+++ b/examples/src/main/java/org/apache/spark/examples/mllib/JavaRegressionMetricsExample.java
@@ -88,5 +88,7 @@ public class JavaRegressionMetricsExample {
LinearRegressionModel sameModel = LinearRegressionModel.load(sc.sc(),
"target/tmp/LogisticRegressionModel");
// $example off$
+
+ sc.stop();
}
}
diff --git a/examples/src/main/java/org/apache/spark/examples/mllib/JavaSVDExample.java b/examples/src/main/java/org/apache/spark/examples/mllib/JavaSVDExample.java
index f3685db9f2..b417da8f85 100644
--- a/examples/src/main/java/org/apache/spark/examples/mllib/JavaSVDExample.java
+++ b/examples/src/main/java/org/apache/spark/examples/mllib/JavaSVDExample.java
@@ -40,6 +40,7 @@ public class JavaSVDExample {
public static void main(String[] args) {
SparkConf conf = new SparkConf().setAppName("SVD Example");
SparkContext sc = new SparkContext(conf);
+ JavaSparkContext jsc = JavaSparkContext.fromSparkContext(sc);
// $example on$
double[][] array = {{1.12, 2.05, 3.12}, {5.56, 6.28, 8.94}, {10.2, 8.0, 20.5}};
@@ -48,7 +49,7 @@ public class JavaSVDExample {
Vector currentRow = Vectors.dense(array[i]);
rowsList.add(currentRow);
}
- JavaRDD<Vector> rows = JavaSparkContext.fromSparkContext(sc).parallelize(rowsList);
+ JavaRDD<Vector> rows = jsc.parallelize(rowsList);
// Create a RowMatrix from JavaRDD<Vector>.
RowMatrix mat = new RowMatrix(rows.rdd());
@@ -66,5 +67,7 @@ public class JavaSVDExample {
}
System.out.println("Singular values are: " + s);
System.out.println("V factor is:\n" + V);
+
+ jsc.stop();
}
}
diff --git a/examples/src/main/java/org/apache/spark/examples/mllib/JavaSimpleFPGrowth.java b/examples/src/main/java/org/apache/spark/examples/mllib/JavaSimpleFPGrowth.java
index 72edaca5e9..7f4fe60042 100644
--- a/examples/src/main/java/org/apache/spark/examples/mllib/JavaSimpleFPGrowth.java
+++ b/examples/src/main/java/org/apache/spark/examples/mllib/JavaSimpleFPGrowth.java
@@ -67,5 +67,7 @@ public class JavaSimpleFPGrowth {
rule.javaAntecedent() + " => " + rule.javaConsequent() + ", " + rule.confidence());
}
// $example off$
+
+ sc.stop();
}
}
diff --git a/extras/kinesis-asl/src/test/java/org/apache/spark/streaming/kinesis/JavaKinesisStreamSuite.java b/extras/kinesis-asl/src/test/java/org/apache/spark/streaming/kinesis/JavaKinesisStreamSuite.java
index 3f0f6793d2..5c2371c543 100644
--- a/extras/kinesis-asl/src/test/java/org/apache/spark/streaming/kinesis/JavaKinesisStreamSuite.java
+++ b/extras/kinesis-asl/src/test/java/org/apache/spark/streaming/kinesis/JavaKinesisStreamSuite.java
@@ -28,8 +28,6 @@ import org.apache.spark.streaming.api.java.JavaDStream;
import com.amazonaws.services.kinesis.clientlibrary.lib.worker.InitialPositionInStream;
-import java.nio.ByteBuffer;
-
/**
* Demonstrate the use of the KinesisUtils Java API
*/
diff --git a/sql/core/src/main/java/org/apache/spark/sql/execution/vectorized/ColumnarBatch.java b/sql/core/src/main/java/org/apache/spark/sql/execution/vectorized/ColumnarBatch.java
index 18763672c6..9a8aedfa56 100644
--- a/sql/core/src/main/java/org/apache/spark/sql/execution/vectorized/ColumnarBatch.java
+++ b/sql/core/src/main/java/org/apache/spark/sql/execution/vectorized/ColumnarBatch.java
@@ -22,7 +22,6 @@ import java.util.Iterator;
import org.apache.commons.lang.NotImplementedException;
import org.apache.spark.memory.MemoryMode;
-import org.apache.spark.sql.Column;
import org.apache.spark.sql.catalyst.InternalRow;
import org.apache.spark.sql.catalyst.expressions.GenericMutableRow;
import org.apache.spark.sql.catalyst.expressions.UnsafeRow;
diff --git a/sql/core/src/test/java/test/org/apache/spark/sql/JavaDatasetSuite.java b/sql/core/src/test/java/test/org/apache/spark/sql/JavaDatasetSuite.java
index e0e56f3fbf..9b624f318c 100644
--- a/sql/core/src/test/java/test/org/apache/spark/sql/JavaDatasetSuite.java
+++ b/sql/core/src/test/java/test/org/apache/spark/sql/JavaDatasetSuite.java
@@ -465,6 +465,9 @@ public class JavaDatasetSuite implements Serializable {
@Override
public boolean equals(Object other) {
+ if (this == other) return true;
+ if (other == null || getClass() != other.getClass()) return false;
+
return this.value.equals(((KryoSerializable) other).value);
}
@@ -483,6 +486,9 @@ public class JavaDatasetSuite implements Serializable {
@Override
public boolean equals(Object other) {
+ if (this == other) return true;
+ if (other == null || getClass() != other.getClass()) return false;
+
return this.value.equals(((JavaSerializable) other).value);
}
@@ -631,7 +637,7 @@ public class JavaDatasetSuite implements Serializable {
if (this == o) return true;
if (o == null || getClass() != o.getClass()) return false;
- SimpleJavaBean that = (SimpleJavaBean) o;
+ SimpleJavaBean2 that = (SimpleJavaBean2) o;
if (!a.equals(that.a)) return false;
if (!b.equals(that.b)) return false;
--
GitLab