diff --git a/project/SparkBuild.scala b/project/SparkBuild.scala
index 4bfa8cede7a8565a0f4086e561dbfd6b96dc5282..a93aa12ce1aac96299bf73e8dfda167a5c64dc9c 100644
--- a/project/SparkBuild.scala
+++ b/project/SparkBuild.scala
@@ -163,6 +163,9 @@ object SparkBuild extends PomBuild {
x => enable(MimaBuild.mimaSettings(sparkHome, x))(x)
}
+ /* Unsafe settings */
+ enable(Unsafe.settings)(unsafe)
+
/* Enable Assembly for all assembly projects */
assemblyProjects.foreach(enable(Assembly.settings))
@@ -216,6 +219,13 @@ object SparkBuild extends PomBuild {
}
+object Unsafe {
+ lazy val settings = Seq(
+ // This option is needed to suppress warnings from sun.misc.Unsafe usage
+ javacOptions in Compile += "-XDignore.symbol.file"
+ )
+}
+
object Flume {
lazy val settings = sbtavro.SbtAvro.avroSettings
}
@@ -424,6 +434,7 @@ object Unidoc {
.map(_.filterNot(_.getCanonicalPath.contains("org/apache/spark/network")))
.map(_.filterNot(_.getCanonicalPath.contains("org/apache/spark/shuffle")))
.map(_.filterNot(_.getCanonicalPath.contains("org/apache/spark/executor")))
+ .map(_.filterNot(_.getCanonicalPath.contains("org/apache/spark/unsafe")))
.map(_.filterNot(_.getCanonicalPath.contains("python")))
.map(_.filterNot(_.getCanonicalPath.contains("org/apache/spark/util/collection")))
.map(_.filterNot(_.getCanonicalPath.contains("org/apache/spark/sql/catalyst")))
diff --git a/unsafe/pom.xml b/unsafe/pom.xml
index 8901d77591932292a7db3c4ae349bad9bb9a04b8..5b0733206b2bc5de85b57cc45771cc55b8170cb3 100644
--- a/unsafe/pom.xml
+++ b/unsafe/pom.xml
@@ -65,5 +65,29 @@
<build>
<outputDirectory>target/scala-${scala.binary.version}/classes</outputDirectory>
<testOutputDirectory>target/scala-${scala.binary.version}/test-classes</testOutputDirectory>
+ <pluginManagement>
+ <plugins>
+ <plugin>
+ <groupId>net.alchim31.maven</groupId>
+ <artifactId>scala-maven-plugin</artifactId>
+ <configuration>
+ <javacArgs>
+ <!-- This option is needed to suppress warnings from sun.misc.Unsafe usage -->
+ <javacArg>-XDignore.symbol.file</javacArg>
+ </javacArgs>
+ </configuration>
+ </plugin>
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-compiler-plugin</artifactId>
+ <configuration>
+ <compilerArgs>
+ <!-- This option is needed to suppress warnings from sun.misc.Unsafe usage -->
+ <arg>-XDignore.symbol.file</arg>
+ </compilerArgs>
+ </configuration>
+ </plugin>
+ </plugins>
+ </pluginManagement>
</build>
</project>
diff --git a/unsafe/src/main/java/org/apache/spark/unsafe/PlatformDependent.java b/unsafe/src/main/java/org/apache/spark/unsafe/PlatformDependent.java
index 91b2f9aa439211b842410a26ff49fdd139a51e35..24b289209805942f824529257d523c216b68dc50 100644
--- a/unsafe/src/main/java/org/apache/spark/unsafe/PlatformDependent.java
+++ b/unsafe/src/main/java/org/apache/spark/unsafe/PlatformDependent.java
@@ -23,7 +23,82 @@ import sun.misc.Unsafe;
public final class PlatformDependent {
- public static final Unsafe UNSAFE;
+ /**
+ * Facade in front of {@link sun.misc.Unsafe}, used to avoid directly exposing Unsafe outside of
+ * this package. This also lets us aovid accidental use of deprecated methods or methods that
+ * aren't present in Java 6.
+ */
+ public static final class UNSAFE {
+
+ private UNSAFE() { }
+
+ public static int getInt(Object object, long offset) {
+ return _UNSAFE.getInt(object, offset);
+ }
+
+ public static void putInt(Object object, long offset, int value) {
+ _UNSAFE.putInt(object, offset, value);
+ }
+
+ public static boolean getBoolean(Object object, long offset) {
+ return _UNSAFE.getBoolean(object, offset);
+ }
+
+ public static void putBoolean(Object object, long offset, boolean value) {
+ _UNSAFE.putBoolean(object, offset, value);
+ }
+
+ public static byte getByte(Object object, long offset) {
+ return _UNSAFE.getByte(object, offset);
+ }
+
+ public static void putByte(Object object, long offset, byte value) {
+ _UNSAFE.putByte(object, offset, value);
+ }
+
+ public static short getShort(Object object, long offset) {
+ return _UNSAFE.getShort(object, offset);
+ }
+
+ public static void putShort(Object object, long offset, short value) {
+ _UNSAFE.putShort(object, offset, value);
+ }
+
+ public static long getLong(Object object, long offset) {
+ return _UNSAFE.getLong(object, offset);
+ }
+
+ public static void putLong(Object object, long offset, long value) {
+ _UNSAFE.putLong(object, offset, value);
+ }
+
+ public static float getFloat(Object object, long offset) {
+ return _UNSAFE.getFloat(object, offset);
+ }
+
+ public static void putFloat(Object object, long offset, float value) {
+ _UNSAFE.putFloat(object, offset, value);
+ }
+
+ public static double getDouble(Object object, long offset) {
+ return _UNSAFE.getDouble(object, offset);
+ }
+
+ public static void putDouble(Object object, long offset, double value) {
+ _UNSAFE.putDouble(object, offset, value);
+ }
+
+ public static long allocateMemory(long size) {
+ return _UNSAFE.allocateMemory(size);
+ }
+
+ public static void freeMemory(long address) {
+ _UNSAFE.freeMemory(address);
+ }
+
+ }
+
+ private static final Unsafe _UNSAFE;
public static final int BYTE_ARRAY_OFFSET;
@@ -48,13 +123,13 @@ public final class PlatformDependent {
} catch (Throwable cause) {
unsafe = null;
}
- UNSAFE = unsafe;
+ _UNSAFE = unsafe;
- if (UNSAFE != null) {
- BYTE_ARRAY_OFFSET = UNSAFE.arrayBaseOffset(byte[].class);
- INT_ARRAY_OFFSET = UNSAFE.arrayBaseOffset(int[].class);
- LONG_ARRAY_OFFSET = UNSAFE.arrayBaseOffset(long[].class);
- DOUBLE_ARRAY_OFFSET = UNSAFE.arrayBaseOffset(double[].class);
+ if (_UNSAFE != null) {
+ BYTE_ARRAY_OFFSET = _UNSAFE.arrayBaseOffset(byte[].class);
+ INT_ARRAY_OFFSET = _UNSAFE.arrayBaseOffset(int[].class);
+ LONG_ARRAY_OFFSET = _UNSAFE.arrayBaseOffset(long[].class);
+ DOUBLE_ARRAY_OFFSET = _UNSAFE.arrayBaseOffset(double[].class);
} else {
BYTE_ARRAY_OFFSET = 0;
INT_ARRAY_OFFSET = 0;
@@ -71,7 +146,7 @@ public final class PlatformDependent {
long length) {
while (length > 0) {
long size = Math.min(length, UNSAFE_COPY_THRESHOLD);
- UNSAFE.copyMemory(src, srcOffset, dst, dstOffset, size);
+ _UNSAFE.copyMemory(src, srcOffset, dst, dstOffset, size);
length -= size;
srcOffset += size;
dstOffset += size;
@@ -82,6 +157,6 @@ public final class PlatformDependent {
* Raises an exception bypassing compiler checks for checked exceptions.
*/
public static void throwException(Throwable t) {
- UNSAFE.throwException(t);
+ _UNSAFE.throwException(t);
}
}
diff --git a/unsafe/src/main/java/org/apache/spark/unsafe/map/BytesToBytesMap.java b/unsafe/src/main/java/org/apache/spark/unsafe/map/BytesToBytesMap.java
index 85b64c083380380d1b2b999f5d199ddc49dfd86d..4e5ebc402be35849378a432f34aa59c9c0f0c4da 100644
--- a/unsafe/src/main/java/org/apache/spark/unsafe/map/BytesToBytesMap.java
+++ b/unsafe/src/main/java/org/apache/spark/unsafe/map/BytesToBytesMap.java
@@ -401,11 +401,11 @@ public final class BytesToBytesMap {
// Copy the key
PlatformDependent.UNSAFE.putLong(pageBaseObject, keySizeOffsetInPage, keyLengthBytes);
- PlatformDependent.UNSAFE.copyMemory(
+ PlatformDependent.copyMemory(
keyBaseObject, keyBaseOffset, pageBaseObject, keyDataOffsetInPage, keyLengthBytes);
// Copy the value
PlatformDependent.UNSAFE.putLong(pageBaseObject, valueSizeOffsetInPage, valueLengthBytes);
- PlatformDependent.UNSAFE.copyMemory(
+ PlatformDependent.copyMemory(
valueBaseObject, valueBaseOffset, pageBaseObject, valueDataOffsetInPage, valueLengthBytes);
final long storedKeyAddress = memoryManager.encodePageNumberAndOffset(
@@ -429,7 +429,7 @@ public final class BytesToBytesMap {
private void allocate(int capacity) {
capacity = Math.max((int) Math.min(Integer.MAX_VALUE, nextPowerOf2(capacity)), 64);
longArray = new LongArray(memoryManager.allocate(capacity * 8 * 2));
- bitset = new BitSet(memoryManager.allocate(capacity / 8).zero());
+ bitset = new BitSet(MemoryBlock.fromLongArray(new long[capacity / 64]));
this.growthThreshold = (int) (capacity * loadFactor);
this.mask = capacity - 1;
@@ -447,7 +447,7 @@ public final class BytesToBytesMap {
longArray = null;
}
if (bitset != null) {
- memoryManager.free(bitset.memoryBlock());
+ // The bitset's heap memory isn't managed by a memory manager, so no need to free it here.
bitset = null;
}
Iterator<MemoryBlock> dataPagesIterator = dataPages.iterator();
@@ -535,7 +535,6 @@ public final class BytesToBytesMap {
// Deallocate the old data structures.
memoryManager.free(oldLongArray.memoryBlock());
- memoryManager.free(oldBitSet.memoryBlock());
if (enablePerfMetrics) {
timeSpentResizingNs += System.nanoTime() - resizeStartTime;
}
diff --git a/unsafe/src/main/java/org/apache/spark/unsafe/memory/MemoryBlock.java b/unsafe/src/main/java/org/apache/spark/unsafe/memory/MemoryBlock.java
index 0beb743e5644eea312eedd235f2bde1a289ffd12..3dc82d8c2eb399ac86571126a29724c0d994c914 100644
--- a/unsafe/src/main/java/org/apache/spark/unsafe/memory/MemoryBlock.java
+++ b/unsafe/src/main/java/org/apache/spark/unsafe/memory/MemoryBlock.java
@@ -46,14 +46,6 @@ public class MemoryBlock extends MemoryLocation {
return length;
}
- /**
- * Clear the contents of this memory block. Returns `this` to facilitate chaining.
- */
- public MemoryBlock zero() {
- PlatformDependent.UNSAFE.setMemory(obj, offset, length, (byte) 0);
- return this;
- }
-
/**
* Creates a memory block pointing to the memory used by the long array.
*/
diff --git a/unsafe/src/test/java/org/apache/spark/unsafe/bitset/BitSetSuite.java b/unsafe/src/test/java/org/apache/spark/unsafe/bitset/BitSetSuite.java
index 4bf132fd4053e01320ecf006b6c4b86535de6fd9..e3a824e29b76869bee2bcec2b88e94e3d061fa5d 100644
--- a/unsafe/src/test/java/org/apache/spark/unsafe/bitset/BitSetSuite.java
+++ b/unsafe/src/test/java/org/apache/spark/unsafe/bitset/BitSetSuite.java
@@ -27,7 +27,7 @@ public class BitSetSuite {
private static BitSet createBitSet(int capacity) {
assert capacity % 64 == 0;
- return new BitSet(MemoryBlock.fromLongArray(new long[capacity / 64]).zero());
+ return new BitSet(MemoryBlock.fromLongArray(new long[capacity / 64]));
}
@Test
diff --git a/unsafe/src/test/java/org/apache/spark/unsafe/map/AbstractBytesToBytesMapSuite.java b/unsafe/src/test/java/org/apache/spark/unsafe/map/AbstractBytesToBytesMapSuite.java
index 9038cf567f1e2d5e3b3f0c221acf6bcfdb9024d2..7a5c0622d1ffbabd08cd61722548d319e98d1ba7 100644
--- a/unsafe/src/test/java/org/apache/spark/unsafe/map/AbstractBytesToBytesMapSuite.java
+++ b/unsafe/src/test/java/org/apache/spark/unsafe/map/AbstractBytesToBytesMapSuite.java
@@ -57,7 +57,7 @@ public abstract class AbstractBytesToBytesMapSuite {
private static byte[] getByteArray(MemoryLocation loc, int size) {
final byte[] arr = new byte[size];
- PlatformDependent.UNSAFE.copyMemory(
+ PlatformDependent.copyMemory(
loc.getBaseObject(),
loc.getBaseOffset(),
arr,