From 2579948bf5d89ac2d822ace605a6a4afce5258d6 Mon Sep 17 00:00:00 2001
From: cafreeman <cfreeman@alteryx.com>
Date: Fri, 26 Jun 2015 17:06:02 -0700
Subject: [PATCH] [SPARK-8607] SparkR -- jars not being added to application
classpath correctly
Add `getStaticClass` method in SparkR's `RBackendHandler`
This is a fix for the problem referenced in [SPARK-5185](https://issues.apache.org/jira/browse/SPARK-5185).
cc shivaram
Author: cafreeman <cfreeman@alteryx.com>
Closes #7001 from cafreeman/branch-1.4 and squashes the following commits:
8f81194 [cafreeman] Add missing license
31aedcf [cafreeman] Refactor test to call an external R script
2c22073 [cafreeman] Merge branch 'branch-1.4' of github.com:apache/spark into branch-1.4
0bea809 [cafreeman] Fixed relative path issue and added smaller JAR
ee25e60 [cafreeman] Merge branch 'branch-1.4' of github.com:apache/spark into branch-1.4
9a5c362 [cafreeman] test for including JAR when launching sparkContext
9101223 [cafreeman] Merge branch 'branch-1.4' of github.com:apache/spark into branch-1.4
5a80844 [cafreeman] Fix style nits
7c6bd0c [cafreeman] [SPARK-8607] SparkR
---
.../test_support/sparktestjar_2.10-1.0.jar | Bin 0 -> 2886 bytes
R/pkg/inst/tests/jarTest.R | 32 +++++++++++++++
R/pkg/inst/tests/test_includeJAR.R | 37 ++++++++++++++++++
.../apache/spark/api/r/RBackendHandler.scala | 17 +++++++-
4 files changed, 85 insertions(+), 1 deletion(-)
create mode 100644 R/pkg/inst/test_support/sparktestjar_2.10-1.0.jar
create mode 100644 R/pkg/inst/tests/jarTest.R
create mode 100644 R/pkg/inst/tests/test_includeJAR.R
diff --git a/R/pkg/inst/test_support/sparktestjar_2.10-1.0.jar b/R/pkg/inst/test_support/sparktestjar_2.10-1.0.jar
new file mode 100644
index 0000000000000000000000000000000000000000..1d5c2af631aa3ae88aa7836e8db598e59cbcf1b7
GIT binary patch
literal 2886
zcmaJ@2T)UK7Y!kyL@*d49Sj{JRS@YAsv!nq=*S|~fYJj-g+=LYfzbORNV8FN{bdCk
zg0zIbAfgZyq{;FFilHpKi8@1Yb?=)u^SycZJMX@^=brE2Fzg^WfQyR@ut^-V0I&oc
z00Lmm?NG{SYYSB@${KB9ZfmE4wb<zf0M@XSo)Z{ll>uIiGP0M_cNecVtU;Sur6_lz
zsaWb^v=SR+A;CLuy3$1vE+`}5lL)YnkQMN#MIA*nA&%;3C~FAI<>zOe_O2pl73Db<
z*~X|~OI8rlVFtFrMKPnCLMpTwAOMHq<Wlu3OCd1I+G5-|MzIh8fD7=iVmTRaiNPMB
z=kTgOPU6{33iBa`ea`&;zYBho>FeX~AEe^t??EK`oE#4vGUh9FCIkePXoe5stL%}&
zNVAhqLvP+N2Ki!4axnH{pCBmT>;l@Gm+pFq9mvWPc4#D^ejDn^&%Hv<bs+Q>UK^OB
z)EgN^0iUSck}R0#%xqa6FDyH=u4lyK#mnchVHF8GuTYWvvwq8}Wg!P7M*Y|;8%rrT
zSMkG(9`ZZmI9>ms=PmBAWd9<1I7oL}szYkqA=5BaS>~7Fg4Fu;I@PigC*Z->SEaFo
z)mA(U+Et_0mnO!HE930f3)`UwV_k-Irmgy4i(gJx=yLphKC9Z|?D+e)*g9|J@Rsv7
zk~1Pi$~vua^rxVv3bbNK;S<5F#?d~J`xnxM^8*4`fzJ2NyI^8D1%q_oI-HdKqvn++
z@j;zGC;;QDtGL>2zA`gW*&5@)PLxO097`UtpP@;T8Uuo*gl5_07hrMQ`oFAcb9Zpv
zU;eNo|IP=^hg}#?g_&HGTvq1g0WHlng}gi$D0%CHfbvHm+#?yqW(9U9_w{s}bsD60
zJbKlZTGQ2eSt2gWZfi(kU)^7K5xfcIeb*Fv%>>#q_2YJy&1isVF^WX@#n6hj6z{`$
z%jC^tPLH<4yHC$bm@gpo;104PgXP`zRac*-ITr)B!AzcE1SyllYyf~A@C(nrPYj{&
z5kuw+Gpg{PnPDKR7ye#IBnz)F3M|+5yB;wmr(<FQZZo}Fn_zdqCjBn<Cow(k{rQh8
z5VA1)1AUQj&yhx#+$a274b2+Fca*Wxjg7@|65!m0C%E^YO6HdGgx8|{{|udk_CSP8
zJ^Cd+w*qfkdANM2x=sUb?~(@z~Fp`6*`jMJlKIl^ZavNZ)v>)vg;(b}QHGS>#(
z3vS3(@8`Xdz_ZHZ5;L)1`Z2Y^eK=$BWymyG{N|B)PX$>RK0=zub0-sStx+Pv>1NSF
zCCw=kgsK<iBlFS~bL`;i#5aXrD&V^x8VDvsgOU4Zg~`vrNP6|oY=#p=MSbp&RL%W%
zl7xBYT5|cBZLkyiU{m>W_ELQGn0T9Y&+I!bpHCO`y5xKugYBh#)_91|y|bv3&bbNn
zhl?VRCe)^$J}q+=Qe+qHvz>9J*ohAqsNP^``th`sTtvmSm>alzorsQR37?+mp&3oj
zY<J!xs};}`E87OE{<MZtV^id4u#}B&gsj$pkt$7<<F!BrKI2ScZ0C%PfrocMIp0k)
znIXHOK5n-MXxw!Y^5RGrazHUJ^a_P*Gs?5Sg%$?!wZ$j&=CZ?!P6um6BoVRKwF$>&
z-k5K3sz5^m;*OY4n|5OT1|(XAJm)oLyn_?h$3hGP<!9OhgykA1tG4X$h8V7;<Ab`T
ztv4Mkovuf<zuY;HYq-DCt^)*XUzV*^`ZHDJF&mR;Q~6>O(hQPO*jb4N2-&N?h;NAI
z=|S}KvIr0K3iAsJ{7SfRa*uVZF+Ab#XH`{T@KaGilFF5MkgL^;S{WGyZiRC-RlWyO
z=G$vRO<l>pZ`kR$=NeU#AyYCMvqt$Fy_Aie4LY#jBp$cw7PqDEPtTh@#xpf`=2-BF
zNjKdZWD>bJ*>Rv?-)y70+L^rhn@RkT%g8=`TM8KUP}ueLr+ORi+?~H)>@&F$&>PUP
zt`~RVo|c@#Qa{x=v06Gob2O>Tdzy1dQPO$o<5>ff*2{Mcw5%I^VYT-&l@prsYNVsX
zI3RXuY@8*{uGUH>j~-8trp2Gz=N0(6E0(-8o*pi$#M6@|wdpI|RQ<=js%)_?XnEIF
z5Htms<LVINdwfMstkKc^aQ<fU3z0(mIll7^q6c}Tui)$S!>lKq+Jh;CpZU$T;;eF?
zxpCQ&39>;eWaswZk0LCAGCDUJ2Jb_)35SFHJ|10PT-J6CIOVOCTx+gwW=oG1vhuKp
z`HXD|GMhCjwbXK&p$fcvRN>oZ9r~|keG!N7%#%Sd6ki7+&^etwJrwvqoSM$1N*KS2
zyVq^A?D&|c{p8Sqd+4LiG{mX-(qIxc^8cb6)3H!PI@>lVO62We8|R@1UGFJIYdx3m
zYCPWArgH<7Usxz`l+icezS#d7@teFp-%!*v)^sWYi7;6&nhcHTdKhm|<k~!h1ILiJ
zp#C7+N7Gyx38eUk=VvkKm%SD`o>MDoF3kZT-3_HO6kWVS)v`toS%e?3*2|~%cZro`
z)VXA?Azv-4#lxOOl5G8Ij-%eTgldL}ZV-}0)X%oM3#AHbR)<u0Us_pBcl<4Il5k7B
zep}z>oLKXTqvz+lZ{3N@(p3yvbfn5G-99m*{uhm!9X}zvQGyzic<!dFnow^!mUG8v
zWJ6-U0$W&~|Ld>Gsj=}Bjwy);9H?Ff==SdETklfOmO6)|W6!QWPkfel(kTtipW5d1
z@LrE>9H@+1qE^&56Lbj5@sZ7BeuDcHvXXB&dthawfgqWa`03h`<{Wtd!F$TDA5oZ-
z8_l=4sUaviPan9nu-=YOd855*67tq9$@oN`%9_5>v?#yMX4~T`V}5wj(|1=tWTzjQ
ztwaJc#Q{sA-k9desi&iQNycsy5F9t4QbsJloP&HNu~-XC-^XN1_zY>^YX(ysKQo05
z2nXz*AgmsSX{+|ek4zR0!v=%^e(ZO4QEu<@@4q%N{m*U;GL~O0(^ogNw`kS_k?Dta
zW1F#L-O1vPn4f3;b5^lqo}IgKkRgBn0{JRztSHP`W1T|8E(Bv01>TGDJ(>I#jkQzE
i$=!{^3>(Q>(;l=hbBx1)IhY$b8P_<Y-S*tfr+)z9gl{7N
literal 0
HcmV?d00001
diff --git a/R/pkg/inst/tests/jarTest.R b/R/pkg/inst/tests/jarTest.R
new file mode 100644
index 0000000000..d68bb20950
--- /dev/null
+++ b/R/pkg/inst/tests/jarTest.R
@@ -0,0 +1,32 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+library(SparkR)
+
+sc <- sparkR.init()
+
+helloTest <- SparkR:::callJStatic("sparkR.test.hello",
+ "helloWorld",
+ "Dave")
+
+basicFunction <- SparkR:::callJStatic("sparkR.test.basicFunction",
+ "addStuff",
+ 2L,
+ 2L)
+
+sparkR.stop()
+output <- c(helloTest, basicFunction)
+writeLines(output)
diff --git a/R/pkg/inst/tests/test_includeJAR.R b/R/pkg/inst/tests/test_includeJAR.R
new file mode 100644
index 0000000000..8bc693be20
--- /dev/null
+++ b/R/pkg/inst/tests/test_includeJAR.R
@@ -0,0 +1,37 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+context("include an external JAR in SparkContext")
+
+runScript <- function() {
+ sparkHome <- Sys.getenv("SPARK_HOME")
+ jarPath <- paste("--jars",
+ shQuote(file.path(sparkHome, "R/lib/SparkR/test_support/sparktestjar_2.10-1.0.jar")))
+ scriptPath <- file.path(sparkHome, "R/lib/SparkR/tests/jarTest.R")
+ submitPath <- file.path(sparkHome, "bin/spark-submit")
+ res <- system2(command = submitPath,
+ args = c(jarPath, scriptPath),
+ stdout = TRUE)
+ tail(res, 2)
+}
+
+test_that("sparkJars tag in SparkContext", {
+ testOutput <- runScript()
+ helloTest <- testOutput[1]
+ expect_true(helloTest == "Hello, Dave")
+ basicFunction <- testOutput[2]
+ expect_true(basicFunction == 4L)
+})
diff --git a/core/src/main/scala/org/apache/spark/api/r/RBackendHandler.scala b/core/src/main/scala/org/apache/spark/api/r/RBackendHandler.scala
index 2e86984c66..4b8f7fe924 100644
--- a/core/src/main/scala/org/apache/spark/api/r/RBackendHandler.scala
+++ b/core/src/main/scala/org/apache/spark/api/r/RBackendHandler.scala
@@ -88,6 +88,21 @@ private[r] class RBackendHandler(server: RBackend)
ctx.close()
}
+ // Looks up a class given a class name. This function first checks the
+ // current class loader and if a class is not found, it looks up the class
+ // in the context class loader. Address [SPARK-5185]
+ def getStaticClass(objId: String): Class[_] = {
+ try {
+ val clsCurrent = Class.forName(objId)
+ clsCurrent
+ } catch {
+ // Use contextLoader if we can't find the JAR in the system class loader
+ case e: ClassNotFoundException =>
+ val clsContext = Class.forName(objId, true, Thread.currentThread().getContextClassLoader)
+ clsContext
+ }
+ }
+
def handleMethodCall(
isStatic: Boolean,
objId: String,
@@ -98,7 +113,7 @@ private[r] class RBackendHandler(server: RBackend)
var obj: Object = null
try {
val cls = if (isStatic) {
- Class.forName(objId)
+ getStaticClass(objId)
} else {
JVMObjectTracker.get(objId) match {
case None => throw new IllegalArgumentException("Object not found " + objId)
--
GitLab