From d52660c96925291eb7b2ce32cb6ad72dc50eade7 Mon Sep 17 00:00:00 2001
From: Matei Zaharia <matei@eecs.berkeley.edu>
Date: Tue, 31 May 2011 19:23:15 -0700
Subject: [PATCH] Ported code generation changes from 2.8 interpreter (to use a
class for each line's object rather than a singleton object so that we can
ship these classes to worker nodes). This is pretty hairy stuff, which would
be nice to avoid in the future by integrating with the interpreter some other
way.
---
.../main/scala/spark/repl/SparkIMain.scala | 34 +++++++++++++++----
.../main/scala/spark/repl/SparkImports.scala | 14 +++++++-
spark-shell | 2 +-
3 files changed, 41 insertions(+), 9 deletions(-)
diff --git a/repl/src/main/scala/spark/repl/SparkIMain.scala b/repl/src/main/scala/spark/repl/SparkIMain.scala
index 1fdc4eaf8a..49d0a27030 100644
--- a/repl/src/main/scala/spark/repl/SparkIMain.scala
+++ b/repl/src/main/scala/spark/repl/SparkIMain.scala
@@ -593,6 +593,7 @@ class SparkIMain(val settings: Settings, protected val out: PrintWriter) extends
*/
def close() {
reporter.flush()
+ classServer.stop()
}
/** Here is where we:
@@ -646,7 +647,9 @@ class SparkIMain(val settings: Settings, protected val out: PrintWriter) extends
* following accessPath into the outer one.
*/
def resolvePathToSymbol(accessPath: String): Symbol = {
- val readRoot = definitions.getModule(readPath) // the outermost wrapper
+ //val readRoot = definitions.getModule(readPath) // the outermost wrapper
+ // MATEI: changed this to getClass because the root object is no longer a module (Scala singleton object)
+ val readRoot = definitions.getClass(readPath) // the outermost wrapper
(accessPath split '.').foldLeft(readRoot) { (sym, name) =>
if (name == "") sym else
lineAfterTyper(sym.info member newTermName(name))
@@ -704,7 +707,8 @@ class SparkIMain(val settings: Settings, protected val out: PrintWriter) extends
/** Code to access a variable with the specified name */
def fullPath(vname: String) = (
- lineRep.readPath + accessPath + ".`%s`".format(vname)
+ //lineRep.readPath + accessPath + ".`%s`".format(vname)
+ lineRep.readPath + ".INSTANCE" + accessPath + ".`%s`".format(vname)
)
/** Same as fullpath, but after it has been flattened, so:
* $line5.$iw.$iw.$iw.Bippy // fullPath
@@ -721,12 +725,23 @@ class SparkIMain(val settings: Settings, protected val out: PrintWriter) extends
/** generate the source code for the object that computes this request */
private object ObjectSourceCode extends CodeAssembler[MemberHandler] {
+ val preamble = """
+ |class %s extends Serializable {
+ | %s%s
+ """.stripMargin.format(lineRep.readName, importsPreamble, indentCode(toCompute))
+ val postamble = importsTrailer + "\n}" + "\n" +
+ "object " + lineRep.readName + " {\n" +
+ " val INSTANCE = new " + lineRep.readName + "();\n" +
+ "}\n"
+ val generate = (m: MemberHandler) => m extraCodeToEvaluate Request.this
+ /*
val preamble = """
|object %s {
| %s%s
""".stripMargin.format(lineRep.readName, importsPreamble, indentCode(toCompute))
val postamble = importsTrailer + "\n}"
val generate = (m: MemberHandler) => m extraCodeToEvaluate Request.this
+ */
}
private object ResultObjectSourceCode extends CodeAssembler[MemberHandler] {
@@ -753,7 +768,7 @@ class SparkIMain(val settings: Settings, protected val out: PrintWriter) extends
| %s
| (""
""".stripMargin.format(
- lineRep.evalName, evalResult, executionWrapper, lineRep.readName + accessPath
+ lineRep.evalName, evalResult, executionWrapper, lineRep.readName + ".INSTANCE" + accessPath
)
val postamble = """
@@ -794,8 +809,12 @@ class SparkIMain(val settings: Settings, protected val out: PrintWriter) extends
def applyToResultMember[T](name: Name, f: Symbol => T) = lineAfterTyper(f(resultSymbol.info.nonPrivateDecl(name)))
/* typeOf lookup with encoding */
- def lookupTypeOf(name: Name) = typeOf.getOrElse(name, typeOf(global.encode(name.toString)))
- def simpleNameOfType(name: TypeName) = (compilerTypeOf get name) map (_.typeSymbol.simpleName)
+ def lookupTypeOf(name: Name) = {
+ typeOf.getOrElse(name, typeOf(global.encode(name.toString)))
+ }
+ def simpleNameOfType(name: TypeName) = {
+ (compilerTypeOf get name) map (_.typeSymbol.simpleName)
+ }
private def typeMap[T](f: Type => T): Map[Name, T] = {
def toType(name: Name): T = {
@@ -1036,7 +1055,7 @@ class SparkIMain(val settings: Settings, protected val out: PrintWriter) extends
/** Secret bookcase entrance for repl debuggers: end the line
* with "// show" and see what's going on.
*/
- if (code.lines exists (_.trim endsWith "// show")) {
+ if (SPARK_DEBUG_REPL || code.lines.exists(_.trim endsWith "// show")) {
echo(code)
parse(code) foreach (ts => ts foreach (t => withoutUnwrapping(DBG(asCompactString(t)))))
}
@@ -1060,7 +1079,8 @@ object SparkIMain {
// $line3.$read$$iw$$iw$Bippy@4a6a00ca
private def removeLineWrapper(s: String) = s.replaceAll("""\$line\d+[./]\$(read|eval|print)[$.]""", "")
private def removeIWPackages(s: String) = s.replaceAll("""\$(iw|read|eval|print)[$.]""", "")
- def stripString(s: String) = removeIWPackages(removeLineWrapper(s))
+ private def removeSparkVals(s: String) = s.replaceAll("""\$VAL[0-9]+[$.]""", "")
+ def stripString(s: String) = removeSparkVals(removeIWPackages(removeLineWrapper(s)))
trait CodeAssembler[T] {
def preamble: String
diff --git a/repl/src/main/scala/spark/repl/SparkImports.scala b/repl/src/main/scala/spark/repl/SparkImports.scala
index e62c9245ea..bd28395f2e 100644
--- a/repl/src/main/scala/spark/repl/SparkImports.scala
+++ b/repl/src/main/scala/spark/repl/SparkImports.scala
@@ -177,7 +177,12 @@ trait SparkImports {
for (imv <- x.definedNames) {
if (currentImps contains imv) addWrapper()
- code append ("import %s\n" format (req fullPath imv))
+ val objName = req.lineRep.readPath
+ val valName = "$VAL" + newValId();
+ code.append("val " + valName + " = " + objName + ".INSTANCE;\n")
+ code.append("import " + valName + req.accessPath + ".`" + imv + "`;\n")
+
+ //code append ("import %s\n" format (req fullPath imv))
currentImps += imv
}
}
@@ -193,4 +198,11 @@ trait SparkImports {
private def membersAtPickler(sym: Symbol): List[Symbol] =
atPickler(sym.info.nonPrivateMembers)
+
+ private var curValId = 0
+
+ private def newValId(): Int = {
+ curValId += 1
+ curValId
+ }
}
diff --git a/spark-shell b/spark-shell
index 009f96fe98..29e5e65da2 100755
--- a/spark-shell
+++ b/spark-shell
@@ -1,3 +1,3 @@
#!/bin/sh
FWDIR="`dirname $0`"
-exec $FWDIR/run spark.repl.Main $@
+exec $FWDIR/run spark.repl.Main "$@"
--
GitLab