[TOREE-487][TOREE-488] Remove PySpark and SparkR interpreters

Instead, please use a supported kernel such IPython or IRKernel

Closes #166

Author: lresende

Dockerfile

@@ -26,7 +26,7 @@ RUN curl -sL https://deb.nodesource.com/setup_0.12 | bash - && \
     npm install -g bower
 
 # for pyspark demos
-ENV APACHE_SPARK_VERSION 2.0.0
+ENV APACHE_SPARK_VERSION 2.2.0
 
 RUN apt-get -y update && \
     apt-get -y install software-properties-common

Makefile

@@ -173,7 +173,7 @@ dist: dist/toree
 
 define JUPYTER_COMMAND
 pip install toree-$(BASE_VERSION).tar.gz
-jupyter toree install --interpreters=PySpark,SQL,Scala,SparkR
+jupyter toree install --interpreters=Scala,SQL
 cd /srv/toree/etc/examples/notebooks
 jupyter notebook --ip=* --no-browser
 endef
@@ -197,7 +197,7 @@ system-test: pip-release .system-test-image
 		$(SYSTEM_TEST_IMAGE) \
 		bash -c "(cd /srv/system-test-resources && python -m http.server 8000 &) && \
 		rm -rf /home/jovyan/.local/share/jupyter/kernels/apache_toree_scala/ && \
-		pip install /srv/toree-pip/toree*.tar.gz && jupyter toree install --interpreters=PySpark,Scala,SparkR && \
+		pip install /srv/toree-pip/toree*.tar.gz && jupyter toree install --interpreters=Scala && \
 		pip install nose jupyter_kernel_test && python /srv/test_toree.py"
 
 

build.sbt

@@ -122,10 +122,10 @@ credentials in ThisBuild+= Credentials(Path.userHome / ".ivy2" / ".credentials")
 lazy val root = (project in file("."))
   .settings(name := "toree")
   .aggregate(
-    macros,protocol,plugins,communication,kernelApi,client,scalaInterpreter,sqlInterpreter,pysparkInterpreter,sparkrInterpreter,kernel
+    macros,protocol,plugins,communication,kernelApi,client,scalaInterpreter,sqlInterpreter,kernel
   )
   .dependsOn(
-    macros,protocol,communication,kernelApi,client,scalaInterpreter,sqlInterpreter,pysparkInterpreter,sparkrInterpreter,kernel
+    macros,protocol,communication,kernelApi,client,scalaInterpreter,sqlInterpreter,kernel
   )
 
 /**
@@ -187,20 +187,6 @@ lazy val sqlInterpreter = (project in file("sql-interpreter"))
   .settings(name := "toree-sql-interpreter")
   .dependsOn(plugins, protocol, kernelApi, scalaInterpreter)
 
-/**
-* Project represents the Python interpreter used by the Spark Kernel.
-*/
-lazy val pysparkInterpreter = (project in file("pyspark-interpreter"))
-  .settings(name := "toree-pyspark-interpreter")
-  .dependsOn(plugins, protocol, kernelApi)
-
-/**
-* Project represents the R interpreter used by the Spark Kernel.
-*/
-lazy val sparkrInterpreter = (project in file("sparkr-interpreter"))
-  .settings(name := "toree-sparkr-interpreter")
-  .dependsOn(plugins, protocol, kernelApi)
-
 /**
 * Project representing the kernel code for the Spark Kernel backend.
 */
@@ -211,9 +197,7 @@ lazy val kernel = (project in file("kernel"))
     protocol % "test->test;compile->compile",
     communication % "test->test;compile->compile",
     kernelApi % "test->test;compile->compile",
-    pysparkInterpreter % "test->test;compile->compile",
     scalaInterpreter % "test->test;compile->compile",
-    sparkrInterpreter % "test->test;compile->compile",
     sqlInterpreter % "test->test;compile->compile"
   )
 

etc/examples/notebooks/sqlcontext_sharing.ipynb

@@ -27,10 +27,8 @@
 from jupyter_client.kernelspec import KernelSpec
 
 INTERPRETER_LANGUAGES = {
-    'PySpark' : 'python',
-    'SparkR' : 'r',
-    'SQL' : 'sql',
-    'Scala' : 'scala'
+    'Scala' : 'scala',
+    'SQL' : 'sql'
 }
 
 PYTHON_PATH = 'PYTHONPATH'
@@ -50,7 +48,7 @@ class ToreeInstall(InstallKernelSpec):
     jupyter toree install --spark_opts='--master=local[4]'
     jupyter toree install --kernel_name=toree_special
     jupyter toree install --toree_opts='--spark-context-initialization-mode none'
-    jupyter toree install --interpreters=PySpark,SQL
+    jupyter toree install --interpreters=SQL
     jupyter toree install --python=python
     '''
 

etc/pip_install/toree/toreeapp.py

@@ -18,9 +18,7 @@
 
 # Files from sparkr which should not be changed
 .lintr
-package-sparkR.sh
 .*md
-.*sparkr-interpreter/src/main/.*
 NAMESPACE
 DESCRIPTION
 .*gitignore

etc/tools/.rat-excludes

@@ -204,8 +204,6 @@ class CommandLineOptions(args: Seq[String]) {
   private def interpreterPlugins: Option[java.util.List[String]] = {
     //val defaults = getAll(_default_interpreter_plugin).getOrElse(List())
     //val defaults = List[String](
-    //  "PySpark:org.apache.toree.kernel.interpreter.pyspark.PySparkInterpreter",
-    //  "SparkR:org.apache.toree.kernel.interpreter.sparkr.SparkRInterpreter",
     //  "SQL:org.apache.toree.kernel.interpreter.sql.SqlInterpreter"
     //)