Merge branch 'master' into release

CHANGELOG.md

@@ -2,6 +2,14 @@
 
 ## NEXT RELEASE
 
+## 0.19.0
+
+### Features
+
+* Added one internal magic to enable retry of session creation. Thanks @edwardps
+* New `%%pretty` magic for pretty printing a dataframe as an HTML table. Thanks @hegary 
+* Update Endpoint widget to shield passwords when entering them in the ipywidget. Thanks @J0rg3M3nd3z @jodom961
+
 ## 0.18.0
 
 ### Updates

autovizwidget/autovizwidget/__init__.py

@@ -1 +1 @@
-__version__ = '0.18.0'
+__version__ = '0.19.0'

autovizwidget/autovizwidget/tests/test_autovizwidget.py

@@ -1,7 +1,7 @@
 from mock import MagicMock, call
 from nose.tools import with_setup, assert_equals
 import pandas as pd
-from pandas.util.testing import assert_frame_equal, assert_series_equal
+from pandas.testing import assert_frame_equal, assert_series_equal
 
 from ..widget.autovizwidget import AutoVizWidget
 from ..widget.encoding import Encoding

examples/Magics in IPython Kernel.ipynb

@@ -11,7 +11,7 @@
    "cell_type": "markdown",
    "metadata": {},
    "source": [
-    "## This notebook will demonstrate how we can use the spark magic to interspere our Python code with code that is running against a Spark cluster"
+    "## This notebook will demonstrate how we can use the spark magic to intersperse our Python code with code that is running against a Spark cluster"
    ]
   },
   {

examples/Pyspark Kernel.ipynb

@@ -681,7 +681,53 @@
     "Use '`%%cleanup -f`' magic to delete all of the sessions for this cluster, including this notebook's session.\n",
     "The force flag `-f` is mandatory."
    ]
-  }
+  },
+  {
+    "cell_type": "markdown",
+    "metadata": {},
+    "source": [
+     "### Pretty Print Spark Dataframes (%%pretty)\n",
+     "\n",
+     "Use '`%%pretty`' magic to display a Spark dataframe as a HTML formatted table"
+    ]
+   },
+  {
+    "cell_type": "code",
+    "execution_count": 3,
+    "metadata": {},
+    "outputs": [
+     {
+      "data": {
+       "application/vnd.jupyter.widget-view+json": {
+        "model_id": "",
+        "version_major": 2,
+        "version_minor": 0
+       },
+       "text/plain": [
+       ]
+      },
+      "metadata": {},
+      "output_type": "display_data"
+     },
+     {
+      "data": {
+       "text/html": [
+        "<table><tr><th>age</th><th>name</th></tr><tr><td>null</td><td>Michael</td></tr><tr><td>30</td><td>Andy</td></tr><tr><td>19</td><td>Justin</td></tr></table><br /><pre></pre>"
+       ],
+       "text/plain": [
+        "<IPython.core.display.HTML object>"
+       ]
+      },
+      "metadata": {},
+      "output_type": "display_data"
+     }
+    ],
+    "source": [
+     "%%pretty\n",
+     "df = spark.read.json(\"/apps/spark-2.3.3/examples/src/main/resources/people.json\")\n",
+     "df.show()"
+    ]
+   }
  ],
  "metadata": {
   "kernelspec": {

examples/Spark Kernel.ipynb

@@ -527,6 +527,59 @@
     "Use '`%%cleanup -f`' magic to delete all of the sessions for this cluster, including this notebook's session.\n",
     "The force flag `-f` is mandatory."
    ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "### Pretty Print Spark Dataframes (%%pretty)\n",
+    "\n",
+    "Use '`%%pretty`' magic to display a Scala Spark dataframe as a HTML formatted table"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": 7,
+   "metadata": {},
+   "outputs": [
+    {
+     "data": {
+      "application/vnd.jupyter.widget-view+json": {
+       "model_id": "",
+       "version_major": 2,
+       "version_minor": 0
+      },
+      "text/plain": [
+       "FloatProgress(value=0.0, bar_style='info', description='Progress:', layout=Layout(height='25px', width='50%'),…"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    },
+    {
+     "data": {
+      "text/html": [
+       "<table><tr><th>col1</th><th>col2</th><th>col3</th></tr><tr><td>28</td><td>44</td><td>36</td></tr><tr><td>16</td><td>41</td><td>72</td></tr><tr><td>27</td><td>14</td><td>45</td></tr></table><br /><pre></pre>"
+      ],
+      "text/plain": [
+       "<IPython.core.display.HTML object>"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    }
+   ],
+   "source": [
+    "%%pretty\n",
+    "df.show()"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": []
   }
  ],
  "metadata": {

hdijupyterutils/hdijupyterutils/__init__.py

@@ -1 +1 @@
-__version__ = '0.18.0'
+__version__ = '0.19.0'

hdijupyterutils/hdijupyterutils/ipywidgetfactory.py

@@ -1,7 +1,7 @@
 # Copyright (c) 2015  aggftw@gmail.com
 # Distributed under the terms of the Modified BSD License.
 
-from ipywidgets import VBox, Output, Button, HTML, HBox, Dropdown, Checkbox, ToggleButtons, Text, Textarea, Tab
+from ipywidgets import VBox, Output, Button, HTML, HBox, Dropdown, Checkbox, ToggleButtons, Text, Textarea, Tab, Password
 
 
 class IpyWidgetFactory(object):
@@ -43,6 +43,10 @@ def get_toggle_buttons(**kwargs):
     def get_text(**kwargs):
         return Text(**kwargs)
 
+    @staticmethod
+    def get_password(**kwargs):
+        return Password(**kwargs)
+
     @staticmethod
     def get_text_area(**kwargs):
         return Textarea(**kwargs)

sparkmagic/sparkmagic/__init__.py

@@ -1,4 +1,4 @@
-__version__ = '0.18.0'
+__version__ = '0.19.0'
 
 from sparkmagic.serverextension.handlers import load_jupyter_server_extension
 

sparkmagic/sparkmagic/auth/basic.py

@@ -18,7 +18,7 @@ def __init__(self, parsed_attributes=None):
             is created from parsing %spark magic command.
         """
         if parsed_attributes is not None:
-            if parsed_attributes.user is '' or parsed_attributes.password is '':
+            if parsed_attributes.user == '' or parsed_attributes.password == '':
                 new_exc = BadUserDataException("Need to supply username and password arguments for "\
                     "Basic Access Authentication. (e.g. -a username -p password).")
                 raise new_exc
@@ -47,7 +47,7 @@ def get_widgets(self, widget_width):
             width=widget_width
         )
 
-        self.password_widget = ipywidget_factory.get_text(
+        self.password_widget = ipywidget_factory.get_password(
             description='Password:',
             value=self.password,
             width=widget_width

sparkmagic/sparkmagic/kernels/kernelmagics.py

@@ -18,9 +18,10 @@
 from sparkmagic.utils.utils import parse_argstring_or_throw, get_coerce_value, initialize_auth, Namespace
 from sparkmagic.utils.sparkevents import SparkEvents
 from sparkmagic.utils.constants import LANGS_SUPPORTED
+from sparkmagic.utils.dataframe_parser import cell_contains_dataframe, CellOutputHtmlParser
 from sparkmagic.livyclientlib.command import Command
 from sparkmagic.livyclientlib.endpoint import Endpoint
-from sparkmagic.magics.sparkmagicsbase import SparkMagicBase
+from sparkmagic.magics.sparkmagicsbase import SparkMagicBase, SparkOutputHandler
 from sparkmagic.livyclientlib.exceptions import handle_expected_exceptions, wrap_unexpected_exceptions, \
     BadUserDataException
 
@@ -57,6 +58,7 @@ def __init__(self, shell, data=None, spark_events=None):
         self.language = u""
         self.endpoint = None
         self.fatal_error = False
+        self.allow_retry_fatal = False
         self.fatal_error_message = u""
         if spark_events is None:
             spark_events = SparkEvents()
@@ -153,6 +155,11 @@ def help(self, line, cell="", local_ns=None):
       </ul>
     </td>
   </tr>
+  <tr>
+    <td>pretty</td>
+    <td>%%pretty</td>
+    <td>If the cell output is a dataframe, like <code>df.show()</code>, then it will pretty print the dataframe as an HTML table</td>
+  </tr>
 </table>
 """
         self.ipython_display.html(help_html)
@@ -258,14 +265,36 @@ def configure(self, line, cell="", local_ns=None):
     @wrap_unexpected_exceptions
     @handle_expected_exceptions
     def spark(self, line, cell="", local_ns=None):
-        if self._do_not_call_start_session(u""):
-            args = parse_argstring_or_throw(self.spark, line)
+        if not self._do_not_call_start_session(u""):
+            return
 
-            coerce = get_coerce_value(args.coerce)
+        args = parse_argstring_or_throw(self.spark, line)
+
+        coerce = get_coerce_value(args.coerce)
+
+        self.execute_spark(cell, args.output, args.samplemethod, args.maxrows, args.samplefraction, None, coerce)
+
+    @cell_magic
+    @needs_local_scope
+    @wrap_unexpected_exceptions
+    @handle_expected_exceptions
+    def pretty(self, line, cell="", local_ns=None):
+        """Evaluates a cell and converts dataframes in cell output to HTML tables."""
+        if not self._do_not_call_start_session(u""):
+            return 
+
+        def pretty_output_handler(out):
+            if cell_contains_dataframe(out):
+                self.ipython_display.html(CellOutputHtmlParser.to_html(out)) 
+            else:    
+                self.ipython_display.write(out)
+
+        so = SparkOutputHandler(html=self.ipython_display.html,
+                        text=pretty_output_handler,
+                        default=self.ipython_display.display)
+
+        self.execute_spark(cell, None, None, None, None, None, None, output_handler=so)
 
-            self.execute_spark(cell, args.output, args.samplemethod, args.maxrows, args.samplefraction, None, coerce)
-        else:
-            return
 
     @magic_arguments()
     @cell_magic
@@ -282,15 +311,15 @@ def spark(self, line, cell="", local_ns=None):
     @wrap_unexpected_exceptions
     @handle_expected_exceptions
     def sql(self, line, cell="", local_ns=None):
-        if self._do_not_call_start_session(""):
-            args = parse_argstring_or_throw(self.sql, line)
+        if not self._do_not_call_start_session(""):
+            return
 
-            coerce = get_coerce_value(args.coerce)
+        args = parse_argstring_or_throw(self.sql, line)
 
-            return self.execute_sqlquery(cell, args.samplemethod, args.maxrows, args.samplefraction,
-                                         None, args.output, args.quiet, coerce)
-        else:
-            return
+        coerce = get_coerce_value(args.coerce)
+
+        return self.execute_sqlquery(cell, args.samplemethod, args.maxrows, args.samplefraction,
+                                     None, args.output, args.quiet, coerce)
 
     @magic_arguments()
     @cell_magic
@@ -347,17 +376,19 @@ def _do_not_call_start_session(self, line, cell="", local_ns=None):
         # No need to add the handle_expected_exceptions decorator to this since we manually catch all
         # exceptions when starting the session.
 
-        if self.fatal_error:
+        if self.fatal_error and not self.allow_retry_fatal:
             self.ipython_display.send_error(self.fatal_error_message)
             return False
 
         if not self.session_started:
             skip = False
             properties = conf.get_session_properties(self.language)
-            self.session_started = True
 
             try:
                 self.spark_controller.add_session(self.session_name, self.endpoint, skip, properties)
+                self.session_started = True
+                self.fatal_error = False
+                self.fatal_error_message = u""
             except Exception as e:
                 self.fatal_error = True
                 self.fatal_error_message = conf.fatal_error_suggestion().format(e)
@@ -371,6 +402,11 @@ def _do_not_call_start_session(self, line, cell="", local_ns=None):
 
         return self.session_started
 
+    @cell_magic
+    def _do_not_call_allow_retry_fatal(self, line, cell="", local_ns=None):
+        # enable the flag to retry session creation
+        self.allow_retry_fatal = True
+
     @cell_magic
     @handle_expected_exceptions
     def _do_not_call_delete_session(self, line, cell="", local_ns=None):

sparkmagic/sparkmagic/livyclientlib/livysession.py

@@ -220,6 +220,9 @@ def endpoint(self):
     def is_final_status(status):
         return status in constants.FINAL_STATUS
 
+    def is_posted(self):
+        return self.status != constants.NOT_STARTED_SESSION_STATUS
+
     def delete(self):
         session_id = self.id
         self._spark_events.emit_session_deletion_start_event(self.guid, self.kind, session_id, self.status)

sparkmagic/sparkmagic/livyclientlib/sparkcontroller.py

@@ -87,8 +87,16 @@ def add_session(self, name, endpoint, skip_if_exists, properties):
             return
         http_client = self._http_client(endpoint)
         session = self._livy_session(http_client, properties, self.ipython_display)
-        self.session_manager.add_session(name, session)
-        session.start()
+
+        try:
+            session.start()
+        except:
+            if session.is_posted():
+                session.delete()
+            raise
+        else:
+            self.session_manager.add_session(name, session)
+
 
     def get_session_id_for_client(self, name):
         return self.session_manager.get_session_id_for_client(name)